a10ce2d9de
- sitemap-news.xml lists the articles (content kind article) first published in the last two days, newest first, at most 1000, with the publication name (news_publication, else app.name), the language of the locale, the publication date and the title - news_sources narrows it to type:<key> and category:<key>:<id> values; the Images and News tab lists the values to use in a partial - the index lists the file while news_enabled is on; otherwise it is 404
293 lines
13 KiB
Go
293 lines
13 KiB
Go
package sitemap_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/party"
|
|
)
|
|
|
|
// journalPluginID is a fake blog plugin with an article type (posts at
|
|
// /blog/<slug>) and a collection type (categories at /kategoria/<slug>)
|
|
// whose records live in contentStore. It knows nothing about the sitemap.
|
|
const journalPluginID = "acme.journal"
|
|
|
|
// The fake blog content types.
|
|
const (
|
|
postType = "acme-post"
|
|
categoryType = "acme-category"
|
|
)
|
|
|
|
// journalPlugin is acme.journal.
|
|
type journalPlugin struct{}
|
|
|
|
func (journalPlugin) ID() string { return journalPluginID }
|
|
func (journalPlugin) Requires() []string { return nil }
|
|
func (journalPlugin) Register(*backpack.App) error { return nil }
|
|
func (journalPlugin) Boot(*backpack.App) error { return nil }
|
|
|
|
func (journalPlugin) MenuItemTypes() []pact.MenuItemType {
|
|
typ := func(key, label, kind, pattern string) pact.MenuItemType {
|
|
return pact.MenuItemType{
|
|
Key: key,
|
|
Label: label,
|
|
DefaultPattern: pattern,
|
|
Placeholders: []string{"slug"},
|
|
Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) {
|
|
return nil, nil
|
|
},
|
|
Content: &pact.ContentDescriptor{
|
|
Model: key,
|
|
Kind: kind,
|
|
Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
|
|
return contentStore.page(key, in), nil
|
|
},
|
|
Describe: func(context.Context, pact.ContentDescribe) (*pact.ContentRecord, error) {
|
|
return nil, nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
return []pact.MenuItemType{
|
|
typ(postType, "Posts", pact.ContentKindArticle, "/blog/:slug"),
|
|
typ(categoryType, "Categories", pact.ContentKindCollection, "/kategoria/:slug"),
|
|
}
|
|
}
|
|
|
|
var _ pact.HasMenuItemTypes = journalPlugin{}
|
|
|
|
func init() {
|
|
party.Register(journalPlugin{})
|
|
}
|
|
|
|
// bootJournal boots the fake blog plugin and the sitemap on a fresh
|
|
// database with an empty content store; overrides set config keys.
|
|
func bootJournal(t *testing.T, overrides ...map[string]any) *sitemapEnv {
|
|
t.Helper()
|
|
contentStore.reset(t)
|
|
return bootPlugins(t, []string{journalPluginID, "golem15.sitemap"}, overrides...)
|
|
}
|
|
|
|
// post is a published post in one locale at /blog/<slug>, first published
|
|
// at published, with images.
|
|
func post(id int, slug, locale string, published time.Time, images ...string) fakeRecord {
|
|
l := pact.ContentLocale{Locale: locale, URL: "/blog/" + slug, Title: "Post " + slug}
|
|
for i, img := range images {
|
|
if i == 0 {
|
|
l.Image = img
|
|
}
|
|
l.Images = append(l.Images, pact.ContentImage{URL: img})
|
|
}
|
|
return fakeRecord{ID: id, Published: true, LastMod: published, PublishedAt: &published, Locales: []pact.ContentLocale{l}}
|
|
}
|
|
|
|
// exec runs one SQL statement on the test database.
|
|
func (e *sitemapEnv) exec(t *testing.T, sql string, args ...any) {
|
|
t.Helper()
|
|
if err := e.gdb.Exec(sql, args...).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// imageLocs lists the image:loc values of an image sitemap body.
|
|
func imageLocs(body string) []string {
|
|
var out []string
|
|
for _, m := range regexp.MustCompile(`<image:loc>([^<]*)</image:loc>`).FindAllStringSubmatch(body, -1) {
|
|
out = append(out, m[1])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestImageSitemap is the tracer of D-31's image sitemap: a post's
|
|
// featured and body images appear, after a rebuild, in sitemap-images-1.xml
|
|
// linked from the index: the page's <loc> and one image:image/image:loc per
|
|
// image (featured first, repeated and unsafe addresses dropped, paths made
|
|
// absolute), only those two tags, at most 1000 per page; a page without
|
|
// images is not listed; images_enabled off takes the files out of the index
|
|
// and answers 404.
|
|
func TestImageSitemap(t *testing.T) {
|
|
e := bootJournal(t)
|
|
at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
|
|
contentStore.put(postType, post(1, "a", "en", at, "/media/a.jpg", "/media/a.jpg", "https://cdn.example.test/b.png", "javascript:alert(1)", "/media/c&d.webp"))
|
|
contentStore.put(postType, post(2, "b", "en", at.Add(time.Hour)))
|
|
many := make([]string, 1200)
|
|
for i := range many {
|
|
many[i] = fmt.Sprintf("/media/many-%d.jpg", i)
|
|
}
|
|
contentStore.put(postType, post(3, "c", "en", at.Add(2*time.Hour), many...))
|
|
e.rebuild(t)
|
|
|
|
index := e.get(t, "/sitemap.xml").Body.String()
|
|
if got := locs(index); strings.Join(got, " ") != "https://api.example.test/sitemap-posts-1.xml https://api.example.test/sitemap-images-1.xml" {
|
|
t.Fatalf("index locs = %v\n%s", got, index)
|
|
}
|
|
rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml")
|
|
body := rec.Body.String()
|
|
if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" {
|
|
t.Fatalf("image file: %d %v %s", rec.Code, rec.Header(), body)
|
|
}
|
|
if !strings.Contains(body, `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">`) {
|
|
t.Fatalf("image file head:\n%s", body[:min(len(body), 300)])
|
|
}
|
|
if got := locs(body); strings.Join(got, " ") != "https://api.example.test/blog/a https://api.example.test/blog/c" {
|
|
t.Fatalf("image file pages = %v", got)
|
|
}
|
|
first := body[strings.Index(body, "<url><loc>https://api.example.test/blog/a</loc>"):]
|
|
first = first[:strings.Index(first, "</url>")]
|
|
if got := imageLocs(first); strings.Join(got, " ") != "https://api.example.test/media/a.jpg https://cdn.example.test/b.png https://api.example.test/media/c&d.webp" {
|
|
t.Fatalf("post a images = %v\n%s", got, first)
|
|
}
|
|
if n := len(imageLocs(body)); n != 3+1000 {
|
|
t.Fatalf("image file lists %d images, want 1003", n)
|
|
}
|
|
for _, banned := range []string{"image:caption", "image:title", "javascript:", "<lastmod>", "xhtml"} {
|
|
if strings.Contains(body, banned) {
|
|
t.Errorf("image file contains %q", banned)
|
|
}
|
|
}
|
|
if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound {
|
|
t.Errorf("image file 2: %d", rec.Code)
|
|
}
|
|
|
|
// Stable numbering: with a file size of 1 the second page with images
|
|
// is renumbered into its own file once, and a page that loses its images
|
|
// leaves a hole instead of moving the later file.
|
|
e.setFileSize(t, 1)
|
|
e.rebuild(t)
|
|
if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" {
|
|
t.Fatalf("image file 2 after the size change = %v", got)
|
|
}
|
|
contentStore.put(postType, post(1, "a", "en", at))
|
|
e.rebuild(t)
|
|
if rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml"); rec.Code != http.StatusNotFound {
|
|
t.Errorf("image file 1 without images: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" {
|
|
t.Fatalf("image file 2 moved = %v", got)
|
|
}
|
|
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET images_enabled = FALSE WHERE id = 1`)
|
|
if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-images") {
|
|
t.Fatalf("index with the image sitemap off:\n%s", index)
|
|
}
|
|
if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound {
|
|
t.Errorf("image file with the image sitemap off: %d", rec.Code)
|
|
}
|
|
}
|
|
|
|
// newsLocs lists the <loc> values of a News sitemap.
|
|
func newsLocs(body string) []string { return locs(body) }
|
|
|
|
// TestNewsSitemap pins D-31's News sitemap (RESEARCH Pitfall 9): with
|
|
// news_enabled the index lists sitemap-news.xml with the articles first
|
|
// published in the last two days (1 and 47 hours ago, not 49), newest
|
|
// first, never a record of another kind; each with the publication name
|
|
// (news_publication, else app.name), the language of its locale, the
|
|
// publication date and the title; news_sources narrows it to the listed
|
|
// types and categories; more than 1000 candidates list the newest 1000;
|
|
// the settings form refuses a malformed source and lists the values to
|
|
// use; news_enabled off takes the file out of the index and answers 404.
|
|
func TestNewsSitemap(t *testing.T) {
|
|
e := bootJournal(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
fresh := post(1, "fresh", "pl", now.Add(-time.Hour))
|
|
fresh.Locales[0].Title = `Grzyby & "huby"`
|
|
fresh.Parents = []pact.ContentRef{{Type: categoryType, ID: "3"}}
|
|
contentStore.put(postType, fresh)
|
|
contentStore.put(postType, post(2, "older", "en", now.Add(-47*time.Hour)))
|
|
contentStore.put(postType, post(3, "stale", "en", now.Add(-49*time.Hour)))
|
|
cat := now.Add(-time.Hour)
|
|
contentStore.put(categoryType, fakeRecord{ID: 3, Published: true, LastMod: cat, PublishedAt: &cat, Locales: []pact.ContentLocale{{Locale: "pl", URL: "/kategoria/lecznicze", Title: "Lecznicze"}}})
|
|
e.rebuild(t)
|
|
|
|
// Off by default: not in the index, the file is a 404.
|
|
if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") {
|
|
t.Fatalf("index with the News sitemap off:\n%s", index)
|
|
}
|
|
if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound {
|
|
t.Fatalf("News file while off: %d", rec.Code)
|
|
}
|
|
|
|
// The settings form: the Images and News tab, a malformed source is
|
|
// refused, the partial lists the values.
|
|
token := e.admin(t, permSitemap)
|
|
const base = "/golem15/sitemap/settings"
|
|
schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
|
|
for _, want := range []string{`"images_enabled"`, `"news_enabled"`, `"news_publication"`, `"news_sources"`, "Images and News"} {
|
|
if !strings.Contains(schema, want) {
|
|
t.Errorf("form schema lacks %s", want)
|
|
}
|
|
}
|
|
for _, bad := range []any{[]string{"posts"}, []string{"category:acme-category"}, []string{"type:a b"}} {
|
|
if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_sources": bad}); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"news_sources"`) {
|
|
t.Errorf("news_sources %v: %d %s", bad, rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
rec := e.call(t, http.MethodGet, base+"/partials/news_sources", token, nil)
|
|
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "type:acme-post") || !strings.Contains(rec.Body.String(), "category:acme-category:3") || !strings.Contains(rec.Body.String(), "Categories: Lecznicze") {
|
|
t.Fatalf("news_sources partial: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_enabled": true, "news_sources": []string{}}); rec.Code != http.StatusOK {
|
|
t.Fatalf("enable News: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
index := e.get(t, "/sitemap.xml").Body.String()
|
|
if !strings.Contains(index, "<loc>https://api.example.test/sitemap-news.xml</loc>") {
|
|
t.Fatalf("index lacks the News file:\n%s", index)
|
|
}
|
|
rec = e.get(t, "/_sitemap/files/sitemap-news.xml")
|
|
body := rec.Body.String()
|
|
if rec.Code != http.StatusOK || rec.Header().Get("X-Robots-Tag") != "noindex" || !strings.Contains(body, `xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"`) {
|
|
t.Fatalf("News file: %d %v\n%s", rec.Code, rec.Header(), body)
|
|
}
|
|
if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh https://api.example.test/blog/older" {
|
|
t.Fatalf("News articles = %v\n%s", got, body)
|
|
}
|
|
wantFresh := "<url><loc>https://api.example.test/blog/fresh</loc><news:news><news:publication><news:name>sitemap-smoke</news:name><news:language>pl</news:language></news:publication><news:publication_date>" +
|
|
now.Add(-time.Hour).Format(time.RFC3339) + "</news:publication_date><news:title>Grzyby & "huby"</news:title></news:news></url>"
|
|
if !strings.Contains(body, wantFresh) {
|
|
t.Fatalf("News entry of the fresh post:\n%s\nwant %s", body, wantFresh)
|
|
}
|
|
|
|
// The publication name setting and the sources.
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET news_publication = 'Gazeta', news_sources = '["category:acme-category:3"]' WHERE id = 1`)
|
|
body = e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()
|
|
if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh" || !strings.Contains(body, "<news:name>Gazeta</news:name>") {
|
|
t.Fatalf("News with a category source = %v\n%s", got, body)
|
|
}
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-post"]' WHERE id = 1`)
|
|
if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 2 {
|
|
t.Fatalf("News with a type source = %v", got)
|
|
}
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-category"]' WHERE id = 1`)
|
|
if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 0 {
|
|
t.Fatalf("a collection type in the News sitemap = %v", got)
|
|
}
|
|
|
|
// More than 1000 candidates: the newest 1000.
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '[]' WHERE id = 1`)
|
|
for i := 0; i < 1005; i++ {
|
|
contentStore.put(postType, post(100+i, fmt.Sprintf("bulk-%d", i), "en", now.Add(-2*time.Hour-time.Duration(i)*time.Minute)))
|
|
}
|
|
e.rebuild(t)
|
|
got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String())
|
|
if len(got) != 1000 || got[0] != "https://api.example.test/blog/fresh" || got[999] != "https://api.example.test/blog/bulk-998" {
|
|
t.Fatalf("News with 1007 candidates lists %d, first %v last %v", len(got), got[:1], got[len(got)-1:])
|
|
}
|
|
|
|
e.exec(t, `UPDATE golem15_sitemap_settings SET news_enabled = FALSE WHERE id = 1`)
|
|
if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") {
|
|
t.Fatalf("index after turning News off:\n%s", index)
|
|
}
|
|
if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound {
|
|
t.Fatalf("News file after turning it off: %d", rec.Code)
|
|
}
|
|
}
|