sm-sitemap-plugin/sitemap_index_test.go

package sitemap_test

import (
	"context"
	"net/http"
	"regexp"
	"strings"
	"testing"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/sm-sitemap-plugin/updates"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"git.golem15.com/golem15/summercms/modules/pact"
	"github.com/go-gormigrate/gormigrate/v2"
)

// bootIndex boots the fake content plugin and the sitemap on a fresh
// database with an empty content store.
func bootIndex(t *testing.T, extra ...string) *sitemapEnv {
	t.Helper()
	contentStore.reset(t)
	return bootPlugins(t, append([]string{contentPluginID, "golem15.sitemap"}, extra...))
}

// rebuild runs the plugin's rebuild job once, as a worker would.
func (e *sitemapEnv) rebuild(t *testing.T) {
	t.Helper()
	for _, p := range e.plugins {
		hj, ok := p.(pact.HasJobs)
		if !ok || p.ID() != "golem15.sitemap" {
			continue
		}
		for _, j := range hj.Jobs() {
			if err := j.Work(context.Background(), classes.RebuildArgs{}); err != nil {
				t.Fatalf("rebuild job: %v", err)
			}
			return
		}
	}
	t.Fatal("the sitemap registers no rebuild job")
}

// setFileSize stores the file size setting.
func (e *sitemapEnv) setFileSize(t *testing.T, n int) {
	t.Helper()
	if err := e.gdb.Exec(`UPDATE golem15_sitemap_settings SET file_size = ? WHERE id = 1`, n).Error; err != nil {
		t.Fatal(err)
	}
}

// locs lists the <loc> values of a sitemap body in order.
func locs(body string) []string {
	var out []string
	for _, m := range regexp.MustCompile(`<loc>([^<]*)</loc>`).FindAllStringSubmatch(body, -1) {
		out = append(out, m[1])
	}
	return out
}

// TestSitemapIndex is the tracer of D-30: a content type's published
// records appear, after a rebuild job, in an index at /sitemap.xml that
// links per-group files of file_size addresses with absolute <loc> and a
// real <lastmod>; a draft is absent, no priority or change frequency is
// written, values are XML-escaped (T-33-09) and an unknown file is a 404.
func TestSitemapIndex(t *testing.T) {
	e := bootIndex(t)
	e.setFileSize(t, 2)
	base := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
	for i, slug := range []string{"a", "b", "c", "d"} {
		contentStore.put(pageType, page1(i+1, slug, base.Add(time.Duration(i)*time.Hour)))
	}
	contentStore.put(pageType, fakeRecord{ID: 5, Published: true, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: `/pages/fish&chips?x=<1>"`, Title: `Fish & "chips" <b>`}}})
	contentStore.put(pageType, fakeRecord{ID: 6, Published: false, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/draft", Title: "Draft"}}})

	if rec := e.get(t, "/sitemap.xml"); rec.Code != http.StatusOK || len(locs(rec.Body.String())) != 0 {
		t.Fatalf("index before the first rebuild: %d %s", rec.Code, rec.Body.String())
	}
	e.rebuild(t)

	rec := e.get(t, "/sitemap.xml")
	body := rec.Body.String()
	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" {
		t.Fatalf("GET /sitemap.xml: %d %q %s", rec.Code, rec.Header().Get("Content-Type"), body)
	}
	if !strings.HasPrefix(body, `<?xml version="1.0" encoding="UTF-8"?>`) || !strings.Contains(body, `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`) {
		t.Fatalf("index head:\n%s", body)
	}
	want := []string{
		"https://api.example.test/sitemap-pages-1.xml",
		"https://api.example.test/sitemap-pages-2.xml",
		"https://api.example.test/sitemap-pages-3.xml",
	}
	if got := locs(body); strings.Join(got, " ") != strings.Join(want, " ") {
		t.Fatalf("index locs = %v, want %v\n%s", got, want, body)
	}
	if n := strings.Count(body, "<lastmod>"); n != 3 {
		t.Fatalf("index has %d lastmod elements:\n%s", n, body)
	}
	if !strings.Contains(body, "<sitemap><loc>https://api.example.test/sitemap-pages-1.xml</loc><lastmod>2026-10-01T09:00:00Z</lastmod></sitemap>") {
		t.Fatalf("file 1 lastmod is not the newest of its addresses:\n%s", body)
	}

	rec = e.get(t, "/_sitemap/files/sitemap-pages-1.xml")
	file := rec.Body.String()
	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" {
		t.Fatalf("file 1: %d %v %s", rec.Code, rec.Header(), file)
	}
	if got := locs(file); strings.Join(got, " ") != "https://api.example.test/pages/a https://api.example.test/pages/b" {
		t.Fatalf("file 1 locs = %v\n%s", got, file)
	}
	if !strings.Contains(file, "<url><loc>https://api.example.test/pages/a</loc><lastmod>2026-10-01T08:00:00Z</lastmod></url>") {
		t.Fatalf("file 1 lacks a real lastmod:\n%s", file)
	}
	file3 := e.get(t, "/_sitemap/files/sitemap-pages-3.xml").Body.String()
	if !strings.Contains(file3, "<loc>https://api.example.test/pages/fish&amp;chips?x=&lt;1&gt;&#34;</loc>") {
		t.Fatalf("file 3 does not escape the address:\n%s", file3)
	}
	all := body + file + e.get(t, "/_sitemap/files/sitemap-pages-2.xml").Body.String() + file3
	for _, banned := range []string{"<priority>", "<changefreq>", "/pages/draft", "xml-stylesheet", "xmlns:xhtml"} {
		if strings.Contains(all, banned) {
			t.Errorf("the sitemap contains %q:\n%s", banned, all)
		}
	}
	for _, path := range []string{
		"/_sitemap/files/sitemap-pages-4.xml",
		"/_sitemap/files/sitemap-posts-1.xml",
		"/_sitemap/files/sitemap-pages-0.xml",
		"/_sitemap/files/other.xml",
		"/sitemap.xsl",
	} {
		if rec := e.get(t, path); rec.Code != http.StatusNotFound {
			t.Errorf("GET %s: %d, want 404", path, rec.Code)
		}
	}
}

// permSitemap is the permission of the sitemap settings.
const permSitemap = "golem15.sitemap.access_definitions"

// riverJobs counts the queued jobs of kind.
func (e *sitemapEnv) riverJobs(t *testing.T, kind string) int64 {
	t.Helper()
	var n int64
	if err := e.gdb.Raw(`SELECT COUNT(*) FROM river_job WHERE kind = ?`, kind).Scan(&n).Error; err != nil {
		t.Fatal(err)
	}
	return n
}

// TestSettingsSingleton drives the settings singleton (UI-SPEC C26): one
// tile opens it, the form has the Files, Content types and Additional
// addresses tabs, a save stores the values and dispatches a rebuild, the
// errors.* refusals answer 422 and leave the row as it was, a type switched
// off leaves the files with reason type and an additional address is listed
// in the pages group with its lastmod; S1 shows files.empty before the
// first build, files.building while the asked-for rebuild has not run, and
// the files with the left-out counts after it.
func TestSettingsSingleton(t *testing.T) {
	e := bootIndex(t)
	token := e.admin(t, permSitemap)
	const base = "/golem15/sitemap/settings"
	base0 := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
	contentStore.put(pageType, page1(1, "a", base0))
	contentStore.put(offerType, fakeRecord{ID: 1, Published: true, LastMod: base0, Locales: []pact.ContentLocale{{Locale: "en", URL: "/offers/x", Title: "Offer"}}})

	// One tile, linked to the singleton; the Phase 17 tiles are gone.
	rec := e.call(t, http.MethodGet, "/settings", token, nil)
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"controller":"golem15.sitemap.settings"`) ||
		strings.Contains(rec.Body.String(), "golem15.sitemap.definitions") || strings.Contains(rec.Body.String(), "sitemap_settings") {
		t.Fatalf("settings index: %d %s", rec.Code, rec.Body.String())
	}
	rec = e.call(t, http.MethodGet, base+"/singleton", token, nil)
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"id":1`) {
		t.Fatalf("GET singleton: %d %s", rec.Code, rec.Body.String())
	}
	rec = e.call(t, http.MethodGet, base+"/schema/form", token, nil)
	schema := rec.Body.String()
	if rec.Code != http.StatusOK || !strings.Contains(schema, `"singleton":true`) {
		t.Fatalf("form schema: %d %s", rec.Code, schema)
	}
	for _, want := range []string{`"site_url"`, `"file_size"`, `"extra_urls"`, `"type_acme_page_include"`, `"type_acme_page_slug"`, `"type_acme_page_title"`, `"type_acme_offer_include"`, "Files", "Content types", "Additional addresses"} {
		if !strings.Contains(schema, want) {
			t.Errorf("form schema lacks %s", want)
		}
	}
	files := func() string {
		t.Helper()
		rec := e.call(t, http.MethodGet, base+"/partials/sitemap_files", token, nil)
		if rec.Code != http.StatusOK {
			t.Fatalf("S1: %d %s", rec.Code, rec.Body.String())
		}
		return rec.Body.String()
	}
	if s1 := files(); !strings.Contains(s1, "No files yet.") || strings.Contains(s1, "Rebuilding now.") {
		t.Fatalf("S1 before the first build:\n%s", s1)
	}

	// Refusals leave the row as it was.
	for _, c := range []struct {
		name  string
		body  map[string]any
		field string
	}{
		{"file size 0", map[string]any{"file_size": 0}, "file_size"},
		{"file size 50 001", map[string]any{"file_size": 50001}, "file_size"},
		{"javascript site", map[string]any{"site_url": "javascript:alert(1)//"}, "site_url"},
		{"relative site", map[string]any{"site_url": "/x"}, "site_url"},
		{"bad address", map[string]any{"extra_urls": []map[string]any{{"url": "javascript:alert(1)", "lastmod": ""}}}, "extra_urls.0.url"},
		{"bad lastmod", map[string]any{"extra_urls": []map[string]any{{"url": "/x", "lastmod": "yesterday"}}}, "extra_urls.0.lastmod"},
		{"bad slug", map[string]any{"type_acme_page_slug": "Bad Slug"}, "type_acme_page_slug"},
		{"duplicate slug", map[string]any{"type_acme_page_slug": "things", "type_acme_offer_slug": "things"}, "type_acme_offer_slug"},
		{"reserved slug", map[string]any{"type_acme_page_slug": "images"}, "type_acme_page_slug"},
	} {
		rec := e.call(t, http.MethodPut, base+"/1", token, c.body)
		if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"`+c.field+`"`) {
			t.Errorf("%s: %d %s", c.name, rec.Code, rec.Body.String())
		}
	}
	var types int64
	e.gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_types`).Scan(&types)
	if types != 0 || e.riverJobs(t, classes.RebuildKind) != 0 {
		t.Fatalf("a refused save wrote %d type rows and queued %d rebuilds", types, e.riverJobs(t, classes.RebuildKind))
	}

	// A save stores everything and dispatches one rebuild.
	rec = e.call(t, http.MethodPut, base+"/1", token, map[string]any{
		"site_url":                "https://www.example.test/",
		"file_size":               500,
		"extra_urls":              []map[string]any{{"url": "/kontakt", "lastmod": "2026-09-30"}, {"url": "/o-nas", "lastmod": ""}},
		"type_acme_page_include":  true,
		"type_acme_page_slug":     "",
		"type_acme_page_title":    "Our pages",
		"type_acme_offer_include": false,
		"type_acme_offer_slug":    "offers",
		"type_acme_offer_title":   "",
	})
	if rec.Code != http.StatusOK {
		t.Fatalf("PUT settings: %d %s", rec.Code, rec.Body.String())
	}
	if n := e.riverJobs(t, classes.RebuildKind); n != 1 {
		t.Fatalf("the save queued %d rebuild jobs, want 1", n)
	}
	var row struct {
		SiteURL  string `gorm:"column:site_url"`
		FileSize int    `gorm:"column:file_size"`
	}
	e.gdb.Raw(`SELECT site_url, file_size FROM golem15_sitemap_settings WHERE id = 1`).Scan(&row)
	if row.SiteURL != "https://www.example.test/" || row.FileSize != 500 {
		t.Fatalf("stored settings = %+v", row)
	}
	var stored []struct {
		TypeKey string `gorm:"column:type_key"`
		Include bool   `gorm:"column:include"`
		Slug    string `gorm:"column:slug"`
		Titles  string `gorm:"column:titles"`
	}
	e.gdb.Raw(`SELECT type_key, include, slug, titles::text AS titles FROM golem15_sitemap_types ORDER BY type_key`).Scan(&stored)
	if len(stored) != 2 || stored[0].TypeKey != offerType || stored[0].Include || stored[0].Slug != "" ||
		stored[1].TypeKey != pageType || !stored[1].Include || !strings.Contains(stored[1].Titles, "Our pages") {
		t.Fatalf("type rows = %+v", stored)
	}
	if s1 := files(); !strings.Contains(s1, "Rebuilding now.") {
		t.Fatalf("S1 while the rebuild is queued:\n%s", s1)
	}
	rec = e.call(t, http.MethodGet, base+"/1", token, nil)
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"type_acme_page_title"`) || !strings.Contains(rec.Body.String(), "Our pages") {
		t.Fatalf("GET settings: %d %s", rec.Code, rec.Body.String())
	}

	e.rebuild(t)
	index := e.get(t, "/sitemap.xml").Body.String()
	if got := locs(index); strings.Join(got, " ") != "https://www.example.test/sitemap-pages-1.xml" {
		t.Fatalf("index after the save = %v\n%s", got, index)
	}
	pages := e.get(t, "/_sitemap/files/sitemap-pages-1.xml").Body.String()
	if got := locs(pages); strings.Join(got, " ") != "https://www.example.test/pages/a https://www.example.test/kontakt https://www.example.test/o-nas" {
		t.Fatalf("pages file = %v\n%s", got, pages)
	}
	if !strings.Contains(pages, "<loc>https://www.example.test/kontakt</loc><lastmod>2026-09-30T00:00:00Z</lastmod>") || !strings.Contains(pages, "<url><loc>https://www.example.test/o-nas</loc></url>") {
		t.Fatalf("additional addresses lack their lastmod:\n%s", pages)
	}
	var reason string
	e.gdb.Raw(`SELECT excluded_reason FROM golem15_sitemap_urls WHERE type_key = ?`, offerType).Scan(&reason)
	if reason != "type" {
		t.Fatalf("the switched-off type's row has reason %q", reason)
	}
	s1 := files()
	for _, want := range []string{"sitemap.xml", "sitemap-pages-1.xml", `"href":"/_sitemap/files/sitemap-pages-1.xml"`, `"href":"/sitemap.xml"`, "Left out of the sitemaps", "Content type turned off", "https://www.example.test/sitemap.xml"} {
		if !strings.Contains(s1, want) {
			t.Errorf("S1 lacks %q:\n%s", want, s1)
		}
	}
	if strings.Contains(s1, "Rebuilding now.") || strings.Contains(s1, "No files yet.") {
		t.Errorf("S1 after the rebuild:\n%s", s1)
	}

	// Without the permission every route answers 403.
	other := e.admin(t)
	for _, c := range []struct{ method, path string }{
		{http.MethodGet, base + "/singleton"},
		{http.MethodPut, base + "/1"},
		{http.MethodGet, base + "/partials/sitemap_files"},
	} {
		if rec := e.call(t, c.method, c.path, other, map[string]any{}); rec.Code != http.StatusForbidden {
			t.Errorf("%s %s without permission: %d %s", c.method, c.path, rec.Code, rec.Body.String())
		}
	}
}

// TestMigrateDefinitions pins the D-32 data move of migration
// 202610280002: the url items of the served definition (theme default)
// become the additional addresses as given (no lastmod, unsafe and repeated
// ones skipped), frontend_url becomes site_url and frontend_default_locale
// default_locale; reference items are dropped, the definitions table and
// the Phase 17 columns keep their data, and a rollback removes only the
// copied values.
func TestMigrateDefinitions(t *testing.T) {
	sqlDB := freshDB(t)
	gdb, err := lagoon.Use(t.Context(), sqlDB)
	if err != nil {
		t.Fatal(err)
	}
	all := updates.All()
	opts := &gormigrate.Options{TableName: "sitemap_migrations", IDColumnName: "id", IDColumnSize: 255, UseTransaction: true}
	upTo := 0
	for i, m := range all {
		if m.ID == "202610110002_create_golem15_sitemap_settings" {
			upTo = i + 1
		}
	}
	if upTo == 0 {
		t.Fatal("the Phase 17 settings migration is missing")
	}
	if err := gormigrate.New(gdb, opts, all[:upTo]).Migrate(); err != nil {
		t.Fatal(err)
	}
	items := `[{"type":"url","url":"/","changefreq":"always","priority":"1.0"},` +
		`{"type":"acme-post","reference":"7","nesting":true},` +
		`{"type":"url","url":"/o-nas"},` +
		`{"type":"url","url":"javascript:alert(1)"},` +
		`{"type":"url","url":"/"},` +
		`{"type":"url","url":"https://www.example.test/kontakt"}]`
	for _, stmt := range []struct {
		sql  string
		args []any
	}{
		{`INSERT INTO golem15_sitemap_definitions (theme, data) VALUES (?, ?)`, []any{"default", items}},
		{`INSERT INTO golem15_sitemap_definitions (theme, data) VALUES (?, ?)`, []any{"other", `[{"type":"url","url":"/other"}]`}},
		{`UPDATE golem15_sitemap_settings SET frontend_url = ?, frontend_default_locale = ?, priority_blog = ? WHERE id = 1`, []any{"https://www.example.test/", "pl", "0.7"}},
	} {
		if err := gdb.Exec(stmt.sql, stmt.args...).Error; err != nil {
			t.Fatal(err)
		}
	}
	if err := gormigrate.New(gdb, opts, all).Migrate(); err != nil {
		t.Fatal(err)
	}
	s, err := classes.LoadSettings(gdb)
	if err != nil {
		t.Fatal(err)
	}
	if got := extraURLs(s.Extra()); got != "/ /o-nas https://www.example.test/kontakt" {
		t.Fatalf("extra_urls = %q", got)
	}
	for _, e := range s.Extra() {
		if e.LastMod != "" {
			t.Fatalf("a copied address has a lastmod: %+v", e)
		}
	}
	if models.Text(s.SiteURL) != "https://www.example.test/" || models.Text(s.DefaultLocale) != "pl" || s.EffectiveFileSize() != 1000 {
		t.Fatalf("settings = %+v", s)
	}
	var defs int64
	gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_definitions`).Scan(&defs)
	var data string
	gdb.Raw(`SELECT data FROM golem15_sitemap_definitions WHERE theme = 'default'`).Scan(&data)
	if defs != 2 || data != items {
		t.Fatalf("the definitions table changed: %d rows, data %s", defs, data)
	}

	if err := gormigrate.New(gdb, opts, all).RollbackTo("202610110002_create_golem15_sitemap_settings"); err != nil {
		t.Fatal(err)
	}
	var old struct {
		FrontendURL string `gorm:"column:frontend_url"`
		Locale      string `gorm:"column:frontend_default_locale"`
		Blog        string `gorm:"column:priority_blog"`
	}
	gdb.Raw(`SELECT frontend_url, frontend_default_locale, priority_blog FROM golem15_sitemap_settings WHERE id = 1`).Scan(&old)
	if old.FrontendURL != "https://www.example.test/" || old.Locale != "pl" || old.Blog != "0.7" {
		t.Fatalf("the rollback changed the Phase 17 settings: %+v", old)
	}
	gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_definitions`).Scan(&defs)
	if defs != 2 {
		t.Fatalf("the rollback changed the definitions: %d rows", defs)
	}
	var columns int64
	gdb.Raw(`SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'golem15_sitemap_settings' AND column_name IN ('site_url', 'extra_urls')`).Scan(&columns)
	if columns != 0 {
		t.Fatalf("the rollback left %d copied columns", columns)
	}
}