sm-sitemap-plugin/sitemap_extras_test.go

package sitemap_test

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/bouncer"
	"git.golem15.com/golem15/summercms/modules/cabana"
	"git.golem15.com/golem15/summercms/modules/conga"
	"git.golem15.com/golem15/summercms/modules/fetchguard"
	"git.golem15.com/golem15/summercms/modules/pact"
	"git.golem15.com/golem15/summercms/modules/party"
	"git.golem15.com/golem15/summercms/modules/pier"
	"git.golem15.com/golem15/summercms/modules/surf"
)

// journalPluginID is a fake blog plugin with an article type (posts at
// /blog/<slug>) and a collection type (categories at /kategoria/<slug>)
// whose records live in contentStore. It knows nothing about the sitemap.
const journalPluginID = "acme.journal"

// The fake blog content types.
const (
	postType     = "acme-post"
	categoryType = "acme-category"
)

// journalPlugin is acme.journal.
type journalPlugin struct{}

func (journalPlugin) ID() string                   { return journalPluginID }
func (journalPlugin) Requires() []string           { return nil }
func (journalPlugin) Register(*backpack.App) error { return nil }
func (journalPlugin) Boot(*backpack.App) error     { return nil }

func (journalPlugin) MenuItemTypes() []pact.MenuItemType {
	typ := func(key, label, kind, pattern string) pact.MenuItemType {
		return pact.MenuItemType{
			Key:            key,
			Label:          label,
			DefaultPattern: pattern,
			Placeholders:   []string{"slug"},
			Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) {
				return nil, nil
			},
			Content: &pact.ContentDescriptor{
				Model: key,
				Kind:  kind,
				Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
					return contentStore.page(key, in), nil
				},
				Describe: func(context.Context, pact.ContentDescribe) (*pact.ContentRecord, error) {
					return nil, nil
				},
			},
		}
	}
	return []pact.MenuItemType{
		typ(postType, "Posts", pact.ContentKindArticle, "/blog/:slug"),
		typ(categoryType, "Categories", pact.ContentKindCollection, "/kategoria/:slug"),
	}
}

var _ pact.HasMenuItemTypes = journalPlugin{}

func init() {
	party.Register(journalPlugin{})
}

// bootJournal boots the fake blog plugin and the sitemap on a fresh
// database with an empty content store; overrides set config keys.
func bootJournal(t *testing.T, overrides ...map[string]any) *sitemapEnv {
	t.Helper()
	contentStore.reset(t)
	return bootPlugins(t, []string{journalPluginID, "golem15.sitemap"}, overrides...)
}

// post is a published post in one locale at /blog/<slug>, first published
// at published, with images.
func post(id int, slug, locale string, published time.Time, images ...string) fakeRecord {
	l := pact.ContentLocale{Locale: locale, URL: "/blog/" + slug, Title: "Post " + slug}
	for i, img := range images {
		if i == 0 {
			l.Image = img
		}
		l.Images = append(l.Images, pact.ContentImage{URL: img})
	}
	return fakeRecord{ID: id, Published: true, LastMod: published, PublishedAt: &published, Locales: []pact.ContentLocale{l}}
}

// exec runs one SQL statement on the test database.
func (e *sitemapEnv) exec(t *testing.T, sql string, args ...any) {
	t.Helper()
	if err := e.gdb.Exec(sql, args...).Error; err != nil {
		t.Fatal(err)
	}
}

// imageLocs lists the image:loc values of an image sitemap body.
func imageLocs(body string) []string {
	var out []string
	for _, m := range regexp.MustCompile(`<image:loc>([^<]*)</image:loc>`).FindAllStringSubmatch(body, -1) {
		out = append(out, m[1])
	}
	return out
}

// TestImageSitemap is the tracer of D-31's image sitemap: a post's
// featured and body images appear, after a rebuild, in sitemap-images-1.xml
// linked from the index: the page's <loc> and one image:image/image:loc per
// image (featured first, repeated and unsafe addresses dropped, paths made
// absolute), only those two tags, at most 1000 per page; a page without
// images is not listed; images_enabled off takes the files out of the index
// and answers 404.
func TestImageSitemap(t *testing.T) {
	e := bootJournal(t)
	at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
	contentStore.put(postType, post(1, "a", "en", at, "/media/a.jpg", "/media/a.jpg", "https://cdn.example.test/b.png", "javascript:alert(1)", "/media/c&d.webp"))
	contentStore.put(postType, post(2, "b", "en", at.Add(time.Hour)))
	many := make([]string, 1200)
	for i := range many {
		many[i] = fmt.Sprintf("/media/many-%d.jpg", i)
	}
	contentStore.put(postType, post(3, "c", "en", at.Add(2*time.Hour), many...))
	e.rebuild(t)

	index := e.get(t, "/sitemap.xml").Body.String()
	if got := locs(index); strings.Join(got, " ") != "https://api.example.test/sitemap-posts-1.xml https://api.example.test/sitemap-images-1.xml" {
		t.Fatalf("index locs = %v\n%s", got, index)
	}
	rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml")
	body := rec.Body.String()
	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" {
		t.Fatalf("image file: %d %v %s", rec.Code, rec.Header(), body)
	}
	if !strings.Contains(body, `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">`) {
		t.Fatalf("image file head:\n%s", body[:min(len(body), 300)])
	}
	if got := locs(body); strings.Join(got, " ") != "https://api.example.test/blog/a https://api.example.test/blog/c" {
		t.Fatalf("image file pages = %v", got)
	}
	first := body[strings.Index(body, "<url><loc>https://api.example.test/blog/a</loc>"):]
	first = first[:strings.Index(first, "</url>")]
	if got := imageLocs(first); strings.Join(got, " ") != "https://api.example.test/media/a.jpg https://cdn.example.test/b.png https://api.example.test/media/c&amp;d.webp" {
		t.Fatalf("post a images = %v\n%s", got, first)
	}
	if n := len(imageLocs(body)); n != 3+1000 {
		t.Fatalf("image file lists %d images, want 1003", n)
	}
	for _, banned := range []string{"image:caption", "image:title", "javascript:", "<lastmod>", "xhtml"} {
		if strings.Contains(body, banned) {
			t.Errorf("image file contains %q", banned)
		}
	}
	if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound {
		t.Errorf("image file 2: %d", rec.Code)
	}

	// Stable numbering: with a file size of 1 the second page with images
	// is renumbered into its own file once, and a page that loses its images
	// leaves a hole instead of moving the later file.
	e.setFileSize(t, 1)
	e.rebuild(t)
	if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" {
		t.Fatalf("image file 2 after the size change = %v", got)
	}
	contentStore.put(postType, post(1, "a", "en", at))
	e.rebuild(t)
	if rec := e.get(t, "/_sitemap/files/sitemap-images-1.xml"); rec.Code != http.StatusNotFound {
		t.Errorf("image file 1 without images: %d %s", rec.Code, rec.Body.String())
	}
	if got := locs(e.get(t, "/_sitemap/files/sitemap-images-2.xml").Body.String()); strings.Join(got, " ") != "https://api.example.test/blog/c" {
		t.Fatalf("image file 2 moved = %v", got)
	}

	e.exec(t, `UPDATE golem15_sitemap_settings SET images_enabled = FALSE WHERE id = 1`)
	if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-images") {
		t.Fatalf("index with the image sitemap off:\n%s", index)
	}
	if rec := e.get(t, "/_sitemap/files/sitemap-images-2.xml"); rec.Code != http.StatusNotFound {
		t.Errorf("image file with the image sitemap off: %d", rec.Code)
	}
}

// newsLocs lists the <loc> values of a News sitemap.
func newsLocs(body string) []string { return locs(body) }

// TestNewsSitemap pins D-31's News sitemap (RESEARCH Pitfall 9): with
// news_enabled the index lists sitemap-news.xml with the articles first
// published in the last two days (1 and 47 hours ago, not 49), newest
// first, never a record of another kind; each with the publication name
// (news_publication, else app.name), the language of its locale, the
// publication date and the title; news_sources narrows it to the listed
// types and categories; more than 1000 candidates list the newest 1000;
// the settings form refuses a malformed source and lists the values to
// use; news_enabled off takes the file out of the index and answers 404.
func TestNewsSitemap(t *testing.T) {
	e := bootJournal(t)
	now := time.Now().UTC().Truncate(time.Second)
	fresh := post(1, "fresh", "pl", now.Add(-time.Hour))
	fresh.Locales[0].Title = `Grzyby & "huby"`
	fresh.Parents = []pact.ContentRef{{Type: categoryType, ID: "3"}}
	contentStore.put(postType, fresh)
	contentStore.put(postType, post(2, "older", "en", now.Add(-47*time.Hour)))
	contentStore.put(postType, post(3, "stale", "en", now.Add(-49*time.Hour)))
	cat := now.Add(-time.Hour)
	contentStore.put(categoryType, fakeRecord{ID: 3, Published: true, LastMod: cat, PublishedAt: &cat, Locales: []pact.ContentLocale{{Locale: "pl", URL: "/kategoria/lecznicze", Title: "Lecznicze"}}})
	e.rebuild(t)

	// Off by default: not in the index, the file is a 404.
	if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") {
		t.Fatalf("index with the News sitemap off:\n%s", index)
	}
	if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound {
		t.Fatalf("News file while off: %d", rec.Code)
	}

	// The settings form: the Images and News tab, a malformed source is
	// refused, the partial lists the values.
	token := e.admin(t, permSitemap)
	const base = "/golem15/sitemap/settings"
	schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
	for _, want := range []string{`"images_enabled"`, `"news_enabled"`, `"news_publication"`, `"news_sources"`, "Images and News"} {
		if !strings.Contains(schema, want) {
			t.Errorf("form schema lacks %s", want)
		}
	}
	for _, bad := range []any{[]string{"posts"}, []string{"category:acme-category"}, []string{"type:a b"}} {
		if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_sources": bad}); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"news_sources"`) {
			t.Errorf("news_sources %v: %d %s", bad, rec.Code, rec.Body.String())
		}
	}
	rec := e.call(t, http.MethodGet, base+"/partials/news_sources", token, nil)
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "type:acme-post") || !strings.Contains(rec.Body.String(), "category:acme-category:3") || !strings.Contains(rec.Body.String(), "Categories: Lecznicze") {
		t.Fatalf("news_sources partial: %d %s", rec.Code, rec.Body.String())
	}
	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"news_enabled": true, "news_sources": []string{}}); rec.Code != http.StatusOK {
		t.Fatalf("enable News: %d %s", rec.Code, rec.Body.String())
	}

	index := e.get(t, "/sitemap.xml").Body.String()
	if !strings.Contains(index, "<loc>https://api.example.test/sitemap-news.xml</loc>") {
		t.Fatalf("index lacks the News file:\n%s", index)
	}
	rec = e.get(t, "/_sitemap/files/sitemap-news.xml")
	body := rec.Body.String()
	if rec.Code != http.StatusOK || rec.Header().Get("X-Robots-Tag") != "noindex" || !strings.Contains(body, `xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"`) {
		t.Fatalf("News file: %d %v\n%s", rec.Code, rec.Header(), body)
	}
	if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh https://api.example.test/blog/older" {
		t.Fatalf("News articles = %v\n%s", got, body)
	}
	wantFresh := "<url><loc>https://api.example.test/blog/fresh</loc><news:news><news:publication><news:name>sitemap-smoke</news:name><news:language>pl</news:language></news:publication><news:publication_date>" +
		now.Add(-time.Hour).Format(time.RFC3339) + "</news:publication_date><news:title>Grzyby &amp; &#34;huby&#34;</news:title></news:news></url>"
	if !strings.Contains(body, wantFresh) {
		t.Fatalf("News entry of the fresh post:\n%s\nwant %s", body, wantFresh)
	}

	// The publication name setting and the sources.
	e.exec(t, `UPDATE golem15_sitemap_settings SET news_publication = 'Gazeta', news_sources = '["category:acme-category:3"]' WHERE id = 1`)
	body = e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()
	if got := newsLocs(body); strings.Join(got, " ") != "https://api.example.test/blog/fresh" || !strings.Contains(body, "<news:name>Gazeta</news:name>") {
		t.Fatalf("News with a category source = %v\n%s", got, body)
	}
	e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-post"]' WHERE id = 1`)
	if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 2 {
		t.Fatalf("News with a type source = %v", got)
	}
	e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '["type:acme-category"]' WHERE id = 1`)
	if got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String()); len(got) != 0 {
		t.Fatalf("a collection type in the News sitemap = %v", got)
	}

	// More than 1000 candidates: the newest 1000.
	e.exec(t, `UPDATE golem15_sitemap_settings SET news_sources = '[]' WHERE id = 1`)
	for i := 0; i < 1005; i++ {
		contentStore.put(postType, post(100+i, fmt.Sprintf("bulk-%d", i), "en", now.Add(-2*time.Hour-time.Duration(i)*time.Minute)))
	}
	e.rebuild(t)
	got := newsLocs(e.get(t, "/_sitemap/files/sitemap-news.xml").Body.String())
	if len(got) != 1000 || got[0] != "https://api.example.test/blog/fresh" || got[999] != "https://api.example.test/blog/bulk-998" {
		t.Fatalf("News with 1007 candidates lists %d, first %v last %v", len(got), got[:1], got[len(got)-1:])
	}

	e.exec(t, `UPDATE golem15_sitemap_settings SET news_enabled = FALSE WHERE id = 1`)
	if index := e.get(t, "/sitemap.xml").Body.String(); strings.Contains(index, "sitemap-news.xml") {
		t.Fatalf("index after turning News off:\n%s", index)
	}
	if rec := e.get(t, "/_sitemap/files/sitemap-news.xml"); rec.Code != http.StatusNotFound {
		t.Fatalf("News file after turning it off: %d", rec.Code)
	}
}

// job returns the sitemap's registered job of kind.
func (e *sitemapEnv) job(t *testing.T, kind string) pact.Job {
	t.Helper()
	for _, p := range e.plugins {
		hj, ok := p.(pact.HasJobs)
		if !ok || p.ID() != "golem15.sitemap" {
			continue
		}
		for _, j := range hj.Jobs() {
			if info, ok := conga.Describe(j); ok && info.Kind == kind {
				return j
			}
		}
	}
	t.Fatalf("the sitemap registers no job %s", kind)
	return nil
}

// fakeIndexNow is the IndexNow engine of a test: it records each request
// and answers status.
type fakeIndexNow struct {
	mu       sync.Mutex
	status   int
	requests []*http.Request
	bodies   []map[string]any
}

func (f *fakeIndexNow) RoundTrip(r *http.Request) (*http.Response, error) {
	raw, _ := io.ReadAll(r.Body)
	body := map[string]any{}
	_ = json.Unmarshal(raw, &body)
	f.mu.Lock()
	f.requests = append(f.requests, r)
	f.bodies = append(f.bodies, body)
	status := f.status
	f.mu.Unlock()
	return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}

// queuedIndexNow decodes the URL lists of the queued IndexNow jobs, oldest
// first.
func (e *sitemapEnv) queuedIndexNow(t *testing.T) [][]string {
	t.Helper()
	var rows []string
	if err := e.gdb.Raw(`SELECT args::text FROM river_job WHERE kind = ? ORDER BY id`, classes.IndexNowKind).Scan(&rows).Error; err != nil {
		t.Fatal(err)
	}
	out := make([][]string, 0, len(rows))
	for _, raw := range rows {
		var args classes.IndexNowArgs
		if err := json.Unmarshal([]byte(raw), &args); err != nil {
			t.Fatal(err)
		}
		out = append(out, args.URLs)
	}
	return out
}

// indexNowKey reads the stored key.
func (e *sitemapEnv) indexNowKey(t *testing.T) string {
	t.Helper()
	var key *string
	if err := e.gdb.Raw(`SELECT indexnow_key FROM golem15_sitemap_settings WHERE id = 1`).Scan(&key).Error; err != nil {
		t.Fatal(err)
	}
	if key == nil {
		return ""
	}
	return *key
}

// TestIndexNow drives D-31's IndexNow in production: enabling and saving
// generates a 32-hex key whose file answers text/plain (another name 404);
// a rebuild with a new and a removed address queues one job with both; the
// job posts {host, key, urlList} to api.indexnow.org and records the
// answer (202); 403 and 422 are recorded without retry, 429 fails the job
// so it retries; S2 shows each state; an address of another host is never
// sent; a new key makes the old file 404. Outside production nothing is
// queued or sent.
func TestIndexNow(t *testing.T) {
	e := bootJournal(t, map[string]any{envOverride: "production"})
	token := e.admin(t, permSitemap)
	const base = "/golem15/sitemap/settings"
	s2 := func() string {
		t.Helper()
		rec := e.call(t, http.MethodGet, base+"/partials/indexnow_status", token, nil)
		if rec.Code != http.StatusOK {
			t.Fatalf("S2: %d %s", rec.Code, rec.Body.String())
		}
		return rec.Body.String()
	}
	if got := s2(); !strings.Contains(got, "A key is generated when you turn IndexNow on and save.") || !strings.Contains(got, "No notification sent yet.") {
		t.Fatalf("S2 before a key:\n%s", got)
	}
	schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
	for _, want := range []string{`"indexnow_enabled"`, `"indexnow_new_key"`, `"indexnow_status"`, "IndexNow"} {
		if !strings.Contains(schema, want) {
			t.Errorf("form schema lacks %s", want)
		}
	}

	// A rebuild before IndexNow is on queues nothing.
	at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
	contentStore.put(postType, post(1, "a", "en", at))
	e.rebuild(t)
	if q := e.queuedIndexNow(t); len(q) != 0 {
		t.Fatalf("queued while off: %v", q)
	}

	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true}); rec.Code != http.StatusOK {
		t.Fatalf("enable IndexNow: %d %s", rec.Code, rec.Body.String())
	}
	key := e.indexNowKey(t)
	if !regexp.MustCompile(`^[0-9a-f]{32}$`).MatchString(key) {
		t.Fatalf("key = %q", key)
	}
	rec := e.get(t, "/_sitemap/indexnow/"+key+".txt")
	if rec.Code != http.StatusOK || rec.Body.String() != key || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
		t.Fatalf("key file: %d %q %v", rec.Code, rec.Body.String(), rec.Header())
	}
	for _, name := range []string{"0123456789abcdef0123456789abcdef.txt", "short.txt", key + ".xml"} {
		if rec := e.get(t, "/_sitemap/indexnow/"+name); rec.Code != http.StatusNotFound {
			t.Errorf("key file %s: %d", name, rec.Code)
		}
	}
	if got := s2(); !strings.Contains(got, "Key: "+key+", served at https://api.example.test/"+key+".txt") {
		t.Fatalf("S2 with a key:\n%s", got)
	}

	// One new and one removed address: one job with both.
	contentStore.put(postType, post(2, "b", "en", at))
	contentStore.remove(postType, 1)
	e.rebuild(t)
	q := e.queuedIndexNow(t)
	if len(q) != 1 || strings.Join(q[0], " ") != "https://api.example.test/blog/b https://api.example.test/blog/a" {
		t.Fatalf("queued = %v", q)
	}

	engine := &fakeIndexNow{status: http.StatusAccepted}
	ctx := fetchguard.WithTransport(t.Context(), engine)
	job := e.job(t, classes.IndexNowKind)
	args := classes.IndexNowArgs{URLs: append(q[0], "https://evil.test/x", "http://api.example.test.evil.test/y")}
	if err := job.Work(ctx, args); err != nil {
		t.Fatalf("IndexNow job: %v", err)
	}
	if len(engine.requests) != 1 {
		t.Fatalf("%d requests", len(engine.requests))
	}
	req, body := engine.requests[0], engine.bodies[0]
	if req.Method != http.MethodPost || req.URL.String() != "https://api.indexnow.org/indexnow" || !strings.HasPrefix(req.Header.Get("Content-Type"), "application/json") {
		t.Fatalf("request = %s %s %v", req.Method, req.URL, req.Header)
	}
	list, _ := body["urlList"].([]any)
	if body["host"] != "api.example.test" || body["key"] != key || len(list) != 2 || body["keyLocation"] != nil {
		t.Fatalf("body = %v", body)
	}
	var last struct {
		Status int `gorm:"column:indexnow_last_status"`
		Count  int `gorm:"column:indexnow_last_count"`
	}
	e.gdb.Raw(`SELECT indexnow_last_status, indexnow_last_count FROM golem15_sitemap_settings WHERE id = 1`).Scan(&last)
	if last.Status != 202 || last.Count != 2 {
		t.Fatalf("recorded = %+v", last)
	}
	if got := s2(); !strings.Contains(got, ", 2 addresses, answer 202.") || strings.Contains(got, "summer-callout--warning") {
		t.Fatalf("S2 after 202:\n%s", got)
	}

	for _, c := range []struct {
		status int
		retry  bool
		want   string
	}{
		{http.StatusForbidden, false, "The engines could not read the key file at https://api.example.test/" + key + ".txt."},
		{http.StatusUnprocessableEntity, false, "The engines refused the addresses because they do not match the site address."},
		{http.StatusBadRequest, false, "answer 400."},
		{http.StatusTooManyRequests, true, "Too many notifications; the next attempt is at "},
	} {
		engine.status = c.status
		err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]})
		if (err != nil) != c.retry {
			t.Errorf("status %d: job error %v, want retry %v", c.status, err, c.retry)
		}
		if got := s2(); !strings.Contains(got, c.want) {
			t.Errorf("S2 after %d lacks %q:\n%s", c.status, c.want, got)
		}
	}

	// A new key: the old file answers 404 at once.
	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true, "indexnow_new_key": true}); rec.Code != http.StatusOK {
		t.Fatalf("new key: %d %s", rec.Code, rec.Body.String())
	}
	fresh := e.indexNowKey(t)
	if fresh == key || len(fresh) != 32 {
		t.Fatalf("new key = %q (old %q)", fresh, key)
	}
	if rec := e.get(t, "/_sitemap/indexnow/"+key+".txt"); rec.Code != http.StatusNotFound {
		t.Fatalf("old key file: %d", rec.Code)
	}
	if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusOK {
		t.Fatalf("new key file: %d", rec.Code)
	}
	// A save without the checkbox keeps the key; turning IndexNow off
	// takes the file down.
	e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true})
	if got := e.indexNowKey(t); got != fresh {
		t.Fatalf("a plain save changed the key to %q", got)
	}
	e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": false})
	if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusNotFound {
		t.Fatalf("key file while off: %d", rec.Code)
	}
	before := len(engine.requests)
	if err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]}); err != nil || len(engine.requests) != before {
		t.Fatalf("job while off: %v, %d requests", err, len(engine.requests)-before)
	}
}

// TestIndexNowStaging pins T-33-15: outside production a rebuild queues no
// IndexNow job and the job sends nothing.
func TestIndexNowStaging(t *testing.T) {
	e := bootJournal(t)
	e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`)
	contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)))
	e.rebuild(t)
	if q := e.queuedIndexNow(t); len(q) != 0 {
		t.Fatalf("queued outside production: %v", q)
	}
	engine := &fakeIndexNow{status: http.StatusOK}
	if err := e.job(t, classes.IndexNowKind).Work(fetchguard.WithTransport(t.Context(), engine), classes.IndexNowArgs{URLs: []string{"https://api.example.test/blog/a"}}); err != nil {
		t.Fatal(err)
	}
	if len(engine.requests) != 0 {
		t.Fatalf("%d requests outside production", len(engine.requests))
	}
	if rec := e.get(t, "/_sitemap/indexnow/0123456789abcdef0123456789abcdef.txt"); rec.Code != http.StatusOK {
		t.Fatalf("key file outside production: %d", rec.Code)
	}
}

// htmlJSON fetches an HTML sitemap API path and decodes its answer.
func (e *sitemapEnv) htmlJSON(t *testing.T, path string, status int) map[string]any {
	t.Helper()
	rec := e.get(t, path)
	if rec.Code != status {
		t.Fatalf("GET %s: %d, want %d: %s", path, rec.Code, status, rec.Body.String())
	}
	if rec.Header().Get("X-Robots-Tag") != "noindex" {
		t.Errorf("GET %s lacks X-Robots-Tag noindex: %v", path, rec.Header())
	}
	out := map[string]any{}
	if status == http.StatusOK {
		if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
			t.Fatalf("decode %s: %v", rec.Body.String(), err)
		}
	}
	return out
}

// titlesOfEntries lists the titles of decoded entries, children in
// brackets.
func titlesOfEntries(v any) string {
	list, _ := v.([]any)
	var parts []string
	for _, item := range list {
		e, _ := item.(map[string]any)
		s, _ := e["title"].(string)
		if kids, ok := e["children"]; ok {
			s += "[" + titlesOfEntries(kids) + "]"
		}
		parts = append(parts, s)
	}
	return strings.Join(parts, ", ")
}

// TestHTMLGroups pins D-34's HTML sitemap API: the overview of a locale
// lists the groups in the L3 order (pages, offers, categories, posts) with
// titles in the locale, categories nested by parent, posts newest first,
// others alphabetical, the first html_page_size links and more when there
// are more; a group page pages the rest; past the last page, an unknown
// group and the HTML sitemap off are 404; excluded addresses never appear;
// every answer is noindex. The settings tab validates its values and
// contract.Listing answers the HTML sitemap and key file settings.
func TestHTMLGroups(t *testing.T) {
	contentStore.reset(t)
	e := bootPlugins(t, []string{contentPluginID, journalPluginID, filterPluginID, "golem15.sitemap"})
	at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
	pl := func(id int, url, title string) fakeRecord {
		return fakeRecord{ID: id, Published: true, LastMod: at, Locales: []pact.ContentLocale{{Locale: "pl", URL: url, Title: title}}}
	}
	contentStore.put(pageType, pl(1, "/pages/o-nas", "O nas"))
	contentStore.put(pageType, pl(2, "/pages/kontakt", "Kontakt"))
	contentStore.put(pageType, pl(3, "/pages/noindex-me", "Ukryta"))
	contentStore.put(pageType, fakeRecord{ID: 4, Published: true, LastMod: at, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/about", Title: "About"}}})
	contentStore.put(offerType, pl(1, "/offers/zestaw", "Zestaw"))
	cat := func(id int, slug, title string, parent int) fakeRecord {
		r := pl(id, "/kategoria/"+slug, title)
		if parent > 0 {
			r.Parents = []pact.ContentRef{{Type: categoryType, ID: fmt.Sprint(parent)}}
		}
		return r
	}
	contentStore.put(categoryType, cat(1, "grzyby", "Grzyby", 0))
	contentStore.put(categoryType, cat(2, "lecznicze", "Lecznicze", 1))
	contentStore.put(categoryType, cat(3, "jadalne", "Jadalne", 1))
	contentStore.put(categoryType, cat(4, "inne", "Inne", 0))
	for i := 1; i <= 160; i++ {
		contentStore.put(postType, post(i, fmt.Sprintf("p%03d", i), "pl", at.Add(time.Duration(i)*time.Hour)))
	}
	e.exec(t, `INSERT INTO golem15_sitemap_types (type_key, include, slug, titles, updated_at) VALUES (?, TRUE, '', '{"pl":"Wpisy","en":"Posts"}', NOW())`, postType)
	e.rebuild(t)

	ov := e.htmlJSON(t, "/_sitemap/api/v1/html?locale=pl", http.StatusOK)
	groups, _ := ov["groups"].([]any)
	var order []string
	byKey := map[string]map[string]any{}
	for _, g := range groups {
		m := g.(map[string]any)
		order = append(order, m["slug"].(string))
		byKey[m["slug"].(string)] = m
	}
	if strings.Join(order, " ") != "pages offers categories posts" {
		t.Fatalf("group order = %v\n%v", order, ov)
	}
	if got := titlesOfEntries(byKey["pages"]["entries"]); got != "Kontakt, O nas" {
		t.Errorf("pages = %s", got)
	}
	if got := titlesOfEntries(byKey["categories"]["entries"]); got != "Grzyby[Jadalne, Lecznicze], Inne" {
		t.Errorf("categories tree = %s", got)
	}
	posts := byKey["posts"]
	entries, _ := posts["entries"].([]any)
	if posts["title"] != "Wpisy" || posts["total"] != float64(160) || posts["more"] != true || len(entries) != 150 {
		t.Fatalf("posts group = title %v total %v more %v entries %d", posts["title"], posts["total"], posts["more"], len(entries))
	}
	if first := entries[0].(map[string]any); first["title"] != "Post p160" || first["url"] != "https://api.example.test/blog/p160" {
		t.Errorf("newest post first: %v", first)
	}
	if byKey["categories"]["title"] != "Categories" || byKey["pages"]["more"] != false {
		t.Errorf("categories title %v, pages more %v", byKey["categories"]["title"], byKey["pages"]["more"])
	}
	all := fmt.Sprint(ov)
	for _, banned := range []string{"Ukryta", "noindex-me", "About"} {
		if strings.Contains(all, banned) {
			t.Errorf("the pl overview lists %q", banned)
		}
	}
	if en := fmt.Sprint(e.htmlJSON(t, "/_sitemap/api/v1/html?locale=en", http.StatusOK)); !strings.Contains(en, "About") || strings.Contains(en, "Kontakt") {
		t.Errorf("the en overview: %s", en)
	}

	page2 := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl&page=2", http.StatusOK)
	if p2, _ := page2["entries"].([]any); page2["page"] != float64(2) || page2["pages"] != float64(2) || page2["group"] != "posts" || page2["title"] != "Wpisy" || len(p2) != 10 {
		t.Fatalf("posts page 2 = %v", page2)
	}
	if got := titlesOfEntries(page2["entries"]); !strings.HasPrefix(got, "Post p010, Post p009") || !strings.HasSuffix(got, "Post p001") {
		t.Errorf("posts page 2 = %s", got)
	}
	if p1 := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl", http.StatusOK); p1["page"] != float64(1) {
		t.Errorf("posts without page = %v", p1["page"])
	}
	if got := titlesOfEntries(e.htmlJSON(t, "/_sitemap/api/v1/html/categories?locale=pl", http.StatusOK)["entries"]); got != "Grzyby[Jadalne, Lecznicze], Inne" {
		t.Errorf("categories page = %s", got)
	}
	for _, path := range []string{
		"/_sitemap/api/v1/html/posts?locale=pl&page=3",
		"/_sitemap/api/v1/html/posts?locale=pl&page=0",
		"/_sitemap/api/v1/html/posts?locale=pl&page=x",
		"/_sitemap/api/v1/html/unknown?locale=pl",
		"/_sitemap/api/v1/html/offers?locale=en",
		"/_sitemap/api/v1/html?locale=%3Cx%3E",
	} {
		e.htmlJSON(t, path, http.StatusNotFound)
	}

	// The settings tab.
	token := e.admin(t, permSitemap)
	const base = "/golem15/sitemap/settings"
	schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
	for _, want := range []string{`"html_enabled"`, `"html_path"`, `"html_page_size"`, "HTML sitemap"} {
		if !strings.Contains(schema, want) {
			t.Errorf("form schema lacks %s", want)
		}
	}
	for _, c := range []struct {
		body  map[string]any
		field string
	}{
		{map[string]any{"html_page_size": 49}, "html_page_size"},
		{map[string]any{"html_page_size": 501}, "html_page_size"},
		{map[string]any{"html_path": "mapa"}, "html_path"},
		{map[string]any{"html_path": "/mapa?x=1"}, "html_path"},
		{map[string]any{"html_path": "//evil.test"}, "html_path"},
	} {
		if rec := e.call(t, http.MethodPut, base+"/1", token, c.body); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"`+c.field+`"`) {
			t.Errorf("%v: %d %s", c.body, rec.Code, rec.Body.String())
		}
	}
	listing, ok := e.app.Lookup[contract.Listing]()
	if !ok {
		t.Fatal("no contract.Listing")
	}
	ls, err := listing.Settings(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	if !ls.HTMLEnabled || ls.HTMLPageSize != 150 || ls.HTMLPaths["pl"] != "/mapa-strony" || ls.HTMLPaths["en"] != "/sitemap" || ls.KeyFile != "" {
		t.Fatalf("listing settings = %+v", ls)
	}
	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"html_path": "/site-map", "html_page_size": 100}); rec.Code != http.StatusOK {
		t.Fatalf("save the HTML tab: %d %s", rec.Code, rec.Body.String())
	}
	e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`)
	ls, _ = listing.Settings(t.Context())
	if ls.HTMLPageSize != 100 || ls.HTMLPaths["en"] != "/site-map" || ls.HTMLPaths["pl"] != "/mapa-strony" || ls.KeyFile != "0123456789abcdef0123456789abcdef.txt" {
		t.Fatalf("listing settings after the save = %+v", ls)
	}
	if p := e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl&page=2", http.StatusOK); p["pages"] != float64(2) || len(p["entries"].([]any)) != 60 {
		t.Errorf("posts page 2 at size 100 = pages %v", p["pages"])
	}

	e.call(t, http.MethodPut, base+"/1", token, map[string]any{"html_enabled": false})
	e.htmlJSON(t, "/_sitemap/api/v1/html?locale=pl", http.StatusNotFound)
	e.htmlJSON(t, "/_sitemap/api/v1/html/posts?locale=pl", http.StatusNotFound)
	if ls, _ := listing.Settings(t.Context()); ls.HTMLEnabled {
		t.Error("listing says the HTML sitemap is on")
	}
}

// api sends a management API request with a bearer token (none when "")
// and a JSON body.
func (e *sitemapEnv) api(t *testing.T, method, path, token string, body any) *httptest.ResponseRecorder {
	t.Helper()
	var reader io.Reader
	if body != nil {
		raw, err := json.Marshal(body)
		if err != nil {
			t.Fatal(err)
		}
		reader = bytes.NewReader(raw)
	}
	req := httptest.NewRequest(method, path, reader)
	req.Host = "api.example.test"
	req.RemoteAddr = fmt.Sprintf("198.51.100.%d:4000", adminSeq.Add(1)%250+1)
	req.Header.Set("Accept-Language", "en")
	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	if token != "" {
		req.Header.Set("Authorization", "Bearer "+token)
	}
	rec := httptest.NewRecorder()
	e.h.ServeHTTP(rec, req)
	return rec
}

// apiJSON sends a management API request, fails unless it answers status
// and decodes the answer.
func (e *sitemapEnv) apiJSON(t *testing.T, status int, method, path, token string, body any) map[string]any {
	t.Helper()
	rec := e.api(t, method, path, token, body)
	if rec.Code != status {
		t.Fatalf("%s %s = %d, want %d: %s", method, path, rec.Code, status, rec.Body.String())
	}
	if rec.Header().Get("X-Robots-Tag") != "noindex" {
		t.Errorf("%s %s lacks X-Robots-Tag noindex", method, path)
	}
	out := map[string]any{}
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("decode %s: %v", rec.Body.String(), err)
	}
	return out
}

// TestSitemapTools pins D-10 for the sitemap: GET and PUT
// /_sitemap/api/v1/settings and POST /_sitemap/api/v1/rebuild need a
// backend administrator with the sitemap permission (401 without one, 403
// without the permission); PUT checks like the form (422, nothing
// stored), merges the HTML sitemap addresses, writes content types,
// generates the IndexNow key and queues a rebuild; an MCP dispatch runs as
// the linked administrator; the MCP catalog declares get_settings,
// update_settings and rebuild bound to those routes behind the permission.
func TestSitemapTools(t *testing.T) {
	e := bootJournal(t)
	contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC), "/media/a.jpg"))
	e.rebuild(t)
	routes := []struct{ method, path string }{
		{http.MethodGet, "/_sitemap/api/v1/settings"},
		{http.MethodPut, "/_sitemap/api/v1/settings"},
		{http.MethodPost, "/_sitemap/api/v1/rebuild"},
	}
	other := e.admin(t)
	for _, r := range routes {
		if rec := e.api(t, r.method, r.path, "", map[string]any{}); rec.Code != http.StatusUnauthorized {
			t.Errorf("%s %s without a token: %d %s", r.method, r.path, rec.Code, rec.Body.String())
		}
		if rec := e.api(t, r.method, r.path, other, map[string]any{}); rec.Code != http.StatusForbidden {
			t.Errorf("%s %s without the permission: %d %s", r.method, r.path, rec.Code, rec.Body.String())
		}
	}

	token := e.admin(t, permSitemap)
	data := e.apiJSON(t, http.StatusOK, http.MethodGet, "/_sitemap/api/v1/settings", token, nil)["data"].(map[string]any)
	got := fmt.Sprint(data)
	for _, want := range []string{"sitemap-posts-1.xml", "sitemap-images-1.xml", "https://api.example.test/sitemap.xml", "type:acme-post", "html_page_size:150", "indexnow:map["} {
		if !strings.Contains(got, want) {
			t.Errorf("GET settings lacks %q:\n%s", want, got)
		}
	}
	if types := data["types"].(map[string]any); types[postType] == nil || types[postType].(map[string]any)["slug"] != "posts" {
		t.Errorf("types = %v", data["types"])
	}

	// Refusals name the fields and store nothing.
	for _, c := range []struct {
		body  map[string]any
		field string
	}{
		{map[string]any{"site_url": "https://other.test", "file_size": 0}, "file_size"},
		{map[string]any{"html_page_size": 49}, "html_page_size"},
		{map[string]any{"html_paths": map[string]string{"pl": "mapa"}}, "html_paths"},
		{map[string]any{"news_sources": []string{"posts"}}, "news_sources"},
		{map[string]any{"types": map[string]any{postType: map[string]any{"slug": "Bad Slug"}}}, "types." + postType + ".slug"},
		{map[string]any{"types": map[string]any{"nope": map[string]any{"include": false}}}, "types.nope"},
	} {
		body := e.apiJSON(t, http.StatusUnprocessableEntity, http.MethodPut, "/_sitemap/api/v1/settings", token, c.body)
		errs, _ := body["errors"].(map[string]any)
		if body["error"] != "validation_failed" || errs[c.field] == nil {
			t.Errorf("%v: %v", c.body, body)
		}
	}
	if rec := e.api(t, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{"unknown_key": 1}); rec.Code != http.StatusUnprocessableEntity {
		t.Errorf("unknown key: %d %s", rec.Code, rec.Body.String())
	}
	var site *string
	e.gdb.Raw(`SELECT site_url FROM golem15_sitemap_settings WHERE id = 1`).Scan(&site)
	if site != nil && *site != "" {
		t.Fatalf("a refused PUT stored site_url %q", *site)
	}

	jobsBefore := e.riverJobs(t, classes.RebuildKind)
	data = e.apiJSON(t, http.StatusOK, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{
		"site_url":         "https://www.example.test",
		"news_enabled":     true,
		"news_sources":     []string{"type:acme-post"},
		"indexnow_enabled": true,
		"html_paths":       map[string]string{"en": "/site-map"},
		"html_page_size":   200,
		"types":            map[string]any{postType: map[string]any{"slug": "blog", "titles": map[string]string{"en": "Blog"}}},
	})["data"].(map[string]any)
	settings := data["settings"].(map[string]any)
	paths := settings["html_paths"].(map[string]any)
	if settings["site_url"] != "https://www.example.test" || settings["news_enabled"] != true || settings["html_page_size"] != float64(200) || paths["en"] != "/site-map" || paths["pl"] != "/mapa-strony" {
		t.Fatalf("settings after PUT = %v", settings)
	}
	key := data["indexnow"].(map[string]any)["key"].(string)
	if len(key) != 32 || data["indexnow"].(map[string]any)["key_url"] != "https://www.example.test/"+key+".txt" {
		t.Fatalf("indexnow after PUT = %v", data["indexnow"])
	}
	if pt := data["types"].(map[string]any)[postType].(map[string]any); pt["slug"] != "blog" || pt["titles"].(map[string]any)["en"] != "Blog" {
		t.Fatalf("post type after PUT = %v", pt)
	}
	if n := e.riverJobs(t, classes.RebuildKind); n != jobsBefore+1 {
		t.Fatalf("PUT queued %d rebuilds", n-jobsBefore)
	}
	data = e.apiJSON(t, http.StatusOK, http.MethodPut, "/_sitemap/api/v1/settings", token, map[string]any{"indexnow_new_key": true})["data"].(map[string]any)
	if fresh := data["indexnow"].(map[string]any)["key"].(string); fresh == key || len(fresh) != 32 {
		t.Fatalf("new key = %q (old %q)", fresh, key)
	}

	if body := e.apiJSON(t, http.StatusAccepted, http.MethodPost, "/_sitemap/api/v1/rebuild", token, nil); fmt.Sprint(body["data"]) != "map[queued:true]" {
		t.Fatalf("rebuild = %v", body)
	}
	if n := e.riverJobs(t, classes.RebuildKind); n != jobsBefore+3 {
		t.Fatalf("%d rebuilds queued, want %d", n, jobsBefore+3)
	}

	// An MCP call: in-process, without a token, as the linked administrator.
	req := httptest.NewRequest(http.MethodGet, "/", nil)
	req.Header.Set("Authorization", "Bearer "+token)
	_, principal, err := cabana.ResolveBackend(e.app, req)
	if err != nil || principal == nil {
		t.Fatalf("principal: %v", err)
	}
	mcpReq := httptest.NewRequest(http.MethodGet, "/_sitemap/api/v1/settings", nil)
	mcpReq.RemoteAddr = "198.51.100.7:4000"
	mcpReq = mcpReq.WithContext(bouncer.WithDispatch(mcpReq.Context(), &bouncer.Dispatch{Guards: []string{"backend"}, Principal: principal}))
	rec := httptest.NewRecorder()
	e.h.ServeHTTP(rec, mcpReq)
	if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"settings"`) {
		t.Fatalf("MCP get_settings: %d %s", rec.Code, rec.Body.String())
	}

	// The catalog.
	var tools []pact.MCPTool
	for _, p := range e.plugins {
		if m, ok := p.(pact.HasMCP); ok && p.ID() == "golem15.sitemap" {
			tools = m.MCP().Tools
		}
	}
	want := map[string]string{"get_settings": "GET /_sitemap/api/v1/settings", "update_settings": "PUT /_sitemap/api/v1/settings", "rebuild": "POST /_sitemap/api/v1/rebuild"}
	if len(tools) != len(want) {
		t.Fatalf("tools = %+v", tools)
	}
	for _, tool := range tools {
		if want[tool.Name] != tool.Method+" "+tool.Path {
			t.Errorf("tool %s is bound to %s %s", tool.Name, tool.Method, tool.Path)
		}
		if tool.Guard != "backend" || len(tool.Permissions) != 1 || tool.Permissions[0] != permSitemap {
			t.Errorf("tool %s: guard %q permissions %v", tool.Name, tool.Guard, tool.Permissions)
		}
	}

	// pier compiles the declarations against the registered routes.
	router, err := surf.BuildRouter(e.app, e.plugins)
	if err != nil {
		t.Fatal(err)
	}
	if err := e.app.Config.Set(pier.EnabledKey, true); err != nil {
		t.Fatal(err)
	}
	if err := pier.Mount(mountHost{r: router}, e.app, e.plugins); err != nil {
		t.Fatalf("pier.Mount: %v", err)
	}
	pr, ok := e.app.Lookup[*pier.Pier]()
	if !ok {
		t.Fatal("pier was not mounted")
	}
	if got := pr.ToolNames(); strings.Join(got, " ") != "sitemap_get_settings sitemap_rebuild sitemap_update_settings" {
		t.Fatalf("compiled tools %v", got)
	}
}

// mountHost adapts a built surf router to pier.Host for a compile-only
// mount.
type mountHost struct{ r *surf.Router }

func (h mountHost) Routes() []pier.RouteInfo {
	var out []pier.RouteInfo
	for _, ri := range h.r.Routes() {
		out = append(out, pier.RouteInfo{Method: ri.Method, Pattern: ri.Pattern, PluginID: ri.PluginID, Middleware: ri.Middleware})
	}
	return out
}
func (h mountHost) Handler(method, pattern string) (http.Handler, bool) {
	return h.r.Handler(method, pattern)
}
func (mountHost) MountStream(string, string, http.Handler, ...string) error { return nil }
func (mountHost) MountJSON(string, string, http.Handler) error              { return nil }
func (h mountHost) Owner(method, path string) (string, bool) {
	for _, ri := range h.r.Routes() {
		if ri.Method == method && ri.Pattern == path {
			return ri.PluginID, true
		}
	}
	return "", false
}