feat: sitemap index and per-type files from a slot table (tracer)

- contract package: Filter, Listing and their types (D-95)
- golem15_sitemap_urls slot table and the new settings columns
- Rebuild over signpost Records, stable slot assignment, rebuild job
- GET /sitemap.xml index and GET /_sitemap/files/{file} from the table

classes/rebuild.go

@@ -0,0 +1,406 @@
+package classes
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"git.golem15.com/golem15/summercms/modules/signpost"
+	"gorm.io/gorm"
+)
+
+// RebuildKind is the River kind of the rebuild job.
+const RebuildKind = "golem15.sitemap.rebuild"
+
+// RebuildArgs are the arguments of the rebuild job: it has none, a rebuild
+// always covers every content type and the additional addresses.
+type RebuildArgs struct{}
+
+// Kind is RebuildKind.
+func (RebuildArgs) Kind() string { return RebuildKind }
+
+// recordsPage is the page size the rebuild asks the content types for.
+const recordsPage = 500
+
+// ErrNoDatabase is returned when the application publishes no database.
+var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured")
+
+// RebuildResult is what one rebuild changed, as absolute URLs: Added are
+// newly listed addresses, Changed listed addresses whose URL changed and
+// Removed addresses that left the files. Listed counts the listed
+// addresses and Files the files they fill.
+type RebuildResult struct {
+	Added   []string
+	Changed []string
+	Removed []string
+	Listed  int
+	Files   int
+}
+
+// DefaultSlug is the file name part of a content type's group: the last
+// segment of its key in plural form (journal-post: posts, journal-category:
+// categories, seo-page: pages).
+func DefaultSlug(key string) string {
+	key = strings.ToLower(strings.TrimSpace(key))
+	if i := strings.LastIndexAny(key, "-_."); i >= 0 {
+		key = key[i+1:]
+	}
+	var b strings.Builder
+	for _, r := range key {
+		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
+			b.WriteRune(r)
+		}
+	}
+	s := b.String()
+	switch {
+	case s == "":
+		return "pages"
+	case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])):
+		return s[:len(s)-1] + "ies"
+	case strings.HasSuffix(s, "s"):
+		return s
+	}
+	return s + "s"
+}
+
+// ExtraRecordID is the record_id of an additional address: the hex SHA-256
+// of the address.
+func ExtraRecordID(u string) string {
+	sum := sha256.Sum256([]byte(u))
+	return hex.EncodeToString(sum[:])
+}
+
+// slotKey identifies a row of the slot table.
+type slotKey struct {
+	typeKey, recordID, locale string
+}
+
+// wanted is one address a rebuild found: its group, its row key, its path
+// and title, its first lastmod (for a new row) and the reason it is left
+// out ("" when listed). An additional address carries its own lastmod,
+// which the editor sets, so it is written on every rebuild.
+type wanted struct {
+	key        slotKey
+	grp        string
+	url        string
+	title      string
+	firstMod   *time.Time
+	reason     string
+	extra      bool
+	extraMod   *time.Time
+	alternates []models.Alternate
+}
+
+// collected is what a rebuild read from the content types and settings.
+type collected struct {
+	rows        []wanted
+	unpublished map[slotKey]bool
+}
+
+// Rebuild reads every content type's published records (signpost Records,
+// D-93) and the additional addresses, writes them into the slot table and
+// numbers the files, in one transaction that holds the settings row, so
+// two rebuilds never interleave. A record that is gone or unpublished
+// leaves its rows in the table with a reason, as a hole; only a changed
+// file size renumbers the files (D-30).
+func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
+	var res RebuildResult
+	if app == nil {
+		return res, ErrNoDatabase
+	}
+	db, ok := app.Lookup[*gorm.DB]()
+	if !ok || db == nil {
+		return res, ErrNoDatabase
+	}
+	db = db.WithContext(ctx)
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return res, err
+	}
+	reg, _ := app.Lookup[*signpost.Registry]()
+	found, err := collect(ctx, reg, settings)
+	if err != nil {
+		return res, err
+	}
+	site := SiteURL(app, settings)
+	err = db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
+			return err
+		}
+		locked, err := LoadSettings(tx)
+		if err != nil {
+			return err
+		}
+		res, err = apply(tx, found, locked, site)
+		return err
+	})
+	return res, err
+}
+
+// collect walks the content types page by page and adds the additional
+// addresses.
+func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings) (collected, error) {
+	out := collected{unpublished: map[slotKey]bool{}}
+	seen := map[slotKey]bool{}
+	add := func(w wanted) {
+		if seen[w.key] {
+			return
+		}
+		seen[w.key] = true
+		out.rows = append(out.rows, w)
+	}
+	unprefixed := models.Text(settings.DefaultLocale)
+	if reg != nil {
+		for _, typ := range reg.ContentTypes() {
+			slug := DefaultSlug(typ.Key)
+			after := ""
+			for {
+				page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed)
+				if err != nil {
+					return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err)
+				}
+				for _, rec := range page.Records {
+					if !rec.Published {
+						out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true
+						continue
+					}
+					for _, l := range rec.Locales {
+						if strings.TrimSpace(l.URL) == "" {
+							continue
+						}
+						add(wanted{
+							key:      slotKey{typ.Key, rec.ID, l.Locale},
+							grp:      slug,
+							url:      l.URL,
+							title:    l.Title,
+							firstMod: rec.LastMod,
+						})
+					}
+				}
+				if page.Next == "" || page.Next == after {
+					break
+				}
+				after = page.Next
+			}
+		}
+	}
+	for _, e := range settings.Extra() {
+		u := strings.TrimSpace(e.URL)
+		if !models.AllowedAddress(u) {
+			continue
+		}
+		mod := ParseDate(e.LastMod)
+		add(wanted{
+			key:      slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""},
+			grp:      models.ExtraGroup,
+			url:      u,
+			extra:    true,
+			extraMod: mod,
+			firstMod: mod,
+		})
+	}
+	return out, nil
+}
+
+// ParseDate reads a date as the admin's date picker or an import sends it
+// (RFC 3339, "2006-01-02 15:04:05" or "2006-01-02", UTC); nil when empty or
+// unreadable.
+func ParseDate(s string) *time.Time {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil
+	}
+	for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} {
+		if t, err := time.Parse(layout, s); err == nil {
+			t = t.UTC()
+			return &t
+		}
+	}
+	return nil
+}
+
+// apply writes the found addresses into the slot table and numbers the
+// files, inside the rebuild's transaction.
+func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) {
+	var res RebuildResult
+	var existing []models.URL
+	if err := tx.Order("id ASC").Find(&existing).Error; err != nil {
+		return res, err
+	}
+	now := time.Now().UTC().Truncate(time.Microsecond)
+	byKey := make(map[slotKey]*models.URL, len(existing))
+	for i := range existing {
+		r := &existing[i]
+		byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r
+	}
+	dirty := map[uint64]bool{}
+	var created []*models.URL
+	matched := map[slotKey]bool{}
+	for _, w := range found.rows {
+		matched[w.key] = true
+		r, ok := byKey[w.key]
+		if !ok {
+			mod := w.firstMod
+			if mod == nil && !w.extra {
+				mod = &now
+			}
+			row := &models.URL{
+				Grp:            w.grp,
+				TypeKey:        w.key.typeKey,
+				RecordID:       w.key.recordID,
+				Locale:         w.key.locale,
+				URL:            w.url,
+				Title:          w.title,
+				FirstSeen:      now,
+				LastMod:        utcPtr(mod),
+				Alternates:     lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true},
+				Images:         lagoon.Jsonable[[]string]{Data: []string{}, Valid: true},
+				ExcludedReason: w.reason,
+				UpdatedAt:      now,
+			}
+			created = append(created, row)
+			if w.reason == "" {
+				res.Added = append(res.Added, Absolute(site, w.url))
+			}
+			continue
+		}
+		wasListed := r.ExcludedReason == ""
+		before := *r
+		if r.Grp != w.grp {
+			r.Grp = w.grp
+			r.FileNo = nil
+		}
+		r.URL = w.url
+		r.Title = w.title
+		r.ExcludedReason = w.reason
+		r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}
+		if w.extra {
+			r.LastMod = utcPtr(w.extraMod)
+		}
+		listed := w.reason == ""
+		switch {
+		case listed && !wasListed:
+			res.Added = append(res.Added, Absolute(site, r.URL))
+		case !listed && wasListed:
+			res.Removed = append(res.Removed, Absolute(site, before.URL))
+		case listed && before.URL != r.URL:
+			res.Changed = append(res.Changed, Absolute(site, r.URL))
+		}
+		if rowChanged(before, *r) {
+			dirty[r.ID] = true
+		}
+	}
+	for i := range existing {
+		r := &existing[i]
+		key := slotKey{r.TypeKey, r.RecordID, r.Locale}
+		if matched[key] {
+			continue
+		}
+		reason := models.ReasonDeleted
+		if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
+			reason = models.ReasonUnpublished
+		}
+		if r.ExcludedReason == reason {
+			continue
+		}
+		if r.ExcludedReason == "" {
+			res.Removed = append(res.Removed, Absolute(site, r.URL))
+		}
+		r.ExcludedReason = reason
+		dirty[r.ID] = true
+	}
+	if len(created) > 0 {
+		if err := tx.CreateInBatches(created, 500).Error; err != nil {
+			return res, err
+		}
+	}
+	size := settings.EffectiveFileSize()
+	renumber := settings.NumberedSize == nil || *settings.NumberedSize != size
+	slots := make([]*Slot, 0, len(existing)+len(created))
+	rowsByID := make(map[uint64]*models.URL, len(existing)+len(created))
+	for i := range existing {
+		r := &existing[i]
+		rowsByID[r.ID] = r
+		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
+	}
+	for _, r := range created {
+		rowsByID[r.ID] = r
+		slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""})
+	}
+	AssignSlots(slots, size, renumber)
+	files := map[string]bool{}
+	for _, s := range slots {
+		r := rowsByID[s.ID]
+		if !sameInt(r.FileNo, s.FileNo) {
+			r.FileNo = s.FileNo
+			dirty[r.ID] = true
+		}
+		if s.Listed {
+			res.Listed++
+			if s.FileNo != nil {
+				files[FileName(s.Grp, *s.FileNo)] = true
+			}
+		}
+	}
+	res.Files = len(files)
+	for id := range dirty {
+		r := rowsByID[id]
+		r.UpdatedAt = now
+		if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{
+			"grp":             r.Grp,
+			"url":             r.URL,
+			"title":           r.Title,
+			"file_no":         r.FileNo,
+			"lastmod":         r.LastMod,
+			"alternates":      r.Alternates,
+			"excluded_reason": r.ExcludedReason,
+			"updated_at":      r.UpdatedAt,
+		}).Error; err != nil {
+			return res, err
+		}
+	}
+	if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
+		return res, err
+	}
+	return res, nil
+}
+
+// rowChanged reports whether a rebuild changed a stored row.
+func rowChanged(a, b models.URL) bool {
+	if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) {
+		return true
+	}
+	av, _ := a.Alternates.Value()
+	bv, _ := b.Alternates.Value()
+	return av != bv
+}
+
+func sameInt(a, b *int) bool {
+	return (a == nil && b == nil) || (a != nil && b != nil && *a == *b)
+}
+
+func sameTime(a, b *time.Time) bool {
+	return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b))
+}
+
+func utcPtr(t *time.Time) *time.Time {
+	if t == nil {
+		return nil
+	}
+	u := t.UTC().Truncate(time.Microsecond)
+	return &u
+}
+
+func nonNil[T any](s []T) []T {
+	if s == nil {
+		return []T{}
+	}
+	return s
+}

classes/settings.go

@@ -0,0 +1,61 @@
+package classes
+
+import (
+	"strings"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"gorm.io/gorm"
+)
+
+// SiteURLConfigKey is the config key of the site address the sitemaps use
+// when the setting is empty.
+const SiteURLConfigKey = "golem15.sitemap.site_url"
+
+// LoadSettings reads the settings row; without one the defaults apply.
+func LoadSettings(db *gorm.DB) (models.Settings, error) {
+	var rows []models.Settings
+	if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
+		return models.Settings{}, err
+	}
+	if len(rows) == 0 {
+		return models.DefaultSettings(), nil
+	}
+	return rows[0], nil
+}
+
+// SiteURL is the absolute site address every <loc> starts with, without a
+// trailing slash: the site_url setting, else the config key
+// golem15.sitemap.site_url, else app.url (CONTEXT 2026-10-28). A value that
+// is not an absolute http(s) URL (models.AllowedFrontendURL) is skipped.
+// The request's Host header is never used; "" means no site address is
+// configured.
+func SiteURL(app *backpack.App, s models.Settings) string {
+	candidates := []string{models.Text(s.SiteURL)}
+	if app != nil && app.Config != nil {
+		candidates = append(candidates, app.Config.String(SiteURLConfigKey), app.Config.String("app.url"))
+	}
+	for _, c := range candidates {
+		c = strings.TrimRight(strings.TrimSpace(c), "/")
+		if c != "" && models.AllowedFrontendURL(c) {
+			return c
+		}
+	}
+	return ""
+}
+
+// Absolute turns a stored address into the absolute URL of a <loc>: a path
+// starting with a single / is put on site; an absolute http(s) URL is kept.
+// Anything else, or a path without a site address, is "".
+func Absolute(site, u string) string {
+	switch {
+	case strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//"):
+		if site == "" {
+			return ""
+		}
+		return site + u
+	case models.AllowedFrontendURL(u):
+		return u
+	}
+	return ""
+}

classes/slots.go

@@ -0,0 +1,95 @@
+package classes
+
+import (
+	"slices"
+	"time"
+)
+
+// Slot is what file numbering needs of one row of the slot table: its
+// group, its file number (nil while unnumbered), when it was first seen and
+// whether it is listed. A row left out keeps its number: its file keeps the
+// hole instead of the later files shifting (D-30, RESEARCH Pitfall 5).
+type Slot struct {
+	ID        uint64
+	Grp       string
+	FileNo    *int
+	FirstSeen time.Time
+	Listed    bool
+}
+
+// AssignSlots numbers the rows for files of size addresses and reports
+// whether any number changed. Without renumber, a numbered row keeps its
+// number and every listed row without one is appended to the last file of
+// its group while that file has room (counting the holes of left-out rows),
+// then to the next file, oldest first (first seen, then id). With renumber
+// (the file size changed), every group's listed rows are numbered again
+// from file 1, oldest first, and the left-out rows lose their numbers.
+func AssignSlots(rows []*Slot, size int, renumber bool) bool {
+	if size < 1 {
+		size = 1
+	}
+	groups := map[string][]*Slot{}
+	var order []string
+	for _, r := range rows {
+		if _, ok := groups[r.Grp]; !ok {
+			order = append(order, r.Grp)
+		}
+		groups[r.Grp] = append(groups[r.Grp], r)
+	}
+	changed := false
+	set := func(r *Slot, n *int) {
+		if (r.FileNo == nil) != (n == nil) || (r.FileNo != nil && *r.FileNo != *n) {
+			changed = true
+		}
+		r.FileNo = n
+	}
+	for _, grp := range order {
+		members := groups[grp]
+		slices.SortStableFunc(members, func(a, b *Slot) int {
+			if c := a.FirstSeen.Compare(b.FirstSeen); c != 0 {
+				return c
+			}
+			switch {
+			case a.ID < b.ID:
+				return -1
+			case a.ID > b.ID:
+				return 1
+			}
+			return 0
+		})
+		if renumber {
+			i := 0
+			for _, r := range members {
+				if !r.Listed {
+					set(r, nil)
+					continue
+				}
+				n := i/size + 1
+				set(r, &n)
+				i++
+			}
+			continue
+		}
+		counts := map[int]int{}
+		last := 0
+		for _, r := range members {
+			if r.FileNo == nil {
+				continue
+			}
+			counts[*r.FileNo]++
+			last = max(last, *r.FileNo)
+		}
+		for _, r := range members {
+			if r.FileNo != nil || !r.Listed {
+				continue
+			}
+			if last == 0 || counts[last] >= size {
+				last++
+			}
+			n := last
+			set(r, &n)
+			counts[last]++
+		}
+	}
+	return changed
+}

classes/tiers_test.go

@@ -1,16 +1,11 @@
 package classes_test
 
 import (
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
-	sitemap "git.golem15.com/golem15/sm-sitemap-plugin"
 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
-	"git.golem15.com/golem15/summercms/modules/backpack"
-	"git.golem15.com/golem15/summercms/modules/compass"
 )
 
 // TestTiers pins the priority tiers (D-13): each default rule of the PHP
@@ -96,20 +91,4 @@ func TestTiers(t *testing.T) {
 		}
 	}
 
-	// A bad host rule fails the plugin's Boot.
-	dir := t.TempDir()
-	if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte("sitemap:\n  priority_tiers:\n    - setting: priority_blog\n      pattern: '^/(blog'\n"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-	cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}})
-	if err != nil {
-		t.Fatal(err)
-	}
-	p := &sitemap.Plugin{}
-	if err := cfg.MergePlugin("golem15.sitemap", p.ConfigFS()); err != nil {
-		t.Fatal(err)
-	}
-	if err := p.Boot(backpack.New(cfg)); err == nil || !strings.Contains(err.Error(), "priority_tiers[0]") {
-		t.Fatalf("Boot with a bad pattern = %v", err)
-	}
 }

classes/writer.go

@@ -0,0 +1,135 @@
+package classes
+
+import (
+	"bufio"
+	"encoding/xml"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// The XML namespaces of the sitemap files.
+const (
+	NSSitemap = "http://www.sitemaps.org/schemas/sitemap/0.9"
+	NSXHTML   = "http://www.w3.org/1999/xhtml"
+)
+
+// IndexFile is one <sitemap> of the index: the file's absolute URL and the
+// newest lastmod of its addresses (nil when none has one).
+type IndexFile struct {
+	Loc     string
+	LastMod *time.Time
+}
+
+// FileAlternate is one xhtml:link alternate of a <url>, with an absolute
+// href.
+type FileAlternate struct {
+	Hreflang string
+	Href     string
+}
+
+// FileEntry is one <url> of a file: the absolute address, its lastmod and
+// its hreflang alternates.
+type FileEntry struct {
+	Loc        string
+	LastMod    *time.Time
+	Alternates []FileAlternate
+}
+
+// fileNamePattern is the name of a per-group file.
+var fileNamePattern = regexp.MustCompile(`^sitemap-([a-z0-9][a-z0-9-]*)-([1-9][0-9]{0,8})\.xml$`)
+
+// FileName is the name of file n of group grp: sitemap-<grp>-<n>.xml.
+func FileName(grp string, n int) string {
+	return "sitemap-" + grp + "-" + strconv.Itoa(n) + ".xml"
+}
+
+// ParseFileName splits a file name made by FileName; ok is false for any
+// other name.
+func ParseFileName(name string) (grp string, n int, ok bool) {
+	m := fileNamePattern.FindStringSubmatch(name)
+	if m == nil {
+		return "", 0, false
+	}
+	n, err := strconv.Atoi(m[2])
+	if err != nil || n < 1 {
+		return "", 0, false
+	}
+	return m[1], n, true
+}
+
+// lastmod formats a lastmod in RFC 3339, UTC.
+func lastmod(t time.Time) string { return t.UTC().Format(time.RFC3339) }
+
+// escape writes s with encoding/xml's escaping, which covers &, <, >, both
+// quotes and control whitespace, so it is safe in text and in attribute
+// values.
+func escape(w *bufio.Writer, s string) {
+	_ = xml.EscapeText(w, []byte(s))
+}
+
+// WriteIndex writes the sitemap index: one <sitemap> per file with its
+// <loc> and, when known, its <lastmod>.
+func WriteIndex(w io.Writer, files []IndexFile) error {
+	bw := bufio.NewWriter(w)
+	bw.WriteString(xml.Header)
+	bw.WriteString(`<sitemapindex xmlns="` + NSSitemap + `">`)
+	for _, f := range files {
+		bw.WriteString("\n<sitemap><loc>")
+		escape(bw, f.Loc)
+		bw.WriteString("</loc>")
+		if f.LastMod != nil {
+			bw.WriteString("<lastmod>" + lastmod(*f.LastMod) + "</lastmod>")
+		}
+		bw.WriteString("</sitemap>")
+	}
+	bw.WriteString("\n</sitemapindex>\n")
+	return bw.Flush()
+}
+
+// WriteURLSet writes one sitemap file: a <url> per entry with its <loc>, its
+// <lastmod> when known and its xhtml:link alternates. The xhtml namespace
+// is declared only when an entry has alternates. No <priority> and no
+// <changefreq> are written (D-30).
+func WriteURLSet(w io.Writer, entries []FileEntry) error {
+	bw := bufio.NewWriter(w)
+	bw.WriteString(xml.Header)
+	bw.WriteString(`<urlset xmlns="` + NSSitemap + `"`)
+	for _, e := range entries {
+		if len(e.Alternates) > 0 {
+			bw.WriteString(` xmlns:xhtml="` + NSXHTML + `"`)
+			break
+		}
+	}
+	bw.WriteString(">")
+	for _, e := range entries {
+		bw.WriteString("\n<url><loc>")
+		escape(bw, e.Loc)
+		bw.WriteString("</loc>")
+		if e.LastMod != nil {
+			bw.WriteString("<lastmod>" + lastmod(*e.LastMod) + "</lastmod>")
+		}
+		for _, a := range e.Alternates {
+			bw.WriteString(`<xhtml:link rel="alternate" hreflang="`)
+			escape(bw, a.Hreflang)
+			bw.WriteString(`" href="`)
+			escape(bw, a.Href)
+			bw.WriteString(`"/>`)
+		}
+		bw.WriteString("</url>")
+	}
+	bw.WriteString("\n</urlset>\n")
+	return bw.Flush()
+}
+
+// validSlug reports whether s may name a file group.
+func validSlug(s string) bool {
+	return s != "" && len(s) <= 64 && fileNamePattern.MatchString(FileName(s, 1)) && !strings.HasSuffix(s, "-")
+}
+
+// ValidSlug reports whether s may name a file group: lowercase letters,
+// digits and hyphens, starting with a letter or digit, at most 64
+// characters.
+func ValidSlug(s string) bool { return validSlug(s) }

content_fixture_test.go

@@ -0,0 +1,157 @@
+package sitemap_test
+
+import (
+	"context"
+	"slices"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/pact"
+	"git.golem15.com/golem15/summercms/modules/party"
+)
+
+// contentPluginID is a fake content plugin with two content types whose
+// records live in memory (contentStore), so a test sets exactly what the
+// sitemap reads through signpost Records. It knows nothing about the
+// sitemap plugin.
+const contentPluginID = "acme.content"
+
+// The fake content types: pages (/pages/<slug>, group pages) and offers
+// (/offers/<slug>, group offers).
+const (
+	pageType   = "acme-page"
+	pageModel  = "acme_pages"
+	offerType  = "acme-offer"
+	offerModel = "acme_offers"
+)
+
+// fakeRecord is one record of a fake content type.
+type fakeRecord struct {
+	ID        int
+	Published bool
+	LastMod   time.Time
+	Locales   []pact.ContentLocale
+}
+
+// store holds the fake records per content type key.
+type store struct {
+	mu      sync.Mutex
+	records map[string][]fakeRecord
+}
+
+var contentStore = &store{records: map[string][]fakeRecord{}}
+
+// reset empties the store and clears it again when the test ends.
+func (s *store) reset(t *testing.T) {
+	t.Helper()
+	s.mu.Lock()
+	s.records = map[string][]fakeRecord{}
+	s.mu.Unlock()
+	t.Cleanup(func() {
+		s.mu.Lock()
+		s.records = map[string][]fakeRecord{}
+		s.mu.Unlock()
+	})
+}
+
+// put adds or replaces a record of typ.
+func (s *store) put(typ string, rec fakeRecord) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	list := s.records[typ]
+	for i := range list {
+		if list[i].ID == rec.ID {
+			list[i] = rec
+			return
+		}
+	}
+	s.records[typ] = append(list, rec)
+}
+
+// remove deletes a record of typ.
+func (s *store) remove(typ string, id int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.records[typ] = slices.DeleteFunc(s.records[typ], func(r fakeRecord) bool { return r.ID == id })
+}
+
+// page answers one ContentList page of typ, by ascending ID.
+func (s *store) page(typ string, in pact.ContentList) pact.ContentPage {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	list := slices.Clone(s.records[typ])
+	slices.SortFunc(list, func(a, b fakeRecord) int { return a.ID - b.ID })
+	after, _ := strconv.Atoi(in.After)
+	limit := in.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	out := pact.ContentPage{Records: []pact.ContentRecord{}}
+	for _, r := range list {
+		if r.ID <= after {
+			continue
+		}
+		if len(out.Records) == limit {
+			out.Next = strconv.Itoa(after)
+			break
+		}
+		mod := r.LastMod
+		out.Records = append(out.Records, pact.ContentRecord{
+			ID:        strconv.Itoa(r.ID),
+			Published: r.Published,
+			LastMod:   &mod,
+			Locales:   slices.Clone(r.Locales),
+		})
+		after = r.ID
+	}
+	return out
+}
+
+// contentPlugin is acme.content.
+type contentPlugin struct{}
+
+func (contentPlugin) ID() string                   { return contentPluginID }
+func (contentPlugin) Requires() []string           { return nil }
+func (contentPlugin) Register(*backpack.App) error { return nil }
+func (contentPlugin) Boot(*backpack.App) error     { return nil }
+
+func (contentPlugin) MenuItemTypes() []pact.MenuItemType {
+	typ := func(key, label, model, pattern string) pact.MenuItemType {
+		return pact.MenuItemType{
+			Key:            key,
+			Label:          label,
+			DefaultPattern: pattern,
+			Placeholders:   []string{"slug"},
+			Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) {
+				return nil, nil
+			},
+			Content: &pact.ContentDescriptor{
+				Model: model,
+				Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) {
+					return contentStore.page(key, in), nil
+				},
+				Describe: func(context.Context, pact.ContentDescribe) (*pact.ContentRecord, error) {
+					return nil, nil
+				},
+			},
+		}
+	}
+	return []pact.MenuItemType{
+		typ(pageType, "Pages", pageModel, "/pages/:slug"),
+		typ(offerType, "Offers", offerModel, "/offers/:slug"),
+	}
+}
+
+var _ pact.HasMenuItemTypes = contentPlugin{}
+
+func init() {
+	party.Register(contentPlugin{})
+}
+
+// page1 is a published record in one locale at /pages/<slug>.
+func page1(id int, slug string, mod time.Time) fakeRecord {
+	return fakeRecord{ID: id, Published: true, LastMod: mod, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/" + slug, Title: "Page " + slug}}}
+}

contract/contract.go

@@ -0,0 +1,148 @@
+// Package contract is the published seam of the golem15.sitemap plugin:
+// the Filter another plugin implements to keep addresses out of the
+// sitemaps or move them to another file group, and the Listing other
+// plugins read to show the sitemap's addresses (an HTML sitemap, IndexNow).
+//
+// It is a package of its own so a plugin can import it without importing
+// the sitemap plugin's root package, whose init registers the plugin. The
+// sitemap plugin depends on no implementation: it looks a Filter up with
+//
+//	f, ok := app.Lookup[contract.Filter]()
+//
+// and includes every published address when none is published. A plugin
+// publishes one in its Register:
+//
+//	app.Publish[contract.Filter](myFilter{})
+package contract
+
+import (
+	"context"
+	"time"
+)
+
+// The reasons a Decision may give for leaving an address out. The sitemap
+// counts excluded addresses by reason; another reason is counted under its
+// own name.
+const (
+	// ReasonNoindex is an address whose page is set to noindex.
+	ReasonNoindex = "noindex"
+	// ReasonCanonical is an address whose canonical URL points elsewhere.
+	ReasonCanonical = "canonical"
+	// ReasonRedirect is an address that redirects.
+	ReasonRedirect = "redirect"
+	// ReasonType is an address of a content type switched off in the
+	// sitemap settings; the sitemap sets it itself, a Filter need not.
+	ReasonType = "type"
+)
+
+// Candidate is one address the sitemap would list: TypeKey is the content
+// type key (a signpost menu item type, or "extra" for an additional address
+// of the settings), RecordID the record's ID within it (for an additional
+// address the hex SHA-256 of its path), Locale the locale ("" when the
+// address has none) and URL its public path or absolute URL.
+type Candidate struct {
+	TypeKey  string
+	RecordID string
+	Locale   string
+	URL      string
+}
+
+// Decision is a Filter's answer for one Candidate. Exclude leaves the
+// address out of the sitemap files, counted under Reason. Group, when not
+// empty, puts the address in another file group: the files are then named
+// sitemap-<group>-<n>.xml, so a Group must use lowercase letters, digits and
+// hyphens (another value is ignored). LastMod, when set, is the first
+// lastmod of an address the sitemap has not listed before; it never moves
+// the lastmod of a listed address.
+type Decision struct {
+	Exclude bool
+	Reason  string
+	Group   string
+	LastMod *time.Time
+}
+
+// Group names a file group a Filter's decisions use: Key is the group as
+// in Decision.Group, Titles its heading per locale for an HTML sitemap.
+type Group struct {
+	Key    string
+	Titles map[string]string
+}
+
+// Filter decides which addresses the sitemap lists. Decide receives one
+// batch (a page of records of one content type, or the additional
+// addresses) and answers one Decision per Candidate in the same order; an
+// answer of another length or an error fails the rebuild, which keeps the
+// files it served before. Groups lists the groups the Filter's decisions
+// may name, with their titles.
+type Filter interface {
+	Decide(ctx context.Context, batch []Candidate) ([]Decision, error)
+	Groups(ctx context.Context) ([]Group, error)
+}
+
+// Listing is what the sitemap publishes for other plugins: its settings and
+// the listed addresses of each group. The sitemap plugin publishes it with
+// app.Publish[contract.Listing] in its Register.
+type Listing interface {
+	Settings(ctx context.Context) (ListingSettings, error)
+	Entries(ctx context.Context, q EntriesQuery) (EntriesPage, error)
+}
+
+// ListingSettings are the sitemap's settings as other plugins need them.
+// SiteURL is the absolute site address every address starts with ("" when
+// none is configured), IndexURL the absolute address of the sitemap index,
+// DefaultLocale the site's default locale ("" when not set) and Groups the
+// file groups with listed addresses. HTMLEnabled, HTMLPaths (locale to
+// path), HTMLPageSize and KeyFile (the IndexNow key file name) describe the
+// HTML sitemap and IndexNow; they stay zero until those settings exist.
+type ListingSettings struct {
+	SiteURL       string
+	IndexURL      string
+	DefaultLocale string
+	Groups        []GroupInfo
+	HTMLEnabled   bool
+	HTMLPaths     map[string]string
+	HTMLPageSize  int
+	KeyFile       string
+}
+
+// GroupInfo is one file group: Key is the group (as Entries takes it), Slug
+// the name part of its files (sitemap-<slug>-<n>.xml; today the same as
+// Key), Titles its heading per locale and Count its listed addresses.
+type GroupInfo struct {
+	Key    string
+	Slug   string
+	Titles map[string]string
+	Count  int
+}
+
+// EntriesQuery asks for one page of a group's listed addresses. Locale,
+// when set, keeps the addresses of that locale and those without one.
+// After is the Next of the previous page ("" for the first) and Limit the
+// page size (at most 1000; 0 means 100).
+type EntriesQuery struct {
+	Group  string
+	Locale string
+	After  string
+	Limit  int
+}
+
+// Entry is one listed address. URL is absolute (on ListingSettings.SiteURL)
+// and LastMod its last real change, nil when unknown.
+type Entry struct {
+	Group    string
+	TypeKey  string
+	RecordID string
+	Locale   string
+	URL      string
+	Title    string
+	LastMod  *time.Time
+}
+
+// EntriesPage is one page of Entries in a stable order: Next is the After
+// of the following page ("" on the last page) and Total the number of
+// addresses the query matches across all pages.
+type EntriesPage struct {
+	Entries []Entry
+	Next    string
+	Total   int
+}

jobs.go

@@ -0,0 +1,26 @@
+package sitemap
+
+import (
+	"context"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/conga"
+	"git.golem15.com/golem15/summercms/modules/pact"
+)
+
+// Jobs registers the rebuild job (classes.RebuildKind) on conga.
+func (p *Plugin) Jobs() []pact.Job {
+	return []pact.Job{
+		conga.Job(p.rebuild, conga.MaxAttempts(3)),
+	}
+}
+
+// rebuild runs one rebuild of the slot table.
+func (p *Plugin) rebuild(ctx context.Context, _ classes.RebuildArgs) error {
+	res, err := classes.Rebuild(ctx, p.app)
+	if err != nil {
+		return err
+	}
+	pluginLog.Info("golem15.sitemap: rebuilt", "listed", res.Listed, "files", res.Files, "added", len(res.Added), "changed", len(res.Changed), "removed", len(res.Removed))
+	return nil
+}

models/settings.go

@@ -3,10 +3,12 @@ package models
 import (
 	"net/url"
 	"strings"
+	"time"
 	"unicode"
 	"unicode/utf8"
 
 	"git.golem15.com/golem15/summercms/modules/cabana"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"gorm.io/gorm"
 )
 
@@ -53,8 +55,44 @@ type Settings struct {
 	PriorityBlog          *string `gorm:"column:priority_blog"`
 	PriorityCategories    *string `gorm:"column:priority_categories"`
 	PriorityLegal         *string `gorm:"column:priority_legal"`
+
+	SiteURL             *string                     `gorm:"column:site_url"`
+	FileSize            int                         `gorm:"column:file_size"`
+	DefaultLocale       *string                     `gorm:"column:default_locale"`
+	ExtraURLs           lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"`
+	RebuildPendingUntil *time.Time                  `gorm:"column:rebuild_pending_until"`
+	NumberedSize        *int                        `gorm:"column:numbered_size"`
+	BuiltAt             *time.Time                  `gorm:"column:built_at"`
+	RebuildRequestedAt  *time.Time                  `gorm:"column:rebuild_requested_at"`
 }
 
+// DefaultFileSize is the number of addresses per sitemap file when the
+// setting is not stored (D-30); MaxFileSize is the protocol's limit.
+const (
+	DefaultFileSize = 1000
+	MaxFileSize     = 50000
+)
+
+// ExtraURL is one additional address of the settings: URL is a path (or an
+// absolute http(s) URL on the site), LastMod its optional last change as
+// the admin's date picker sends it.
+type ExtraURL struct {
+	URL     string `json:"url"`
+	LastMod string `json:"lastmod"`
+}
+
+// EffectiveFileSize is the stored file size, or DefaultFileSize when it is
+// outside 1..MaxFileSize.
+func (s Settings) EffectiveFileSize() int {
+	if s.FileSize < 1 || s.FileSize > MaxFileSize {
+		return DefaultFileSize
+	}
+	return s.FileSize
+}
+
+// Extra returns the additional addresses.
+func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() }
+
 func (Settings) TableName() string { return "golem15_sitemap_settings" }
 
 // Fillable lists only the settings columns.
@@ -114,7 +152,7 @@ func AllowedFrontendURL(u string) bool {
 // DefaultSettings is the settings PHP read when nothing was saved: tiers on
 // with the default priorities.
 func DefaultSettings() Settings {
-	return Settings{ID: 1, EnablePriorityTiers: true}
+	return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize}
 }
 
 // Priority returns the value of a priority setting: the stored text, or the

models/url.go

@@ -0,0 +1,76 @@
+package models
+
+import (
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+)
+
+// ExtraTypeKey is the type_key of the settings' additional addresses in
+// golem15_sitemap_urls; their record_id is the hex SHA-256 of the address.
+const ExtraTypeKey = "extra"
+
+// ExtraGroup is the file group of the additional addresses.
+const ExtraGroup = "pages"
+
+// The excluded reasons the sitemap sets itself. A Filter's reasons
+// (contract.ReasonNoindex and the others) are stored as it gives them.
+const (
+	// ReasonDeleted is an address whose record or locale is gone.
+	ReasonDeleted = "deleted"
+	// ReasonUnpublished is an address whose record is not published.
+	ReasonUnpublished = "unpublished"
+)
+
+// Alternate is one hreflang alternate of an address: Hreflang the locale
+// code or x-default, Href the alternate's path or absolute URL.
+type Alternate struct {
+	Hreflang string `json:"hreflang"`
+	Href     string `json:"href"`
+}
+
+// URL is one row of the slot table golem15_sitemap_urls (D-30): an address
+// of a content type's record in one locale, or an additional address. Grp
+// and FileNo place it in the file sitemap-<grp>-<file_no>.xml; FileNo is
+// nil until the address is numbered. ExcludedReason is "" for a listed
+// address; a row left out keeps its FileNo, so its file keeps a hole.
+type URL struct {
+	ID             uint64                       `gorm:"column:id;primaryKey"`
+	Grp            string                       `gorm:"column:grp"`
+	TypeKey        string                       `gorm:"column:type_key"`
+	RecordID       string                       `gorm:"column:record_id"`
+	Locale         string                       `gorm:"column:locale"`
+	URL            string                       `gorm:"column:url"`
+	Title          string                       `gorm:"column:title"`
+	FileNo         *int                         `gorm:"column:file_no"`
+	FirstSeen      time.Time                    `gorm:"column:first_seen"`
+	LastMod        *time.Time                   `gorm:"column:lastmod"`
+	Alternates     lagoon.Jsonable[[]Alternate] `gorm:"column:alternates"`
+	Images         lagoon.Jsonable[[]string]    `gorm:"column:images"`
+	ExcludedReason string                       `gorm:"column:excluded_reason"`
+	UpdatedAt      time.Time                    `gorm:"column:updated_at"`
+}
+
+func (URL) TableName() string { return "golem15_sitemap_urls" }
+
+func init() {
+	Register(URL{})
+}
+
+// AllowedAddress reports whether u may be a stored address: a path that
+// starts with a single / (no whitespace, control characters, quotes, < or
+// >), or an absolute http(s) URL that AllowedFrontendURL accepts.
+func AllowedAddress(u string) bool {
+	if u == "" || !utf8.ValidString(u) {
+		return false
+	}
+	if strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//") {
+		return !strings.ContainsFunc(u, func(r rune) bool {
+			return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '\\'
+		})
+	}
+	return AllowedFrontendURL(u)
+}

plugin.go

@@ -1,16 +1,15 @@
-// Package sitemap is the golem15.sitemap plugin: it serves /sitemap.xml
-// from the application's one sitemap definition, byte for byte the way the
-// WinterCMS Golem15.Sitemap plugin did, with items that are explicit URLs
-// or menu item types of the framework's registry.
+// Package sitemap is the golem15.sitemap plugin: it serves a sitemap index
+// at /sitemap.xml whose entries are per-group files of a configurable size
+// (sitemap-<slug>-<n>.xml, D-30), built from the published records of every
+// signpost content type and the additional addresses of its settings, with
+// stable file numbers and real lastmod. Other plugins narrow it through the
+// Filter of the contract package and read it through its Listing.
 package sitemap
 
 import (
 	"embed"
-	"fmt"
 	"io/fs"
-	"time"
 
-	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/console"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"git.golem15.com/golem15/sm-sitemap-plugin/updates"
@@ -29,28 +28,22 @@ var (
 	_ pact.HasModels      = (*Plugin)(nil)
 	_ pact.HasRoutes      = (*Plugin)(nil)
 	_ pact.HasCommands    = (*Plugin)(nil)
+	_ pact.HasJobs        = (*Plugin)(nil)
 	_ surf.BucketProvider = (*Plugin)(nil)
 )
 
 //go:embed config
 var configFS embed.FS
 
-// sitemapXSL is the PHP plugin's assets/sitemap.xsl, byte for byte.
-//
-//go:embed assets/sitemap.xsl
-var sitemapXSL []byte
-
 // Plugin is the golem15.sitemap plugin.
 type Plugin struct {
-	app   *backpack.App
-	tiers classes.Tiers
-	cache *sitemapCache
+	app *backpack.App
 }
 
 func (p *Plugin) ID() string { return "golem15.sitemap" }
 
-// Requires is empty: the sitemap reads content types through the framework's
-// menu item type registry, not from a named plugin.
+// Requires is empty: the sitemap reads content types through signpost, not
+// from a named plugin, and a Filter through the service registry.
 func (p *Plugin) Requires() []string { return nil }
 
 func (p *Plugin) Register(app *backpack.App) error {
@@ -58,29 +51,9 @@ func (p *Plugin) Register(app *backpack.App) error {
 	return nil
 }
 
-// Boot compiles the priority tier rules (classes.TiersConfigKey); a rule
-// with an unknown setting or a pattern that does not compile fails the boot.
-// It also starts an empty sitemap cache for CacheConfigKey seconds; a
-// negative value fails the boot.
+// Boot keeps the booted application for the routes, jobs and commands.
 func (p *Plugin) Boot(app *backpack.App) error {
 	p.app = app
-	var tiers classes.Tiers
-	var err error
-	seconds := 0
-	if app != nil {
-		tiers, err = classes.LoadTiers(app.Config)
-		if app.Config != nil {
-			seconds = app.Config.Int(CacheConfigKey)
-		}
-	}
-	if err != nil {
-		return fmt.Errorf("golem15.sitemap: %w", err)
-	}
-	if seconds < 0 {
-		return fmt.Errorf("golem15.sitemap: %s must not be negative, got %d", CacheConfigKey, seconds)
-	}
-	p.tiers = tiers
-	p.cache = newSitemapCache(time.Duration(seconds) * time.Second)
 	return nil
 }
 

routes.go

@@ -1,52 +1,41 @@
 package sitemap
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
+	"bytes"
 	"errors"
-	"net"
 	"net/http"
-	"net/netip"
-	"strings"
-	"sync"
 	"time"
 
 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/pact"
-	"git.golem15.com/golem15/summercms/modules/signpost"
 	"git.golem15.com/golem15/summercms/modules/surf"
 	"gorm.io/gorm"
 )
 
-// The response headers of the PHP routes (live responses).
+// The response headers of the sitemap routes.
 const (
 	contentTypeXML = "application/xml"
-	contentTypeXSL = "text/xsl; charset=UTF-8"
 	cacheControl   = "no-cache, private"
 )
 
-// BucketSitemap is the throttle bucket of GET /sitemap.xml.
+// BucketSitemap is the throttle bucket of the sitemap routes.
 const BucketSitemap = "sitemap-xml"
 
-// CacheConfigKey is the config key of the number of seconds a generated
-// sitemap is served from memory; 0 turns the cache off.
-const CacheConfigKey = "golem15.sitemap.cache_seconds"
+// FilesPath is the route of the per-group files. The index lists them at
+// the site root (sitemap-<slug>-<n>.xml); the site's SEO layer or proxy
+// maps the root names to this route.
+const FilesPath = "/_sitemap/files/"
 
-// cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the
-// request's scheme and Host when app.url is not set) and stored definition
-// and settings. A full cache is emptied.
-const cacheMaxEntries = 16
-
-// Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application
-// root without the backend middleware: search engines and browsers fetch
-// them anonymously. /sitemap.xml resolves every registry item, so it is
-// throttled per client IP (BucketSitemap) and its output is cached.
+// Routes mounts GET /sitemap.xml (the index) and GET
+// /_sitemap/files/{file} (one file) without the backend middleware: search
+// engines fetch them anonymously. Both are served from the slot table, never
+// generated per request, and throttled per client IP (BucketSitemap).
 func (p *Plugin) Routes(r pact.Router) error {
-	r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap)
-	r.Get("/sitemap.xsl", p.sitemapXSL)
+	r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap)
+	r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap)
+	r.Where("file", `sitemap-[a-z0-9-]+-[0-9]+\.xml`)
 	return nil
 }
 
@@ -70,159 +59,119 @@ func (p *Plugin) Buckets() map[string]surf.Bucket {
 	}
 }
 
-// sitemapCache holds generated sitemaps for the configured time. The key
-// covers the root URL and the stored definition and settings, so a save of
-// either is served at once; registry content (a new post) shows after the
-// entry expires. gen lets one generation run at a time, so concurrent misses
-// wait for it instead of each resolving every registry item.
-type sitemapCache struct {
-	ttl     time.Duration
-	gen     sync.Mutex
-	mu      sync.Mutex
-	entries map[string]cachedSitemap
+// indexRow is one file of the index query.
+type indexRow struct {
+	Grp     string     `gorm:"column:grp"`
+	FileNo  int        `gorm:"column:file_no"`
+	LastMod *time.Time `gorm:"column:lastmod"`
 }
 
-type cachedSitemap struct {
-	xml     string
-	expires time.Time
-}
-
-func newSitemapCache(ttl time.Duration) *sitemapCache {
-	return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}}
-}
-
-func (c *sitemapCache) get(key string) (string, bool) {
-	if c == nil || c.ttl <= 0 || key == "" {
-		return "", false
-	}
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	e, ok := c.entries[key]
-	if !ok || time.Now().After(e.expires) {
-		return "", false
-	}
-	return e.xml, true
-}
-
-func (c *sitemapCache) put(key, xml string) {
-	if c == nil || c.ttl <= 0 || key == "" {
-		return
-	}
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if len(c.entries) >= cacheMaxEntries {
-		clear(c.entries)
-	}
-	c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)}
-}
-
-// cacheKey is a digest of the root URL, the definition's items and the
-// settings; "" (no caching) when either does not encode.
-func cacheKey(root string, def models.Definition, settings models.Settings) string {
-	items, err := json.Marshal(def.Items())
-	if err != nil {
-		return ""
-	}
-	set, err := json.Marshal(settings)
-	if err != nil {
-		return ""
-	}
-	h := sha256.New()
-	for _, part := range [][]byte{[]byte(root), items, set} {
-		h.Write(part)
-		h.Write([]byte{0})
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}
-
-// sitemapXSL serves the embedded stylesheet, the PHP file byte for byte.
-func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) {
-	w.Header().Set("Content-Type", contentTypeXSL)
-	w.Header().Set("Cache-Control", cacheControl)
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(sitemapXSL)
-}
-
-// sitemapXML serves the definition with the fixed theme key: 404 when there
-// is none, an empty application/xml body when it has no items, else the
-// generated XML with the settings' frontend_url rewrite applied.
-func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) {
-	app := p.app
-	db, err := database(app)
+// sitemapIndex serves the sitemap index: one <sitemap> per file that lists
+// at least one address, with the newest lastmod of its addresses. Without a
+// site address (the setting, golem15.sitemap.site_url or app.url) it is a
+// 404: a <loc> is never built from the request's Host.
+func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) {
+	db, err := database(p.app)
 	if err != nil {
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
-	var defs []models.Definition
-	if err := db.WithContext(r.Context()).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil {
-		pluginLog.Error("golem15.sitemap: load definition", "error", err)
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-	if len(defs) == 0 {
-		pluginLog.Info("golem15.sitemap: No sitemap definition was found. Try creating one first.")
-		http.NotFound(w, r)
-		return
-	}
-	settings, err := loadSettings(db.WithContext(r.Context()))
+	db = db.WithContext(r.Context())
+	settings, err := classes.LoadSettings(db)
 	if err != nil {
 		pluginLog.Error("golem15.sitemap: load settings", "error", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
-	root := backendURL(app, r)
-	xml, err := p.generate(r, root, defs[0], settings)
-	if err != nil {
-		pluginLog.Error("golem15.sitemap: generate", "error", err)
+	site := classes.SiteURL(p.app, settings)
+	if site == "" {
+		pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url")
+		http.NotFound(w, r)
+		return
+	}
+	var rows []indexRow
+	if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls
+WHERE excluded_reason = '' AND file_no IS NOT NULL
+GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil {
+		pluginLog.Error("golem15.sitemap: index", "error", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
+	files := make([]classes.IndexFile, 0, len(rows))
+	for _, row := range rows {
+		files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod})
+	}
+	var buf bytes.Buffer
+	if err := classes.WriteIndex(&buf, files); err != nil {
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	writeXML(w, buf.Bytes(), false)
+}
+
+// sitemapFile serves one file: its listed addresses in the order they were
+// first seen. An unknown file, or one without a listed address, is a 404.
+func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) {
+	grp, n, ok := classes.ParseFileName(r.PathValue("file"))
+	if !ok {
+		http.NotFound(w, r)
+		return
+	}
+	db, err := database(p.app)
+	if err != nil {
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	db = db.WithContext(r.Context())
+	settings, err := classes.LoadSettings(db)
+	if err != nil {
+		pluginLog.Error("golem15.sitemap: load settings", "error", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	site := classes.SiteURL(p.app, settings)
+	var rows []models.URL
+	if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil {
+		pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	entries := make([]classes.FileEntry, 0, len(rows))
+	for _, row := range rows {
+		loc := classes.Absolute(site, row.URL)
+		if loc == "" {
+			continue
+		}
+		entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod}
+		for _, a := range row.Alternates.Get() {
+			if href := classes.Absolute(site, a.Href); href != "" {
+				entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href})
+			}
+		}
+		entries = append(entries, entry)
+	}
+	if len(entries) == 0 {
+		http.NotFound(w, r)
+		return
+	}
+	var buf bytes.Buffer
+	if err := classes.WriteURLSet(&buf, entries); err != nil {
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	writeXML(w, buf.Bytes(), true)
+}
+
+// writeXML answers 200 with an XML body. A file carries X-Robots-Tag
+// noindex, so search engines read it without indexing it as a page.
+func writeXML(w http.ResponseWriter, body []byte, noindex bool) {
 	w.Header().Set("Content-Type", contentTypeXML)
 	w.Header().Set("Cache-Control", cacheControl)
+	if noindex {
+		w.Header().Set("X-Robots-Tag", "noindex")
+	}
 	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write([]byte(xml))
-}
-
-// generate returns the sitemap of def with the frontend_url rewrite
-// applied, from the cache when it holds it.
-func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) {
-	cache := p.cache
-	key := ""
-	if cache != nil && cache.ttl > 0 {
-		key = cacheKey(root, def, settings)
-		if xml, ok := cache.get(key); ok {
-			return xml, nil
-		}
-		cache.gen.Lock()
-		defer cache.gen.Unlock()
-		if xml, ok := cache.get(key); ok {
-			return xml, nil
-		}
-	}
-	app := p.app
-	gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events}
-	if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
-		gen.Resolver = reg
-	}
-	xml, err := gen.Generate(r.Context(), def.Items())
-	if err != nil {
-		return "", err
-	}
-	xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL))
-	cache.put(key, xml)
-	return xml, nil
-}
-
-// loadSettings reads the settings row; without one the PHP defaults apply.
-func loadSettings(db *gorm.DB) (models.Settings, error) {
-	var rows []models.Settings
-	if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil {
-		return models.Settings{}, err
-	}
-	if len(rows) == 0 {
-		return models.DefaultSettings(), nil
-	}
-	return rows[0], nil
+	_, _ = w.Write(body)
 }
 
 // database returns the app's published *gorm.DB.
@@ -236,59 +185,3 @@ func database(app *backpack.App) (*gorm.DB, error) {
 	}
 	return db, nil
 }
-
-// backendURL is the root URL the sitemap's absolute URLs start with, PHP
-// url('/') right-trimmed of slashes: app.url when it is set, else the
-// request's scheme and Host. The scheme is https for a TLS connection, or
-// when the immediate peer is in http.trusted_proxies and its last
-// X-Forwarded-Proto value says https; a client that sends the header
-// directly is ignored.
-func backendURL(app *backpack.App, r *http.Request) string {
-	if app != nil && app.Config != nil {
-		if u := strings.TrimRight(strings.TrimSpace(app.Config.String("app.url")), "/"); u != "" {
-			return u
-		}
-	}
-	scheme := "http"
-	if isHTTPS(app, r) {
-		scheme = "https"
-	}
-	return strings.TrimRight(scheme+"://"+r.Host, "/")
-}
-
-func isHTTPS(app *backpack.App, r *http.Request) bool {
-	if r.TLS != nil {
-		return true
-	}
-	if app == nil || app.Config == nil {
-		return false
-	}
-	trusted := surf.TrustedProxies(app.Config)
-	if len(trusted) == 0 || !peerTrusted(r.RemoteAddr, trusted) {
-		return false
-	}
-	values := r.Header.Values("X-Forwarded-Proto")
-	if len(values) == 0 {
-		return false
-	}
-	hops := strings.Split(values[len(values)-1], ",")
-	return strings.EqualFold(strings.TrimSpace(hops[len(hops)-1]), "https")
-}
-
-func peerTrusted(remoteAddr string, trusted []netip.Prefix) bool {
-	host, _, err := net.SplitHostPort(remoteAddr)
-	if err != nil {
-		host = remoteAddr
-	}
-	addr, err := netip.ParseAddr(strings.Trim(host, "[]"))
-	if err != nil {
-		return false
-	}
-	addr = addr.Unmap()
-	for _, p := range trusted {
-		if p.Contains(addr) {
-			return true
-		}
-	}
-	return false
-}

routes_test.go

@@ -1,22 +1,21 @@
 package sitemap
 
 import (
-	"crypto/tls"
-	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/compass"
 )
 
-// TestBackendURL pins the root URL the sitemap's absolute URLs and the
-// frontend_url rewrite start from: app.url when set; else the request's
-// Host with https for TLS, or for the last X-Forwarded-Proto hop of a peer
-// in http.trusted_proxies, while a client that sends the header directly
-// gets http.
-func TestBackendURL(t *testing.T) {
+// TestSiteURLPrecedence pins the site address every <loc> starts with
+// (CONTEXT 2026-10-28): the setting, else golem15.sitemap.site_url, else
+// app.url, each only when it is an absolute http(s) URL, trailing slashes
+// trimmed; nothing configured is "" (the request's Host is never used).
+func TestSiteURLPrecedence(t *testing.T) {
 	app := func(t *testing.T, files map[string]string) *backpack.App {
 		t.Helper()
 		dir := t.TempDir()
@@ -31,39 +30,52 @@ func TestBackendURL(t *testing.T) {
 		}
 		return backpack.New(cfg)
 	}
-	trusted := app(t, map[string]string{"http.yaml": "trusted_proxies:\n  - 10.0.0.0/8\n"})
+	text := func(s string) *string { return &s }
+	both := app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n", "golem15.yaml": "sitemap:\n  site_url: https://config.example.test/\n"})
+	appOnly := app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n"})
 	for _, tc := range []struct {
-		name   string
-		app    *backpack.App
-		remote string
-		proto  []string
-		tls    bool
-		want   string
+		name     string
+		app      *backpack.App
+		settings models.Settings
+		want     string
 	}{
-		{"app.url wins", app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n"}), "203.0.113.9:1234", []string{"http"}, false, "https://api.example.test"},
-		{"plain request", trusted, "203.0.113.9:1234", nil, false, "http://h.test"},
-		{"TLS", trusted, "203.0.113.9:1234", nil, true, "https://h.test"},
-		{"trusted proxy says https", trusted, "10.1.2.3:5000", []string{"https"}, false, "https://h.test"},
-		{"trusted proxy, last hop wins", trusted, "10.1.2.3:5000", []string{"https, http"}, false, "http://h.test"},
-		{"trusted proxy, last header wins", trusted, "10.1.2.3:5000", []string{"http", "HTTPS"}, false, "https://h.test"},
-		{"untrusted client says https", trusted, "203.0.113.9:1234", []string{"https"}, false, "http://h.test"},
-		{"no trusted proxies configured", app(t, nil), "10.1.2.3:5000", []string{"https"}, false, "http://h.test"},
-		{"nil app", nil, "10.1.2.3:5000", []string{"https"}, false, "http://h.test"},
-		{"IPv4-mapped trusted peer", trusted, "[::ffff:10.1.2.3]:5000", []string{"https"}, false, "https://h.test"},
-		{"unparsable peer", trusted, "not-an-address", []string{"https"}, false, "http://h.test"},
+		{"setting wins", both, models.Settings{SiteURL: text("https://www.example.test/")}, "https://www.example.test"},
+		{"config key", both, models.Settings{}, "https://config.example.test"},
+		{"app.url", appOnly, models.Settings{}, "https://api.example.test"},
+		{"unsafe setting skipped", appOnly, models.Settings{SiteURL: text("javascript:alert(1)")}, "https://api.example.test"},
+		{"nothing configured", app(t, nil), models.Settings{}, ""},
+		{"nil app", nil, models.Settings{}, ""},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			r := httptest.NewRequest("GET", "http://h.test/sitemap.xml", nil)
-			r.RemoteAddr = tc.remote
-			for _, v := range tc.proto {
-				r.Header.Add("X-Forwarded-Proto", v)
-			}
-			if tc.tls {
-				r.TLS = &tls.ConnectionState{}
-			}
-			if got := backendURL(tc.app, r); got != tc.want {
-				t.Fatalf("backendURL = %q, want %q", got, tc.want)
+			if got := classes.SiteURL(tc.app, tc.settings); got != tc.want {
+				t.Fatalf("SiteURL = %q, want %q", got, tc.want)
 			}
 		})
 	}
 }
+
+// TestFileNames pins the per-group file names the index lists and the file
+// route accepts.
+func TestFileNames(t *testing.T) {
+	if got := classes.FileName("posts", 3); got != "sitemap-posts-3.xml" {
+		t.Fatalf("FileName = %q", got)
+	}
+	for name, want := range map[string]struct {
+		grp string
+		n   int
+		ok  bool
+	}{
+		"sitemap-posts-1.xml":          {"posts", 1, true},
+		"sitemap-landing-pages-12.xml": {"landing-pages", 12, true},
+		"sitemap-posts-2-1.xml":        {"posts-2", 1, true},
+		"sitemap-posts-0.xml":          {"", 0, false},
+		"sitemap-Posts-1.xml":          {"", 0, false},
+		"sitemap-posts.xml":            {"", 0, false},
+		"sitemap-../x-1.xml":           {"", 0, false},
+	} {
+		grp, n, ok := classes.ParseFileName(name)
+		if grp != want.grp || n != want.n || ok != want.ok {
+			t.Errorf("ParseFileName(%q) = %q %d %v, want %+v", name, grp, n, ok, want)
+		}
+	}
+}

sitemap_index_test.go

@@ -0,0 +1,133 @@
+package sitemap_test
+
+import (
+	"context"
+	"net/http"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/pact"
+)
+
+// bootIndex boots the fake content plugin and the sitemap on a fresh
+// database with an empty content store.
+func bootIndex(t *testing.T, extra ...string) *sitemapEnv {
+	t.Helper()
+	contentStore.reset(t)
+	return bootPlugins(t, append([]string{contentPluginID, "golem15.sitemap"}, extra...))
+}
+
+// rebuild runs the plugin's rebuild job once, as a worker would.
+func (e *sitemapEnv) rebuild(t *testing.T) {
+	t.Helper()
+	for _, p := range e.plugins {
+		hj, ok := p.(pact.HasJobs)
+		if !ok || p.ID() != "golem15.sitemap" {
+			continue
+		}
+		for _, j := range hj.Jobs() {
+			if err := j.Work(context.Background(), classes.RebuildArgs{}); err != nil {
+				t.Fatalf("rebuild job: %v", err)
+			}
+			return
+		}
+	}
+	t.Fatal("the sitemap registers no rebuild job")
+}
+
+// setFileSize stores the file size setting.
+func (e *sitemapEnv) setFileSize(t *testing.T, n int) {
+	t.Helper()
+	if err := e.gdb.Exec(`UPDATE golem15_sitemap_settings SET file_size = ? WHERE id = 1`, n).Error; err != nil {
+		t.Fatal(err)
+	}
+}
+
+// locs lists the <loc> values of a sitemap body in order.
+func locs(body string) []string {
+	var out []string
+	for _, m := range regexp.MustCompile(`<loc>([^<]*)</loc>`).FindAllStringSubmatch(body, -1) {
+		out = append(out, m[1])
+	}
+	return out
+}
+
+// TestSitemapIndex is the tracer of D-30: a content type's published
+// records appear, after a rebuild job, in an index at /sitemap.xml that
+// links per-group files of file_size addresses with absolute <loc> and a
+// real <lastmod>; a draft is absent, no priority or change frequency is
+// written, values are XML-escaped (T-33-09) and an unknown file is a 404.
+func TestSitemapIndex(t *testing.T) {
+	e := bootIndex(t)
+	e.setFileSize(t, 2)
+	base := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
+	for i, slug := range []string{"a", "b", "c", "d"} {
+		contentStore.put(pageType, page1(i+1, slug, base.Add(time.Duration(i)*time.Hour)))
+	}
+	contentStore.put(pageType, fakeRecord{ID: 5, Published: true, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: `/pages/fish&chips?x=<1>"`, Title: `Fish & "chips" <b>`}}})
+	contentStore.put(pageType, fakeRecord{ID: 6, Published: false, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/draft", Title: "Draft"}}})
+
+	if rec := e.get(t, "/sitemap.xml"); rec.Code != http.StatusOK || len(locs(rec.Body.String())) != 0 {
+		t.Fatalf("index before the first rebuild: %d %s", rec.Code, rec.Body.String())
+	}
+	e.rebuild(t)
+
+	rec := e.get(t, "/sitemap.xml")
+	body := rec.Body.String()
+	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" {
+		t.Fatalf("GET /sitemap.xml: %d %q %s", rec.Code, rec.Header().Get("Content-Type"), body)
+	}
+	if !strings.HasPrefix(body, `<?xml version="1.0" encoding="UTF-8"?>`) || !strings.Contains(body, `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`) {
+		t.Fatalf("index head:\n%s", body)
+	}
+	want := []string{
+		"https://api.example.test/sitemap-pages-1.xml",
+		"https://api.example.test/sitemap-pages-2.xml",
+		"https://api.example.test/sitemap-pages-3.xml",
+	}
+	if got := locs(body); strings.Join(got, " ") != strings.Join(want, " ") {
+		t.Fatalf("index locs = %v, want %v\n%s", got, want, body)
+	}
+	if n := strings.Count(body, "<lastmod>"); n != 3 {
+		t.Fatalf("index has %d lastmod elements:\n%s", n, body)
+	}
+	if !strings.Contains(body, "<sitemap><loc>https://api.example.test/sitemap-pages-1.xml</loc><lastmod>2026-10-01T09:00:00Z</lastmod></sitemap>") {
+		t.Fatalf("file 1 lastmod is not the newest of its addresses:\n%s", body)
+	}
+
+	rec = e.get(t, "/_sitemap/files/sitemap-pages-1.xml")
+	file := rec.Body.String()
+	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" {
+		t.Fatalf("file 1: %d %v %s", rec.Code, rec.Header(), file)
+	}
+	if got := locs(file); strings.Join(got, " ") != "https://api.example.test/pages/a https://api.example.test/pages/b" {
+		t.Fatalf("file 1 locs = %v\n%s", got, file)
+	}
+	if !strings.Contains(file, "<url><loc>https://api.example.test/pages/a</loc><lastmod>2026-10-01T08:00:00Z</lastmod></url>") {
+		t.Fatalf("file 1 lacks a real lastmod:\n%s", file)
+	}
+	file3 := e.get(t, "/_sitemap/files/sitemap-pages-3.xml").Body.String()
+	if !strings.Contains(file3, "<loc>https://api.example.test/pages/fish&amp;chips?x=&lt;1&gt;&#34;</loc>") {
+		t.Fatalf("file 3 does not escape the address:\n%s", file3)
+	}
+	all := body + file + e.get(t, "/_sitemap/files/sitemap-pages-2.xml").Body.String() + file3
+	for _, banned := range []string{"<priority>", "<changefreq>", "/pages/draft", "xml-stylesheet", "xmlns:xhtml"} {
+		if strings.Contains(all, banned) {
+			t.Errorf("the sitemap contains %q:\n%s", banned, all)
+		}
+	}
+	for _, path := range []string{
+		"/_sitemap/files/sitemap-pages-4.xml",
+		"/_sitemap/files/sitemap-posts-1.xml",
+		"/_sitemap/files/sitemap-pages-0.xml",
+		"/_sitemap/files/other.xml",
+		"/sitemap.xsl",
+	} {
+		if rec := e.get(t, path); rec.Code != http.StatusNotFound {
+			t.Errorf("GET %s: %d, want 404", path, rec.Code)
+		}
+	}
+}

sitemap_smoke_test.go

@@ -15,9 +15,7 @@ import (
 	"testing"
 	"time"
 
-	sitemap "git.golem15.com/golem15/sm-sitemap-plugin"
-	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
-	"git.golem15.com/golem15/sm-sitemap-plugin/console"
+	_ "git.golem15.com/golem15/sm-sitemap-plugin"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/bonfire"
@@ -27,7 +25,6 @@ import (
 	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/pact"
 	"git.golem15.com/golem15/summercms/modules/party"
-	"git.golem15.com/golem15/summercms/modules/signpost"
 	"git.golem15.com/golem15/summercms/modules/surf"
 	"gorm.io/gorm"
 )
@@ -101,15 +98,22 @@ func init() {
 
 // sitemapEnv is a booted application with the sitemap and the fake types.
 type sitemapEnv struct {
-	h   http.Handler
-	app *backpack.App
-	gdb *gorm.DB
+	h       http.Handler
+	app     *backpack.App
+	gdb     *gorm.DB
+	plugins []party.Plugin
 }
 
-// bootSitemap boots the application. The sitemap cache is off
-// (golem15.sitemap.cache_seconds 0): the smoke tests change listeners and
-// registry content between requests. overrides set further config keys.
+// bootSitemap boots the application with the sitemap and the fake types;
+// overrides set further config keys.
 func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
+	t.Helper()
+	return bootPlugins(t, []string{testTypesPluginID, "golem15.sitemap"}, overrides...)
+}
+
+// bootPlugins boots the application with the plugins ids on a fresh
+// database; overrides set further config keys.
+func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv {
 	t.Helper()
 	sqlDB := freshDB(t)
 	gdb, err := lagoon.Use(t.Context(), sqlDB)
@@ -132,7 +136,6 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
 		"http.body_limits.upload_bytes":  8388608,
 		"mail.driver":                    "log",
 		"mail.from":                      "noreply@example.test",
-		sitemap.CacheConfigKey:           0,
 	}
 	for _, o := range overrides {
 		for key, value := range o {
@@ -148,7 +151,7 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
 	if err := lagoon.Publish(app, sqlDB, gdb); err != nil {
 		t.Fatal(err)
 	}
-	plugins, err := party.Activate(app, []string{testTypesPluginID, "golem15.sitemap"})
+	plugins, err := party.Activate(app, ids)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -159,7 +162,7 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
 	if err != nil {
 		t.Fatal(err)
 	}
-	return &sitemapEnv{h: h, app: app, gdb: gdb}
+	return &sitemapEnv{h: h, app: app, gdb: gdb, plugins: plugins}
 }
 
 func (e *sitemapEnv) get(t *testing.T, path string) *httptest.ResponseRecorder {
@@ -184,100 +187,6 @@ func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem
 
 func text(s string) *string { return &s }
 
-// TestSitemapCacheAndThrottle pins WR-05: with the cache on, a repeated GET
-// is served from memory (a listener added later does not run), a save of
-// the definition is served at once, and the 61st request of a client within
-// a minute is a 429.
-func TestSitemapCacheAndThrottle(t *testing.T) {
-	e := bootSitemap(t, map[string]any{sitemap.CacheConfigKey: 60})
-	def := e.storeDefinition(t, []models.DefinitionItem{
-		{Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")},
-		{Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")},
-	})
-	requests := 0
-	get := func() *httptest.ResponseRecorder {
-		requests++
-		return e.get(t, "/sitemap.xml")
-	}
-	first := get()
-	if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), "<loc>https://api.example.test/</loc>") {
-		t.Fatalf("first GET: %d %s", first.Code, first.Body.String())
-	}
-	e.app.Events.Listen("acme.skipall", func(_ context.Context, ev *classes.BeforeAddItem) error {
-		ev.Skip = true
-		return nil
-	})
-	if again := get(); again.Code != http.StatusOK || again.Body.String() != first.Body.String() {
-		t.Fatalf("a repeated GET was generated again:\n%s", again.Body.String())
-	}
-	def.SetItems(append(def.Items(), models.DefinitionItem{Type: models.ItemTypeURL, URL: text("/x")}))
-	if err := e.gdb.Save(&def).Error; err != nil {
-		t.Fatal(err)
-	}
-	if saved := get(); saved.Code != http.StatusOK || strings.Contains(saved.Body.String(), "<url>") {
-		t.Fatalf("a saved definition was served from the cache:\n%s", saved.Body.String())
-	}
-	for requests < 60 {
-		if rec := get(); rec.Code != http.StatusOK {
-			t.Fatalf("request %d: %d", requests, rec.Code)
-		}
-	}
-	if rec := get(); rec.Code != http.StatusTooManyRequests {
-		t.Fatalf("request %d: %d, want 429", requests, rec.Code)
-	}
-	if rec := e.get(t, "/sitemap.xsl"); rec.Code != http.StatusOK {
-		t.Fatalf("/sitemap.xsl is throttled with the sitemap: %d", rec.Code)
-	}
-}
-
-// TestSitemapTracer stores a definition with a url item and a registry
-// item, GETs /sitemap.xml and compares the body with the PHP golden.
-func TestSitemapTracer(t *testing.T) {
-	e := bootSitemap(t)
-
-	if rec := e.get(t, "/sitemap.xml"); rec.Code != http.StatusNotFound {
-		t.Fatalf("no definition: %d, want 404", rec.Code)
-	}
-
-	def := e.storeDefinition(t, nil)
-	rec := e.get(t, "/sitemap.xml")
-	if rec.Code != http.StatusOK || rec.Body.Len() != 0 || rec.Header().Get("Content-Type") != "application/xml" {
-		t.Fatalf("empty definition: %d %q %q", rec.Code, rec.Header().Get("Content-Type"), rec.Body.String())
-	}
-
-	def.SetItems([]models.DefinitionItem{
-		{Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")},
-		{Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")},
-	})
-	if err := e.gdb.Save(&def).Error; err != nil {
-		t.Fatal(err)
-	}
-	rec = e.get(t, "/sitemap.xml")
-	if rec.Code != http.StatusOK {
-		t.Fatalf("GET /sitemap.xml: %d %s", rec.Code, rec.Body.String())
-	}
-	if ct := rec.Header().Get("Content-Type"); ct != "application/xml" {
-		t.Fatalf("Content-Type = %q", ct)
-	}
-	want, err := os.ReadFile(filepath.Join("classes", "testdata", "tracer.golden"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := rec.Body.String(); got != string(want) {
-		t.Fatalf("/sitemap.xml differs from the PHP golden\n got: %q\nwant: %q", got, want)
-	}
-
-	// The stored data keeps the PHP item keys.
-	var raw string
-	if err := e.gdb.Raw(`SELECT data FROM golem15_sitemap_definitions WHERE id = ?`, def.ID).Scan(&raw).Error; err != nil {
-		t.Fatal(err)
-	}
-	const wantData = `[{"changefreq":"always","priority":"1.0","nesting":false,"type":"url","url":"/","reference":null,"cmsPage":null,"urlPattern":null},{"changefreq":"weekly","priority":"0.5","nesting":true,"type":"acme-post","url":null,"reference":"7","cmsPage":null,"urlPattern":""}]`
-	if raw != wantData {
-		t.Fatalf("stored data = %s", raw)
-	}
-}
-
 func writeFile(t *testing.T, dir, name, body string) string {
 	t.Helper()
 	path := filepath.Join(dir, name)
@@ -292,163 +201,6 @@ func testOutput() (bonfire.Output, *bytes.Buffer) {
 	return bonfire.NewOutput(strings.NewReader(""), &buf, &buf), &buf
 }
 
-// TestSitemapImportSmoke copies a Winter export through sitemap:import-winter,
-// then checks the frontend_url rewrite, the stylesheet route, a
-// BeforeAddItem listener and sitemap:import.
-func TestSitemapImportSmoke(t *testing.T) {
-	e := bootSitemap(t)
-	ctx := context.Background()
-	reg, ok := e.app.Lookup[*signpost.Registry]()
-	if !ok {
-		t.Fatal("signpost registry is not published")
-	}
-	dir := t.TempDir()
-	data := `[{"changefreq":"always","priority":"1.0","nesting":"0","type":"url","url":"/","code":null,"reference":null,"cmsPage":null,"urlPattern":null,"exists":false,"fillable":["changefreq"]},` +
-		`{"changefreq":"monthly","priority":"0.5","nesting":"0","type":"static-page","url":null,"reference":"about","cmsPage":null,"urlPattern":null},` +
-		`{"changefreq":"weekly","priority":"0.4","nesting":"1","type":"blog-posts","url":null,"reference":null,"cmsPage":"blog/post","urlPattern":"/news/:slug"},` +
-		`{"changefreq":"weekly","priority":"0.4","nesting":"0","type":"acme-post","url":null,"reference":"99","cmsPage":null,"urlPattern":null},` +
-		`{"changefreq":"daily","priority":"0.7","nesting":"false","type":"cms-page","url":"","reference":"news","cmsPage":"","urlPattern":""}]`
-	quotedRaw, _ := json.Marshal(data)
-	quoted := string(quotedRaw)
-	defs := writeFile(t, dir, "definitions.jsonl", `{"id": 3, "theme": "site-theme", "data": `+quoted+`, "created_at": "2026-01-01 10:00:00", "updated_at": "2026-01-02 10:00:00"}`+"\n")
-	settings := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test/","frontend_default_locale":"pl","enable_priority_tiers":"0","priority_homepage":"0.9","legacy_key":"x"}`)
-
-	out, buf := testOutput()
-	report, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{
-		DefinitionsFile: defs, SettingsFile: settings, Maps: map[string]string{"blog-posts": "acme-list"},
-		PageURLs: map[string]string{"news": "/news"},
-	}, out)
-	if err != nil {
-		t.Fatalf("import-winter: %v\n%s", err, buf.String())
-	}
-	if report.Kept != 3 || report.Dropped != 2 || report.Remapped != 2 || !report.Settings {
-		t.Fatalf("report = %+v\n%s", report, buf.String())
-	}
-	for _, want := range []string{
-		"item 1 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)",
-		"item 2 (type 'blog-posts'): type 'blog-posts' mapped to 'acme-list'",
-		"item 2 (type 'blog-posts'): cmsPage 'blog/post' dropped",
-		"item 3 (type 'acme-post', reference '99'): dropped (reference '99' is not among the current references of 'acme-post')",
-		"item 4 (type 'cms-page', reference 'news'): copied as type 'url' with url '/news'",
-		"setting 'legacy_key' ignored",
-		"theme 'site-theme'",
-	} {
-		if !strings.Contains(buf.String(), want) {
-			t.Errorf("report lacks %q:\n%s", want, buf.String())
-		}
-	}
-	var def models.Definition
-	if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil {
-		t.Fatal(err)
-	}
-	if items := def.Items(); len(items) != 3 || items[0].Type != "url" || items[1].Type != "acme-list" || models.Text(items[1].URLPattern) != "/news/:slug" ||
-		items[2].Type != "url" || models.Text(items[2].URL) != "/news" || items[2].Reference != nil {
-		t.Fatalf("copied items = %+v", items)
-	}
-	var st models.Settings
-	if err := e.gdb.First(&st, 1).Error; err != nil {
-		t.Fatal(err)
-	}
-	if models.Text(st.FrontendURL) != "https://www.example.test/" || st.EnablePriorityTiers || st.Priority(models.PriorityHomepage) != "0.9" || st.Priority(models.PriorityBlog) != "0.6" {
-		t.Fatalf("copied settings = %+v", st)
-	}
-
-	// A second copy without --force is refused.
-	out, buf = testOutput()
-	if _, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{DefinitionsFile: defs}, out); err == nil || !strings.Contains(err.Error(), "--force") {
-		t.Fatalf("second import: %v\n%s", err, buf.String())
-	}
-
-	// The frontend URL replaces the backend URL in loc, hreflang and the
-	// stylesheet href; the list type expands per alternate.
-	rec := e.get(t, "/sitemap.xml")
-	body := rec.Body.String()
-	if rec.Code != http.StatusOK || rec.Header().Get("Cache-Control") != "no-cache, private" {
-		t.Fatalf("GET /sitemap.xml: %d %v", rec.Code, rec.Header())
-	}
-	for _, want := range []string{
-		`href="https://www.example.test/sitemap.xsl"`,
-		`<loc>https://www.example.test/</loc>`,
-		`<loc>https://www.example.test/news</loc>`,
-		`<loc>https://www.example.test/news/a</loc>`,
-		`<loc>https://www.example.test/en/news/a</loc>`,
-		`hreflang="en" href="https://www.example.test/en/news/a"/>`,
-	} {
-		if !strings.Contains(body, want) {
-			t.Errorf("sitemap lacks %s:\n%s", want, body)
-		}
-	}
-	if strings.Contains(body, "api.example.test") {
-		t.Errorf("the backend URL is left in the sitemap:\n%s", body)
-	}
-
-	// /sitemap.xsl answers the PHP file.
-	rec = e.get(t, "/sitemap.xsl")
-	want, err := os.ReadFile(filepath.Join("assets", "sitemap.xsl"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "text/xsl; charset=UTF-8" || !bytes.Equal(rec.Body.Bytes(), want) {
-		t.Fatalf("GET /sitemap.xsl: %d %q (%d bytes)", rec.Code, rec.Header().Get("Content-Type"), rec.Body.Len())
-	}
-
-	// A BeforeAddItem listener that sets Skip drops its URL.
-	e.app.Events.Listen("acme.skip", func(_ context.Context, ev *classes.BeforeAddItem) error {
-		if ev.Info.URL == "/en/news/a" {
-			ev.Skip = true
-		}
-		return nil
-	})
-	body = e.get(t, "/sitemap.xml").Body.String()
-	if strings.Contains(body, "<loc>https://www.example.test/en/news/a</loc>") || !strings.Contains(body, "<loc>https://www.example.test/news/a</loc>") {
-		t.Fatalf("the skipped URL is still written:\n%s", body)
-	}
-
-	// sitemap:import replaces the url items and keeps the dynamic ones first.
-	manifest := writeFile(t, dir, "urls.json", `[{"url":"/x","changefreq":"daily"},{"url":"/y","priority":"0.9"}]`)
-	out, buf = testOutput()
-	if err := console.ImportFile(ctx, e.gdb, manifest, "", false, out); err != nil {
-		t.Fatalf("sitemap:import: %v", err)
-	}
-	if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil {
-		t.Fatal(err)
-	}
-	items := def.Items()
-	if len(items) != 3 || items[0].Type != "acme-list" || models.Text(items[1].URL) != "/x" || models.Text(items[1].Priority) != "0.5" || models.Text(items[2].Changefreq) != "monthly" {
-		t.Fatalf("imported items = %+v", items)
-	}
-	if !strings.Contains(buf.String(), "Imported 2 URLs into sitemap definition for theme 'default'.") || !strings.Contains(buf.String(), "Kept 1 existing non-URL (dynamic) items.") {
-		t.Fatalf("sitemap:import output:\n%s", buf.String())
-	}
-	bad := writeFile(t, dir, "bad.json", `[{"url":"/x"},{"changefreq":"daily"}]`)
-	if err := console.ImportFile(ctx, e.gdb, bad, "", true, out); err == nil || err.Error() != "Entry 1 is missing a 'url' field" {
-		t.Fatalf("missing url: %v", err)
-	}
-
-	// T-17-15: a frontend URL written straight into the row (one stored
-	// before the save check existed) is not applied; the sitemap keeps the
-	// backend URL and none of the value reaches it.
-	for _, value := range []string{"javascript:alert(document.domain)//", `https://x"/><y a="`} {
-		if err := e.gdb.Model(&models.Settings{}).Where("id = ?", 1).Update("frontend_url", value).Error; err != nil {
-			t.Fatal(err)
-		}
-		rec := e.get(t, "/sitemap.xml")
-		body := rec.Body.String()
-		if rec.Code != http.StatusOK {
-			t.Fatalf("GET /sitemap.xml with frontend_url %q: %d %s", value, rec.Code, body)
-		}
-		if lower := strings.ToLower(body); strings.Contains(lower, "javascript") || strings.Contains(lower, "<y a=") {
-			t.Errorf("frontend_url %q reached the sitemap:\n%s", value, body)
-		}
-		if !strings.Contains(body, `href="https://api.example.test/sitemap.xsl"`) || !strings.Contains(body, "<loc>https://api.example.test/x</loc>") {
-			t.Errorf("frontend_url %q: the sitemap is not on the backend URL:\n%s", value, body)
-		}
-		if strings.Contains(body, "www.example.test") {
-			t.Errorf("frontend_url %q: the earlier frontend URL is left in the sitemap:\n%s", value, body)
-		}
-	}
-}
-
 var adminSeq atomic.Uint64
 
 // call sends an admin API request with an optional bearer token and JSON

updates/202610280001_create_golem15_sitemap_urls.go

@@ -0,0 +1,52 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// createURLs creates golem15_sitemap_urls, the slot table the sitemap files
+// are served from (D-30): one row per address (content type, record and
+// locale; an additional address has type_key extra). grp is the file group,
+// file_no the file within it (NULL until the address is numbered), so a
+// rebuild keeps every listed address in its file. An address left out keeps
+// its row with excluded_reason set and leaves a hole in its file.
+var createURLs = []*gormigrate.Migration{
+	{
+		ID: "202610280001_create_golem15_sitemap_urls",
+		Migrate: func(tx *gorm.DB) error {
+			for _, stmt := range []string{
+				`CREATE TABLE golem15_sitemap_urls (
+	id BIGSERIAL PRIMARY KEY,
+	grp TEXT NOT NULL,
+	type_key TEXT NOT NULL,
+	record_id TEXT NOT NULL,
+	locale TEXT NOT NULL DEFAULT '',
+	url TEXT NOT NULL,
+	title TEXT NOT NULL DEFAULT '',
+	file_no INTEGER NULL,
+	first_seen TIMESTAMPTZ NOT NULL,
+	lastmod TIMESTAMPTZ NULL,
+	alternates JSONB NOT NULL DEFAULT '[]',
+	images JSONB NOT NULL DEFAULT '[]',
+	excluded_reason TEXT NOT NULL DEFAULT '',
+	updated_at TIMESTAMPTZ NOT NULL
+)`,
+				`CREATE UNIQUE INDEX golem15_sitemap_urls_record_unique ON golem15_sitemap_urls (type_key, record_id, locale)`,
+				`CREATE INDEX golem15_sitemap_urls_file_index ON golem15_sitemap_urls (grp, file_no)`,
+			} {
+				if err := tx.Exec(stmt).Error; err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return tx.Exec(`DROP TABLE IF EXISTS golem15_sitemap_urls`).Error
+		},
+	},
+}
+
+func init() {
+	Register(createURLs...)
+}

updates/202610280002_update_golem15_sitemap_settings.go

@@ -0,0 +1,51 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// settingsColumns are the columns the per-type sitemap format adds to the
+// settings singleton: the site address, the file size, the default locale,
+// the additional addresses, the debounce time of the next rebuild
+// (rebuild_pending_until), the file size the files are numbered for
+// (numbered_size; a different file_size renumbers them), when the last
+// rebuild finished (built_at) and when one was last requested from the
+// admin (rebuild_requested_at). The Phase 17 columns stay in the table,
+// unread.
+var settingsColumns = []string{
+	"site_url TEXT NULL",
+	"file_size INTEGER NOT NULL DEFAULT 1000",
+	"default_locale TEXT NULL",
+	"extra_urls JSONB NULL DEFAULT '[]'",
+	"rebuild_pending_until TIMESTAMPTZ NULL",
+	"numbered_size INTEGER NULL",
+	"built_at TIMESTAMPTZ NULL",
+	"rebuild_requested_at TIMESTAMPTZ NULL",
+}
+
+var updateSettings = []*gormigrate.Migration{
+	{
+		ID: "202610280002_update_golem15_sitemap_settings",
+		Migrate: func(tx *gorm.DB) error {
+			for _, column := range settingsColumns {
+				if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings ADD COLUMN ` + column).Error; err != nil {
+					return err
+				}
+			}
+			return tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			for _, name := range []string{"site_url", "file_size", "default_locale", "extra_urls", "rebuild_pending_until", "numbered_size", "built_at", "rebuild_requested_at"} {
+				if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings DROP COLUMN IF EXISTS ` + name).Error; err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	},
+}
+
+func init() {
+	Register(updateSettings...)
+}