From 3359f1a1965504c6badcefc5745f65a544d91e30 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Wed, 4 Nov 2026 10:58:05 +0100 Subject: [PATCH] feat: sitemap index and per-type files from a slot table (tracer) - contract package: Filter, Listing and their types (D-95) - golem15_sitemap_urls slot table and the new settings columns - Rebuild over signpost Records, stable slot assignment, rebuild job - GET /sitemap.xml index and GET /_sitemap/files/{file} from the table --- classes/rebuild.go | 406 ++++++++++++++++++ classes/settings.go | 61 +++ classes/slots.go | 95 ++++ classes/tiers_test.go | 21 - classes/writer.go | 135 ++++++ content_fixture_test.go | 157 +++++++ contract/contract.go | 148 +++++++ jobs.go | 26 ++ models/settings.go | 40 +- models/url.go | 76 ++++ plugin.go | 49 +-- routes.go | 327 +++++--------- routes_test.go | 84 ++-- sitemap_index_test.go | 133 ++++++ sitemap_smoke_test.go | 280 +----------- ...02610280001_create_golem15_sitemap_urls.go | 52 +++ ...0280002_update_golem15_sitemap_settings.go | 51 +++ 17 files changed, 1564 insertions(+), 577 deletions(-) create mode 100644 classes/rebuild.go create mode 100644 classes/settings.go create mode 100644 classes/slots.go create mode 100644 classes/writer.go create mode 100644 content_fixture_test.go create mode 100644 contract/contract.go create mode 100644 jobs.go create mode 100644 models/url.go create mode 100644 sitemap_index_test.go create mode 100644 updates/202610280001_create_golem15_sitemap_urls.go create mode 100644 updates/202610280002_update_golem15_sitemap_settings.go diff --git a/classes/rebuild.go b/classes/rebuild.go new file mode 100644 index 0000000..c9adeda --- /dev/null +++ b/classes/rebuild.go @@ -0,0 +1,406 @@ +package classes + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/golem15/summercms/modules/signpost" + "gorm.io/gorm" +) + +// RebuildKind is the River kind of the rebuild job. +const RebuildKind = "golem15.sitemap.rebuild" + +// RebuildArgs are the arguments of the rebuild job: it has none, a rebuild +// always covers every content type and the additional addresses. +type RebuildArgs struct{} + +// Kind is RebuildKind. +func (RebuildArgs) Kind() string { return RebuildKind } + +// recordsPage is the page size the rebuild asks the content types for. +const recordsPage = 500 + +// ErrNoDatabase is returned when the application publishes no database. +var ErrNoDatabase = errors.New("golem15.sitemap: database is not configured") + +// RebuildResult is what one rebuild changed, as absolute URLs: Added are +// newly listed addresses, Changed listed addresses whose URL changed and +// Removed addresses that left the files. Listed counts the listed +// addresses and Files the files they fill. +type RebuildResult struct { + Added []string + Changed []string + Removed []string + Listed int + Files int +} + +// DefaultSlug is the file name part of a content type's group: the last +// segment of its key in plural form (journal-post: posts, journal-category: +// categories, seo-page: pages). +func DefaultSlug(key string) string { + key = strings.ToLower(strings.TrimSpace(key)) + if i := strings.LastIndexAny(key, "-_."); i >= 0 { + key = key[i+1:] + } + var b strings.Builder + for _, r := range key { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } + } + s := b.String() + switch { + case s == "": + return "pages" + case strings.HasSuffix(s, "y") && len(s) > 1 && !strings.ContainsRune("aeiou", rune(s[len(s)-2])): + return s[:len(s)-1] + "ies" + case strings.HasSuffix(s, "s"): + return s + } + return s + "s" +} + +// ExtraRecordID is the record_id of an additional address: the hex SHA-256 +// of the address. +func ExtraRecordID(u string) string { + sum := sha256.Sum256([]byte(u)) + return hex.EncodeToString(sum[:]) +} + +// slotKey identifies a row of the slot table. +type slotKey struct { + typeKey, recordID, locale string +} + +// wanted is one address a rebuild found: its group, its row key, its path +// and title, its first lastmod (for a new row) and the reason it is left +// out ("" when listed). An additional address carries its own lastmod, +// which the editor sets, so it is written on every rebuild. +type wanted struct { + key slotKey + grp string + url string + title string + firstMod *time.Time + reason string + extra bool + extraMod *time.Time + alternates []models.Alternate +} + +// collected is what a rebuild read from the content types and settings. +type collected struct { + rows []wanted + unpublished map[slotKey]bool +} + +// Rebuild reads every content type's published records (signpost Records, +// D-93) and the additional addresses, writes them into the slot table and +// numbers the files, in one transaction that holds the settings row, so +// two rebuilds never interleave. A record that is gone or unpublished +// leaves its rows in the table with a reason, as a hole; only a changed +// file size renumbers the files (D-30). +func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { + var res RebuildResult + if app == nil { + return res, ErrNoDatabase + } + db, ok := app.Lookup[*gorm.DB]() + if !ok || db == nil { + return res, ErrNoDatabase + } + db = db.WithContext(ctx) + settings, err := LoadSettings(db) + if err != nil { + return res, err + } + reg, _ := app.Lookup[*signpost.Registry]() + found, err := collect(ctx, reg, settings) + if err != nil { + return res, err + } + site := SiteURL(app, settings) + err = db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil { + return err + } + locked, err := LoadSettings(tx) + if err != nil { + return err + } + res, err = apply(tx, found, locked, site) + return err + }) + return res, err +} + +// collect walks the content types page by page and adds the additional +// addresses. +func collect(ctx context.Context, reg *signpost.Registry, settings models.Settings) (collected, error) { + out := collected{unpublished: map[slotKey]bool{}} + seen := map[slotKey]bool{} + add := func(w wanted) { + if seen[w.key] { + return + } + seen[w.key] = true + out.rows = append(out.rows, w) + } + unprefixed := models.Text(settings.DefaultLocale) + if reg != nil { + for _, typ := range reg.ContentTypes() { + slug := DefaultSlug(typ.Key) + after := "" + for { + page, err := reg.Records(ctx, typ.Key, after, recordsPage, unprefixed) + if err != nil { + return out, fmt.Errorf("golem15.sitemap: records of %s: %w", typ.Key, err) + } + for _, rec := range page.Records { + if !rec.Published { + out.unpublished[slotKey{typ.Key, rec.ID, ""}] = true + continue + } + for _, l := range rec.Locales { + if strings.TrimSpace(l.URL) == "" { + continue + } + add(wanted{ + key: slotKey{typ.Key, rec.ID, l.Locale}, + grp: slug, + url: l.URL, + title: l.Title, + firstMod: rec.LastMod, + }) + } + } + if page.Next == "" || page.Next == after { + break + } + after = page.Next + } + } + } + for _, e := range settings.Extra() { + u := strings.TrimSpace(e.URL) + if !models.AllowedAddress(u) { + continue + } + mod := ParseDate(e.LastMod) + add(wanted{ + key: slotKey{models.ExtraTypeKey, ExtraRecordID(u), ""}, + grp: models.ExtraGroup, + url: u, + extra: true, + extraMod: mod, + firstMod: mod, + }) + } + return out, nil +} + +// ParseDate reads a date as the admin's date picker or an import sends it +// (RFC 3339, "2006-01-02 15:04:05" or "2006-01-02", UTC); nil when empty or +// unreadable. +func ParseDate(s string) *time.Time { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, s); err == nil { + t = t.UTC() + return &t + } + } + return nil +} + +// apply writes the found addresses into the slot table and numbers the +// files, inside the rebuild's transaction. +func apply(tx *gorm.DB, found collected, settings models.Settings, site string) (RebuildResult, error) { + var res RebuildResult + var existing []models.URL + if err := tx.Order("id ASC").Find(&existing).Error; err != nil { + return res, err + } + now := time.Now().UTC().Truncate(time.Microsecond) + byKey := make(map[slotKey]*models.URL, len(existing)) + for i := range existing { + r := &existing[i] + byKey[slotKey{r.TypeKey, r.RecordID, r.Locale}] = r + } + dirty := map[uint64]bool{} + var created []*models.URL + matched := map[slotKey]bool{} + for _, w := range found.rows { + matched[w.key] = true + r, ok := byKey[w.key] + if !ok { + mod := w.firstMod + if mod == nil && !w.extra { + mod = &now + } + row := &models.URL{ + Grp: w.grp, + TypeKey: w.key.typeKey, + RecordID: w.key.recordID, + Locale: w.key.locale, + URL: w.url, + Title: w.title, + FirstSeen: now, + LastMod: utcPtr(mod), + Alternates: lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true}, + Images: lagoon.Jsonable[[]string]{Data: []string{}, Valid: true}, + ExcludedReason: w.reason, + UpdatedAt: now, + } + created = append(created, row) + if w.reason == "" { + res.Added = append(res.Added, Absolute(site, w.url)) + } + continue + } + wasListed := r.ExcludedReason == "" + before := *r + if r.Grp != w.grp { + r.Grp = w.grp + r.FileNo = nil + } + r.URL = w.url + r.Title = w.title + r.ExcludedReason = w.reason + r.Alternates = lagoon.Jsonable[[]models.Alternate]{Data: nonNil(w.alternates), Valid: true} + if w.extra { + r.LastMod = utcPtr(w.extraMod) + } + listed := w.reason == "" + switch { + case listed && !wasListed: + res.Added = append(res.Added, Absolute(site, r.URL)) + case !listed && wasListed: + res.Removed = append(res.Removed, Absolute(site, before.URL)) + case listed && before.URL != r.URL: + res.Changed = append(res.Changed, Absolute(site, r.URL)) + } + if rowChanged(before, *r) { + dirty[r.ID] = true + } + } + for i := range existing { + r := &existing[i] + key := slotKey{r.TypeKey, r.RecordID, r.Locale} + if matched[key] { + continue + } + reason := models.ReasonDeleted + if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] { + reason = models.ReasonUnpublished + } + if r.ExcludedReason == reason { + continue + } + if r.ExcludedReason == "" { + res.Removed = append(res.Removed, Absolute(site, r.URL)) + } + r.ExcludedReason = reason + dirty[r.ID] = true + } + if len(created) > 0 { + if err := tx.CreateInBatches(created, 500).Error; err != nil { + return res, err + } + } + size := settings.EffectiveFileSize() + renumber := settings.NumberedSize == nil || *settings.NumberedSize != size + slots := make([]*Slot, 0, len(existing)+len(created)) + rowsByID := make(map[uint64]*models.URL, len(existing)+len(created)) + for i := range existing { + r := &existing[i] + rowsByID[r.ID] = r + slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""}) + } + for _, r := range created { + rowsByID[r.ID] = r + slots = append(slots, &Slot{ID: r.ID, Grp: r.Grp, FileNo: r.FileNo, FirstSeen: r.FirstSeen, Listed: r.ExcludedReason == ""}) + } + AssignSlots(slots, size, renumber) + files := map[string]bool{} + for _, s := range slots { + r := rowsByID[s.ID] + if !sameInt(r.FileNo, s.FileNo) { + r.FileNo = s.FileNo + dirty[r.ID] = true + } + if s.Listed { + res.Listed++ + if s.FileNo != nil { + files[FileName(s.Grp, *s.FileNo)] = true + } + } + } + res.Files = len(files) + for id := range dirty { + r := rowsByID[id] + r.UpdatedAt = now + if err := tx.Model(&models.URL{}).Where("id = ?", id).Updates(map[string]any{ + "grp": r.Grp, + "url": r.URL, + "title": r.Title, + "file_no": r.FileNo, + "lastmod": r.LastMod, + "alternates": r.Alternates, + "excluded_reason": r.ExcludedReason, + "updated_at": r.UpdatedAt, + }).Error; err != nil { + return res, err + } + } + if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil { + return res, err + } + return res, nil +} + +// rowChanged reports whether a rebuild changed a stored row. +func rowChanged(a, b models.URL) bool { + if a.Grp != b.Grp || a.URL != b.URL || a.Title != b.Title || a.ExcludedReason != b.ExcludedReason || !sameInt(a.FileNo, b.FileNo) || !sameTime(a.LastMod, b.LastMod) { + return true + } + av, _ := a.Alternates.Value() + bv, _ := b.Alternates.Value() + return av != bv +} + +func sameInt(a, b *int) bool { + return (a == nil && b == nil) || (a != nil && b != nil && *a == *b) +} + +func sameTime(a, b *time.Time) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Equal(*b)) +} + +func utcPtr(t *time.Time) *time.Time { + if t == nil { + return nil + } + u := t.UTC().Truncate(time.Microsecond) + return &u +} + +func nonNil[T any](s []T) []T { + if s == nil { + return []T{} + } + return s +} diff --git a/classes/settings.go b/classes/settings.go new file mode 100644 index 0000000..3b23aa6 --- /dev/null +++ b/classes/settings.go @@ -0,0 +1,61 @@ +package classes + +import ( + "strings" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "gorm.io/gorm" +) + +// SiteURLConfigKey is the config key of the site address the sitemaps use +// when the setting is empty. +const SiteURLConfigKey = "golem15.sitemap.site_url" + +// LoadSettings reads the settings row; without one the defaults apply. +func LoadSettings(db *gorm.DB) (models.Settings, error) { + var rows []models.Settings + if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { + return models.Settings{}, err + } + if len(rows) == 0 { + return models.DefaultSettings(), nil + } + return rows[0], nil +} + +// SiteURL is the absolute site address every starts with, without a +// trailing slash: the site_url setting, else the config key +// golem15.sitemap.site_url, else app.url (CONTEXT 2026-10-28). A value that +// is not an absolute http(s) URL (models.AllowedFrontendURL) is skipped. +// The request's Host header is never used; "" means no site address is +// configured. +func SiteURL(app *backpack.App, s models.Settings) string { + candidates := []string{models.Text(s.SiteURL)} + if app != nil && app.Config != nil { + candidates = append(candidates, app.Config.String(SiteURLConfigKey), app.Config.String("app.url")) + } + for _, c := range candidates { + c = strings.TrimRight(strings.TrimSpace(c), "/") + if c != "" && models.AllowedFrontendURL(c) { + return c + } + } + return "" +} + +// Absolute turns a stored address into the absolute URL of a : a path +// starting with a single / is put on site; an absolute http(s) URL is kept. +// Anything else, or a path without a site address, is "". +func Absolute(site, u string) string { + switch { + case strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//"): + if site == "" { + return "" + } + return site + u + case models.AllowedFrontendURL(u): + return u + } + return "" +} diff --git a/classes/slots.go b/classes/slots.go new file mode 100644 index 0000000..dd3d51d --- /dev/null +++ b/classes/slots.go @@ -0,0 +1,95 @@ +package classes + +import ( + "slices" + "time" +) + +// Slot is what file numbering needs of one row of the slot table: its +// group, its file number (nil while unnumbered), when it was first seen and +// whether it is listed. A row left out keeps its number: its file keeps the +// hole instead of the later files shifting (D-30, RESEARCH Pitfall 5). +type Slot struct { + ID uint64 + Grp string + FileNo *int + FirstSeen time.Time + Listed bool +} + +// AssignSlots numbers the rows for files of size addresses and reports +// whether any number changed. Without renumber, a numbered row keeps its +// number and every listed row without one is appended to the last file of +// its group while that file has room (counting the holes of left-out rows), +// then to the next file, oldest first (first seen, then id). With renumber +// (the file size changed), every group's listed rows are numbered again +// from file 1, oldest first, and the left-out rows lose their numbers. +func AssignSlots(rows []*Slot, size int, renumber bool) bool { + if size < 1 { + size = 1 + } + groups := map[string][]*Slot{} + var order []string + for _, r := range rows { + if _, ok := groups[r.Grp]; !ok { + order = append(order, r.Grp) + } + groups[r.Grp] = append(groups[r.Grp], r) + } + changed := false + set := func(r *Slot, n *int) { + if (r.FileNo == nil) != (n == nil) || (r.FileNo != nil && *r.FileNo != *n) { + changed = true + } + r.FileNo = n + } + for _, grp := range order { + members := groups[grp] + slices.SortStableFunc(members, func(a, b *Slot) int { + if c := a.FirstSeen.Compare(b.FirstSeen); c != 0 { + return c + } + switch { + case a.ID < b.ID: + return -1 + case a.ID > b.ID: + return 1 + } + return 0 + }) + if renumber { + i := 0 + for _, r := range members { + if !r.Listed { + set(r, nil) + continue + } + n := i/size + 1 + set(r, &n) + i++ + } + continue + } + counts := map[int]int{} + last := 0 + for _, r := range members { + if r.FileNo == nil { + continue + } + counts[*r.FileNo]++ + last = max(last, *r.FileNo) + } + for _, r := range members { + if r.FileNo != nil || !r.Listed { + continue + } + if last == 0 || counts[last] >= size { + last++ + } + n := last + set(r, &n) + counts[last]++ + } + } + return changed +} diff --git a/classes/tiers_test.go b/classes/tiers_test.go index 05c44d8..ab550bb 100644 --- a/classes/tiers_test.go +++ b/classes/tiers_test.go @@ -1,16 +1,11 @@ package classes_test import ( - "os" - "path/filepath" "strings" "testing" - sitemap "git.golem15.com/golem15/sm-sitemap-plugin" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" - "git.golem15.com/golem15/summercms/modules/backpack" - "git.golem15.com/golem15/summercms/modules/compass" ) // TestTiers pins the priority tiers (D-13): each default rule of the PHP @@ -96,20 +91,4 @@ func TestTiers(t *testing.T) { } } - // A bad host rule fails the plugin's Boot. - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "golem15.yaml"), []byte("sitemap:\n priority_tiers:\n - setting: priority_blog\n pattern: '^/(blog'\n"), 0o644); err != nil { - t.Fatal(err) - } - cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) - if err != nil { - t.Fatal(err) - } - p := &sitemap.Plugin{} - if err := cfg.MergePlugin("golem15.sitemap", p.ConfigFS()); err != nil { - t.Fatal(err) - } - if err := p.Boot(backpack.New(cfg)); err == nil || !strings.Contains(err.Error(), "priority_tiers[0]") { - t.Fatalf("Boot with a bad pattern = %v", err) - } } diff --git a/classes/writer.go b/classes/writer.go new file mode 100644 index 0000000..0862a7d --- /dev/null +++ b/classes/writer.go @@ -0,0 +1,135 @@ +package classes + +import ( + "bufio" + "encoding/xml" + "io" + "regexp" + "strconv" + "strings" + "time" +) + +// The XML namespaces of the sitemap files. +const ( + NSSitemap = "http://www.sitemaps.org/schemas/sitemap/0.9" + NSXHTML = "http://www.w3.org/1999/xhtml" +) + +// IndexFile is one of the index: the file's absolute URL and the +// newest lastmod of its addresses (nil when none has one). +type IndexFile struct { + Loc string + LastMod *time.Time +} + +// FileAlternate is one xhtml:link alternate of a , with an absolute +// href. +type FileAlternate struct { + Hreflang string + Href string +} + +// FileEntry is one of a file: the absolute address, its lastmod and +// its hreflang alternates. +type FileEntry struct { + Loc string + LastMod *time.Time + Alternates []FileAlternate +} + +// fileNamePattern is the name of a per-group file. +var fileNamePattern = regexp.MustCompile(`^sitemap-([a-z0-9][a-z0-9-]*)-([1-9][0-9]{0,8})\.xml$`) + +// FileName is the name of file n of group grp: sitemap--.xml. +func FileName(grp string, n int) string { + return "sitemap-" + grp + "-" + strconv.Itoa(n) + ".xml" +} + +// ParseFileName splits a file name made by FileName; ok is false for any +// other name. +func ParseFileName(name string) (grp string, n int, ok bool) { + m := fileNamePattern.FindStringSubmatch(name) + if m == nil { + return "", 0, false + } + n, err := strconv.Atoi(m[2]) + if err != nil || n < 1 { + return "", 0, false + } + return m[1], n, true +} + +// lastmod formats a lastmod in RFC 3339, UTC. +func lastmod(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +// escape writes s with encoding/xml's escaping, which covers &, <, >, both +// quotes and control whitespace, so it is safe in text and in attribute +// values. +func escape(w *bufio.Writer, s string) { + _ = xml.EscapeText(w, []byte(s)) +} + +// WriteIndex writes the sitemap index: one per file with its +// and, when known, its . +func WriteIndex(w io.Writer, files []IndexFile) error { + bw := bufio.NewWriter(w) + bw.WriteString(xml.Header) + bw.WriteString(``) + for _, f := range files { + bw.WriteString("\n") + escape(bw, f.Loc) + bw.WriteString("") + if f.LastMod != nil { + bw.WriteString("" + lastmod(*f.LastMod) + "") + } + bw.WriteString("") + } + bw.WriteString("\n\n") + return bw.Flush() +} + +// WriteURLSet writes one sitemap file: a per entry with its , its +// when known and its xhtml:link alternates. The xhtml namespace +// is declared only when an entry has alternates. No and no +// are written (D-30). +func WriteURLSet(w io.Writer, entries []FileEntry) error { + bw := bufio.NewWriter(w) + bw.WriteString(xml.Header) + bw.WriteString(` 0 { + bw.WriteString(` xmlns:xhtml="` + NSXHTML + `"`) + break + } + } + bw.WriteString(">") + for _, e := range entries { + bw.WriteString("\n") + escape(bw, e.Loc) + bw.WriteString("") + if e.LastMod != nil { + bw.WriteString("" + lastmod(*e.LastMod) + "") + } + for _, a := range e.Alternates { + bw.WriteString(``) + } + bw.WriteString("") + } + bw.WriteString("\n\n") + return bw.Flush() +} + +// validSlug reports whether s may name a file group. +func validSlug(s string) bool { + return s != "" && len(s) <= 64 && fileNamePattern.MatchString(FileName(s, 1)) && !strings.HasSuffix(s, "-") +} + +// ValidSlug reports whether s may name a file group: lowercase letters, +// digits and hyphens, starting with a letter or digit, at most 64 +// characters. +func ValidSlug(s string) bool { return validSlug(s) } diff --git a/content_fixture_test.go b/content_fixture_test.go new file mode 100644 index 0000000..3f7b934 --- /dev/null +++ b/content_fixture_test.go @@ -0,0 +1,157 @@ +package sitemap_test + +import ( + "context" + "slices" + "strconv" + "sync" + "testing" + "time" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/pact" + "git.golem15.com/golem15/summercms/modules/party" +) + +// contentPluginID is a fake content plugin with two content types whose +// records live in memory (contentStore), so a test sets exactly what the +// sitemap reads through signpost Records. It knows nothing about the +// sitemap plugin. +const contentPluginID = "acme.content" + +// The fake content types: pages (/pages/, group pages) and offers +// (/offers/, group offers). +const ( + pageType = "acme-page" + pageModel = "acme_pages" + offerType = "acme-offer" + offerModel = "acme_offers" +) + +// fakeRecord is one record of a fake content type. +type fakeRecord struct { + ID int + Published bool + LastMod time.Time + Locales []pact.ContentLocale +} + +// store holds the fake records per content type key. +type store struct { + mu sync.Mutex + records map[string][]fakeRecord +} + +var contentStore = &store{records: map[string][]fakeRecord{}} + +// reset empties the store and clears it again when the test ends. +func (s *store) reset(t *testing.T) { + t.Helper() + s.mu.Lock() + s.records = map[string][]fakeRecord{} + s.mu.Unlock() + t.Cleanup(func() { + s.mu.Lock() + s.records = map[string][]fakeRecord{} + s.mu.Unlock() + }) +} + +// put adds or replaces a record of typ. +func (s *store) put(typ string, rec fakeRecord) { + s.mu.Lock() + defer s.mu.Unlock() + list := s.records[typ] + for i := range list { + if list[i].ID == rec.ID { + list[i] = rec + return + } + } + s.records[typ] = append(list, rec) +} + +// remove deletes a record of typ. +func (s *store) remove(typ string, id int) { + s.mu.Lock() + defer s.mu.Unlock() + s.records[typ] = slices.DeleteFunc(s.records[typ], func(r fakeRecord) bool { return r.ID == id }) +} + +// page answers one ContentList page of typ, by ascending ID. +func (s *store) page(typ string, in pact.ContentList) pact.ContentPage { + s.mu.Lock() + defer s.mu.Unlock() + list := slices.Clone(s.records[typ]) + slices.SortFunc(list, func(a, b fakeRecord) int { return a.ID - b.ID }) + after, _ := strconv.Atoi(in.After) + limit := in.Limit + if limit <= 0 { + limit = 100 + } + out := pact.ContentPage{Records: []pact.ContentRecord{}} + for _, r := range list { + if r.ID <= after { + continue + } + if len(out.Records) == limit { + out.Next = strconv.Itoa(after) + break + } + mod := r.LastMod + out.Records = append(out.Records, pact.ContentRecord{ + ID: strconv.Itoa(r.ID), + Published: r.Published, + LastMod: &mod, + Locales: slices.Clone(r.Locales), + }) + after = r.ID + } + return out +} + +// contentPlugin is acme.content. +type contentPlugin struct{} + +func (contentPlugin) ID() string { return contentPluginID } +func (contentPlugin) Requires() []string { return nil } +func (contentPlugin) Register(*backpack.App) error { return nil } +func (contentPlugin) Boot(*backpack.App) error { return nil } + +func (contentPlugin) MenuItemTypes() []pact.MenuItemType { + typ := func(key, label, model, pattern string) pact.MenuItemType { + return pact.MenuItemType{ + Key: key, + Label: label, + DefaultPattern: pattern, + Placeholders: []string{"slug"}, + Resolve: func(context.Context, pact.MenuItemResolve) (*pact.ResolvedMenuItem, error) { + return nil, nil + }, + Content: &pact.ContentDescriptor{ + Model: model, + Records: func(_ context.Context, in pact.ContentList) (pact.ContentPage, error) { + return contentStore.page(key, in), nil + }, + Describe: func(context.Context, pact.ContentDescribe) (*pact.ContentRecord, error) { + return nil, nil + }, + }, + } + } + return []pact.MenuItemType{ + typ(pageType, "Pages", pageModel, "/pages/:slug"), + typ(offerType, "Offers", offerModel, "/offers/:slug"), + } +} + +var _ pact.HasMenuItemTypes = contentPlugin{} + +func init() { + party.Register(contentPlugin{}) +} + +// page1 is a published record in one locale at /pages/. +func page1(id int, slug string, mod time.Time) fakeRecord { + return fakeRecord{ID: id, Published: true, LastMod: mod, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/" + slug, Title: "Page " + slug}}} +} diff --git a/contract/contract.go b/contract/contract.go new file mode 100644 index 0000000..198690f --- /dev/null +++ b/contract/contract.go @@ -0,0 +1,148 @@ +// Package contract is the published seam of the golem15.sitemap plugin: +// the Filter another plugin implements to keep addresses out of the +// sitemaps or move them to another file group, and the Listing other +// plugins read to show the sitemap's addresses (an HTML sitemap, IndexNow). +// +// It is a package of its own so a plugin can import it without importing +// the sitemap plugin's root package, whose init registers the plugin. The +// sitemap plugin depends on no implementation: it looks a Filter up with +// +// f, ok := app.Lookup[contract.Filter]() +// +// and includes every published address when none is published. A plugin +// publishes one in its Register: +// +// app.Publish[contract.Filter](myFilter{}) +package contract + +import ( + "context" + "time" +) + +// The reasons a Decision may give for leaving an address out. The sitemap +// counts excluded addresses by reason; another reason is counted under its +// own name. +const ( + // ReasonNoindex is an address whose page is set to noindex. + ReasonNoindex = "noindex" + // ReasonCanonical is an address whose canonical URL points elsewhere. + ReasonCanonical = "canonical" + // ReasonRedirect is an address that redirects. + ReasonRedirect = "redirect" + // ReasonType is an address of a content type switched off in the + // sitemap settings; the sitemap sets it itself, a Filter need not. + ReasonType = "type" +) + +// Candidate is one address the sitemap would list: TypeKey is the content +// type key (a signpost menu item type, or "extra" for an additional address +// of the settings), RecordID the record's ID within it (for an additional +// address the hex SHA-256 of its path), Locale the locale ("" when the +// address has none) and URL its public path or absolute URL. +type Candidate struct { + TypeKey string + RecordID string + Locale string + URL string +} + +// Decision is a Filter's answer for one Candidate. Exclude leaves the +// address out of the sitemap files, counted under Reason. Group, when not +// empty, puts the address in another file group: the files are then named +// sitemap--.xml, so a Group must use lowercase letters, digits and +// hyphens (another value is ignored). LastMod, when set, is the first +// lastmod of an address the sitemap has not listed before; it never moves +// the lastmod of a listed address. +type Decision struct { + Exclude bool + Reason string + Group string + LastMod *time.Time +} + +// Group names a file group a Filter's decisions use: Key is the group as +// in Decision.Group, Titles its heading per locale for an HTML sitemap. +type Group struct { + Key string + Titles map[string]string +} + +// Filter decides which addresses the sitemap lists. Decide receives one +// batch (a page of records of one content type, or the additional +// addresses) and answers one Decision per Candidate in the same order; an +// answer of another length or an error fails the rebuild, which keeps the +// files it served before. Groups lists the groups the Filter's decisions +// may name, with their titles. +type Filter interface { + Decide(ctx context.Context, batch []Candidate) ([]Decision, error) + Groups(ctx context.Context) ([]Group, error) +} + +// Listing is what the sitemap publishes for other plugins: its settings and +// the listed addresses of each group. The sitemap plugin publishes it with +// app.Publish[contract.Listing] in its Register. +type Listing interface { + Settings(ctx context.Context) (ListingSettings, error) + Entries(ctx context.Context, q EntriesQuery) (EntriesPage, error) +} + +// ListingSettings are the sitemap's settings as other plugins need them. +// SiteURL is the absolute site address every address starts with ("" when +// none is configured), IndexURL the absolute address of the sitemap index, +// DefaultLocale the site's default locale ("" when not set) and Groups the +// file groups with listed addresses. HTMLEnabled, HTMLPaths (locale to +// path), HTMLPageSize and KeyFile (the IndexNow key file name) describe the +// HTML sitemap and IndexNow; they stay zero until those settings exist. +type ListingSettings struct { + SiteURL string + IndexURL string + DefaultLocale string + Groups []GroupInfo + HTMLEnabled bool + HTMLPaths map[string]string + HTMLPageSize int + KeyFile string +} + +// GroupInfo is one file group: Key is the group (as Entries takes it), Slug +// the name part of its files (sitemap--.xml; today the same as +// Key), Titles its heading per locale and Count its listed addresses. +type GroupInfo struct { + Key string + Slug string + Titles map[string]string + Count int +} + +// EntriesQuery asks for one page of a group's listed addresses. Locale, +// when set, keeps the addresses of that locale and those without one. +// After is the Next of the previous page ("" for the first) and Limit the +// page size (at most 1000; 0 means 100). +type EntriesQuery struct { + Group string + Locale string + After string + Limit int +} + +// Entry is one listed address. URL is absolute (on ListingSettings.SiteURL) +// and LastMod its last real change, nil when unknown. +type Entry struct { + Group string + TypeKey string + RecordID string + Locale string + URL string + Title string + LastMod *time.Time +} + +// EntriesPage is one page of Entries in a stable order: Next is the After +// of the following page ("" on the last page) and Total the number of +// addresses the query matches across all pages. +type EntriesPage struct { + Entries []Entry + Next string + Total int +} diff --git a/jobs.go b/jobs.go new file mode 100644 index 0000000..20aebbb --- /dev/null +++ b/jobs.go @@ -0,0 +1,26 @@ +package sitemap + +import ( + "context" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/summercms/modules/conga" + "git.golem15.com/golem15/summercms/modules/pact" +) + +// Jobs registers the rebuild job (classes.RebuildKind) on conga. +func (p *Plugin) Jobs() []pact.Job { + return []pact.Job{ + conga.Job(p.rebuild, conga.MaxAttempts(3)), + } +} + +// rebuild runs one rebuild of the slot table. +func (p *Plugin) rebuild(ctx context.Context, _ classes.RebuildArgs) error { + res, err := classes.Rebuild(ctx, p.app) + if err != nil { + return err + } + pluginLog.Info("golem15.sitemap: rebuilt", "listed", res.Listed, "files", res.Files, "added", len(res.Added), "changed", len(res.Changed), "removed", len(res.Removed)) + return nil +} diff --git a/models/settings.go b/models/settings.go index 207335b..83666a2 100644 --- a/models/settings.go +++ b/models/settings.go @@ -3,10 +3,12 @@ package models import ( "net/url" "strings" + "time" "unicode" "unicode/utf8" "git.golem15.com/golem15/summercms/modules/cabana" + "git.golem15.com/golem15/summercms/modules/lagoon" "gorm.io/gorm" ) @@ -53,8 +55,44 @@ type Settings struct { PriorityBlog *string `gorm:"column:priority_blog"` PriorityCategories *string `gorm:"column:priority_categories"` PriorityLegal *string `gorm:"column:priority_legal"` + + SiteURL *string `gorm:"column:site_url"` + FileSize int `gorm:"column:file_size"` + DefaultLocale *string `gorm:"column:default_locale"` + ExtraURLs lagoon.Jsonable[[]ExtraURL] `gorm:"column:extra_urls"` + RebuildPendingUntil *time.Time `gorm:"column:rebuild_pending_until"` + NumberedSize *int `gorm:"column:numbered_size"` + BuiltAt *time.Time `gorm:"column:built_at"` + RebuildRequestedAt *time.Time `gorm:"column:rebuild_requested_at"` } +// DefaultFileSize is the number of addresses per sitemap file when the +// setting is not stored (D-30); MaxFileSize is the protocol's limit. +const ( + DefaultFileSize = 1000 + MaxFileSize = 50000 +) + +// ExtraURL is one additional address of the settings: URL is a path (or an +// absolute http(s) URL on the site), LastMod its optional last change as +// the admin's date picker sends it. +type ExtraURL struct { + URL string `json:"url"` + LastMod string `json:"lastmod"` +} + +// EffectiveFileSize is the stored file size, or DefaultFileSize when it is +// outside 1..MaxFileSize. +func (s Settings) EffectiveFileSize() int { + if s.FileSize < 1 || s.FileSize > MaxFileSize { + return DefaultFileSize + } + return s.FileSize +} + +// Extra returns the additional addresses. +func (s Settings) Extra() []ExtraURL { return s.ExtraURLs.Get() } + func (Settings) TableName() string { return "golem15_sitemap_settings" } // Fillable lists only the settings columns. @@ -114,7 +152,7 @@ func AllowedFrontendURL(u string) bool { // DefaultSettings is the settings PHP read when nothing was saved: tiers on // with the default priorities. func DefaultSettings() Settings { - return Settings{ID: 1, EnablePriorityTiers: true} + return Settings{ID: 1, EnablePriorityTiers: true, FileSize: DefaultFileSize} } // Priority returns the value of a priority setting: the stored text, or the diff --git a/models/url.go b/models/url.go new file mode 100644 index 0000000..8d020ea --- /dev/null +++ b/models/url.go @@ -0,0 +1,76 @@ +package models + +import ( + "strings" + "time" + "unicode" + "unicode/utf8" + + "git.golem15.com/golem15/summercms/modules/lagoon" +) + +// ExtraTypeKey is the type_key of the settings' additional addresses in +// golem15_sitemap_urls; their record_id is the hex SHA-256 of the address. +const ExtraTypeKey = "extra" + +// ExtraGroup is the file group of the additional addresses. +const ExtraGroup = "pages" + +// The excluded reasons the sitemap sets itself. A Filter's reasons +// (contract.ReasonNoindex and the others) are stored as it gives them. +const ( + // ReasonDeleted is an address whose record or locale is gone. + ReasonDeleted = "deleted" + // ReasonUnpublished is an address whose record is not published. + ReasonUnpublished = "unpublished" +) + +// Alternate is one hreflang alternate of an address: Hreflang the locale +// code or x-default, Href the alternate's path or absolute URL. +type Alternate struct { + Hreflang string `json:"hreflang"` + Href string `json:"href"` +} + +// URL is one row of the slot table golem15_sitemap_urls (D-30): an address +// of a content type's record in one locale, or an additional address. Grp +// and FileNo place it in the file sitemap--.xml; FileNo is +// nil until the address is numbered. ExcludedReason is "" for a listed +// address; a row left out keeps its FileNo, so its file keeps a hole. +type URL struct { + ID uint64 `gorm:"column:id;primaryKey"` + Grp string `gorm:"column:grp"` + TypeKey string `gorm:"column:type_key"` + RecordID string `gorm:"column:record_id"` + Locale string `gorm:"column:locale"` + URL string `gorm:"column:url"` + Title string `gorm:"column:title"` + FileNo *int `gorm:"column:file_no"` + FirstSeen time.Time `gorm:"column:first_seen"` + LastMod *time.Time `gorm:"column:lastmod"` + Alternates lagoon.Jsonable[[]Alternate] `gorm:"column:alternates"` + Images lagoon.Jsonable[[]string] `gorm:"column:images"` + ExcludedReason string `gorm:"column:excluded_reason"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (URL) TableName() string { return "golem15_sitemap_urls" } + +func init() { + Register(URL{}) +} + +// AllowedAddress reports whether u may be a stored address: a path that +// starts with a single / (no whitespace, control characters, quotes, < or +// >), or an absolute http(s) URL that AllowedFrontendURL accepts. +func AllowedAddress(u string) bool { + if u == "" || !utf8.ValidString(u) { + return false + } + if strings.HasPrefix(u, "/") && !strings.HasPrefix(u, "//") { + return !strings.ContainsFunc(u, func(r rune) bool { + return unicode.IsControl(r) || unicode.IsSpace(r) || r == '"' || r == '<' || r == '>' || r == '\\' + }) + } + return AllowedFrontendURL(u) +} diff --git a/plugin.go b/plugin.go index 48eb548..7fb4fd5 100644 --- a/plugin.go +++ b/plugin.go @@ -1,16 +1,15 @@ -// Package sitemap is the golem15.sitemap plugin: it serves /sitemap.xml -// from the application's one sitemap definition, byte for byte the way the -// WinterCMS Golem15.Sitemap plugin did, with items that are explicit URLs -// or menu item types of the framework's registry. +// Package sitemap is the golem15.sitemap plugin: it serves a sitemap index +// at /sitemap.xml whose entries are per-group files of a configurable size +// (sitemap--.xml, D-30), built from the published records of every +// signpost content type and the additional addresses of its settings, with +// stable file numbers and real lastmod. Other plugins narrow it through the +// Filter of the contract package and read it through its Listing. package sitemap import ( "embed" - "fmt" "io/fs" - "time" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/sm-sitemap-plugin/updates" @@ -29,28 +28,22 @@ var ( _ pact.HasModels = (*Plugin)(nil) _ pact.HasRoutes = (*Plugin)(nil) _ pact.HasCommands = (*Plugin)(nil) + _ pact.HasJobs = (*Plugin)(nil) _ surf.BucketProvider = (*Plugin)(nil) ) //go:embed config var configFS embed.FS -// sitemapXSL is the PHP plugin's assets/sitemap.xsl, byte for byte. -// -//go:embed assets/sitemap.xsl -var sitemapXSL []byte - // Plugin is the golem15.sitemap plugin. type Plugin struct { - app *backpack.App - tiers classes.Tiers - cache *sitemapCache + app *backpack.App } func (p *Plugin) ID() string { return "golem15.sitemap" } -// Requires is empty: the sitemap reads content types through the framework's -// menu item type registry, not from a named plugin. +// Requires is empty: the sitemap reads content types through signpost, not +// from a named plugin, and a Filter through the service registry. func (p *Plugin) Requires() []string { return nil } func (p *Plugin) Register(app *backpack.App) error { @@ -58,29 +51,9 @@ func (p *Plugin) Register(app *backpack.App) error { return nil } -// Boot compiles the priority tier rules (classes.TiersConfigKey); a rule -// with an unknown setting or a pattern that does not compile fails the boot. -// It also starts an empty sitemap cache for CacheConfigKey seconds; a -// negative value fails the boot. +// Boot keeps the booted application for the routes, jobs and commands. func (p *Plugin) Boot(app *backpack.App) error { p.app = app - var tiers classes.Tiers - var err error - seconds := 0 - if app != nil { - tiers, err = classes.LoadTiers(app.Config) - if app.Config != nil { - seconds = app.Config.Int(CacheConfigKey) - } - } - if err != nil { - return fmt.Errorf("golem15.sitemap: %w", err) - } - if seconds < 0 { - return fmt.Errorf("golem15.sitemap: %s must not be negative, got %d", CacheConfigKey, seconds) - } - p.tiers = tiers - p.cache = newSitemapCache(time.Duration(seconds) * time.Second) return nil } diff --git a/routes.go b/routes.go index 6a18bb9..e05cbb2 100644 --- a/routes.go +++ b/routes.go @@ -1,52 +1,41 @@ package sitemap import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" + "bytes" "errors" - "net" "net/http" - "net/netip" - "strings" - "sync" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" - "git.golem15.com/golem15/summercms/modules/signpost" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) -// The response headers of the PHP routes (live responses). +// The response headers of the sitemap routes. const ( contentTypeXML = "application/xml" - contentTypeXSL = "text/xsl; charset=UTF-8" cacheControl = "no-cache, private" ) -// BucketSitemap is the throttle bucket of GET /sitemap.xml. +// BucketSitemap is the throttle bucket of the sitemap routes. const BucketSitemap = "sitemap-xml" -// CacheConfigKey is the config key of the number of seconds a generated -// sitemap is served from memory; 0 turns the cache off. -const CacheConfigKey = "golem15.sitemap.cache_seconds" +// FilesPath is the route of the per-group files. The index lists them at +// the site root (sitemap--.xml); the site's SEO layer or proxy +// maps the root names to this route. +const FilesPath = "/_sitemap/files/" -// cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the -// request's scheme and Host when app.url is not set) and stored definition -// and settings. A full cache is emptied. -const cacheMaxEntries = 16 - -// Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application -// root without the backend middleware: search engines and browsers fetch -// them anonymously. /sitemap.xml resolves every registry item, so it is -// throttled per client IP (BucketSitemap) and its output is cached. +// Routes mounts GET /sitemap.xml (the index) and GET +// /_sitemap/files/{file} (one file) without the backend middleware: search +// engines fetch them anonymously. Both are served from the slot table, never +// generated per request, and throttled per client IP (BucketSitemap). func (p *Plugin) Routes(r pact.Router) error { - r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap) - r.Get("/sitemap.xsl", p.sitemapXSL) + r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) + r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) + r.Where("file", `sitemap-[a-z0-9-]+-[0-9]+\.xml`) return nil } @@ -70,159 +59,119 @@ func (p *Plugin) Buckets() map[string]surf.Bucket { } } -// sitemapCache holds generated sitemaps for the configured time. The key -// covers the root URL and the stored definition and settings, so a save of -// either is served at once; registry content (a new post) shows after the -// entry expires. gen lets one generation run at a time, so concurrent misses -// wait for it instead of each resolving every registry item. -type sitemapCache struct { - ttl time.Duration - gen sync.Mutex - mu sync.Mutex - entries map[string]cachedSitemap +// indexRow is one file of the index query. +type indexRow struct { + Grp string `gorm:"column:grp"` + FileNo int `gorm:"column:file_no"` + LastMod *time.Time `gorm:"column:lastmod"` } -type cachedSitemap struct { - xml string - expires time.Time -} - -func newSitemapCache(ttl time.Duration) *sitemapCache { - return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}} -} - -func (c *sitemapCache) get(key string) (string, bool) { - if c == nil || c.ttl <= 0 || key == "" { - return "", false - } - c.mu.Lock() - defer c.mu.Unlock() - e, ok := c.entries[key] - if !ok || time.Now().After(e.expires) { - return "", false - } - return e.xml, true -} - -func (c *sitemapCache) put(key, xml string) { - if c == nil || c.ttl <= 0 || key == "" { - return - } - c.mu.Lock() - defer c.mu.Unlock() - if len(c.entries) >= cacheMaxEntries { - clear(c.entries) - } - c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)} -} - -// cacheKey is a digest of the root URL, the definition's items and the -// settings; "" (no caching) when either does not encode. -func cacheKey(root string, def models.Definition, settings models.Settings) string { - items, err := json.Marshal(def.Items()) - if err != nil { - return "" - } - set, err := json.Marshal(settings) - if err != nil { - return "" - } - h := sha256.New() - for _, part := range [][]byte{[]byte(root), items, set} { - h.Write(part) - h.Write([]byte{0}) - } - return hex.EncodeToString(h.Sum(nil)) -} - -// sitemapXSL serves the embedded stylesheet, the PHP file byte for byte. -func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeXSL) - w.Header().Set("Cache-Control", cacheControl) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(sitemapXSL) -} - -// sitemapXML serves the definition with the fixed theme key: 404 when there -// is none, an empty application/xml body when it has no items, else the -// generated XML with the settings' frontend_url rewrite applied. -func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { - app := p.app - db, err := database(app) +// sitemapIndex serves the sitemap index: one per file that lists +// at least one address, with the newest lastmod of its addresses. Without a +// site address (the setting, golem15.sitemap.site_url or app.url) it is a +// 404: a is never built from the request's Host. +func (p *Plugin) sitemapIndex(w http.ResponseWriter, r *http.Request) { + db, err := database(p.app) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } - var defs []models.Definition - if err := db.WithContext(r.Context()).Where("theme = ?", models.DefinitionTheme).Order("id ASC").Limit(1).Find(&defs).Error; err != nil { - pluginLog.Error("golem15.sitemap: load definition", "error", err) - http.Error(w, "Internal Server Error", http.StatusInternalServerError) - return - } - if len(defs) == 0 { - pluginLog.Info("golem15.sitemap: No sitemap definition was found. Try creating one first.") - http.NotFound(w, r) - return - } - settings, err := loadSettings(db.WithContext(r.Context())) + db = db.WithContext(r.Context()) + settings, err := classes.LoadSettings(db) if err != nil { pluginLog.Error("golem15.sitemap: load settings", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } - root := backendURL(app, r) - xml, err := p.generate(r, root, defs[0], settings) - if err != nil { - pluginLog.Error("golem15.sitemap: generate", "error", err) + site := classes.SiteURL(p.app, settings) + if site == "" { + pluginLog.Warn("golem15.sitemap: no site address; set the sitemap setting, golem15.sitemap.site_url or app.url") + http.NotFound(w, r) + return + } + var rows []indexRow + if err := db.Raw(`SELECT grp, file_no, MAX(lastmod) AS lastmod FROM golem15_sitemap_urls +WHERE excluded_reason = '' AND file_no IS NOT NULL +GROUP BY grp, file_no ORDER BY grp, file_no`).Scan(&rows).Error; err != nil { + pluginLog.Error("golem15.sitemap: index", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } + files := make([]classes.IndexFile, 0, len(rows)) + for _, row := range rows { + files = append(files, classes.IndexFile{Loc: site + "/" + classes.FileName(row.Grp, row.FileNo), LastMod: row.LastMod}) + } + var buf bytes.Buffer + if err := classes.WriteIndex(&buf, files); err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + writeXML(w, buf.Bytes(), false) +} + +// sitemapFile serves one file: its listed addresses in the order they were +// first seen. An unknown file, or one without a listed address, is a 404. +func (p *Plugin) sitemapFile(w http.ResponseWriter, r *http.Request) { + grp, n, ok := classes.ParseFileName(r.PathValue("file")) + if !ok { + http.NotFound(w, r) + return + } + db, err := database(p.app) + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + db = db.WithContext(r.Context()) + settings, err := classes.LoadSettings(db) + if err != nil { + pluginLog.Error("golem15.sitemap: load settings", "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + site := classes.SiteURL(p.app, settings) + var rows []models.URL + if err := db.Where("grp = ? AND file_no = ? AND excluded_reason = ''", grp, n).Order("first_seen ASC, id ASC").Find(&rows).Error; err != nil { + pluginLog.Error("golem15.sitemap: file", "file", r.PathValue("file"), "error", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + entries := make([]classes.FileEntry, 0, len(rows)) + for _, row := range rows { + loc := classes.Absolute(site, row.URL) + if loc == "" { + continue + } + entry := classes.FileEntry{Loc: loc, LastMod: row.LastMod} + for _, a := range row.Alternates.Get() { + if href := classes.Absolute(site, a.Href); href != "" { + entry.Alternates = append(entry.Alternates, classes.FileAlternate{Hreflang: a.Hreflang, Href: href}) + } + } + entries = append(entries, entry) + } + if len(entries) == 0 { + http.NotFound(w, r) + return + } + var buf bytes.Buffer + if err := classes.WriteURLSet(&buf, entries); err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + writeXML(w, buf.Bytes(), true) +} + +// writeXML answers 200 with an XML body. A file carries X-Robots-Tag +// noindex, so search engines read it without indexing it as a page. +func writeXML(w http.ResponseWriter, body []byte, noindex bool) { w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) + if noindex { + w.Header().Set("X-Robots-Tag", "noindex") + } w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(xml)) -} - -// generate returns the sitemap of def with the frontend_url rewrite -// applied, from the cache when it holds it. -func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) { - cache := p.cache - key := "" - if cache != nil && cache.ttl > 0 { - key = cacheKey(root, def, settings) - if xml, ok := cache.get(key); ok { - return xml, nil - } - cache.gen.Lock() - defer cache.gen.Unlock() - if xml, ok := cache.get(key); ok { - return xml, nil - } - } - app := p.app - gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events} - if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { - gen.Resolver = reg - } - xml, err := gen.Generate(r.Context(), def.Items()) - if err != nil { - return "", err - } - xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL)) - cache.put(key, xml) - return xml, nil -} - -// loadSettings reads the settings row; without one the PHP defaults apply. -func loadSettings(db *gorm.DB) (models.Settings, error) { - var rows []models.Settings - if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { - return models.Settings{}, err - } - if len(rows) == 0 { - return models.DefaultSettings(), nil - } - return rows[0], nil + _, _ = w.Write(body) } // database returns the app's published *gorm.DB. @@ -236,59 +185,3 @@ func database(app *backpack.App) (*gorm.DB, error) { } return db, nil } - -// backendURL is the root URL the sitemap's absolute URLs start with, PHP -// url('/') right-trimmed of slashes: app.url when it is set, else the -// request's scheme and Host. The scheme is https for a TLS connection, or -// when the immediate peer is in http.trusted_proxies and its last -// X-Forwarded-Proto value says https; a client that sends the header -// directly is ignored. -func backendURL(app *backpack.App, r *http.Request) string { - if app != nil && app.Config != nil { - if u := strings.TrimRight(strings.TrimSpace(app.Config.String("app.url")), "/"); u != "" { - return u - } - } - scheme := "http" - if isHTTPS(app, r) { - scheme = "https" - } - return strings.TrimRight(scheme+"://"+r.Host, "/") -} - -func isHTTPS(app *backpack.App, r *http.Request) bool { - if r.TLS != nil { - return true - } - if app == nil || app.Config == nil { - return false - } - trusted := surf.TrustedProxies(app.Config) - if len(trusted) == 0 || !peerTrusted(r.RemoteAddr, trusted) { - return false - } - values := r.Header.Values("X-Forwarded-Proto") - if len(values) == 0 { - return false - } - hops := strings.Split(values[len(values)-1], ",") - return strings.EqualFold(strings.TrimSpace(hops[len(hops)-1]), "https") -} - -func peerTrusted(remoteAddr string, trusted []netip.Prefix) bool { - host, _, err := net.SplitHostPort(remoteAddr) - if err != nil { - host = remoteAddr - } - addr, err := netip.ParseAddr(strings.Trim(host, "[]")) - if err != nil { - return false - } - addr = addr.Unmap() - for _, p := range trusted { - if p.Contains(addr) { - return true - } - } - return false -} diff --git a/routes_test.go b/routes_test.go index 63a83f1..476636c 100644 --- a/routes_test.go +++ b/routes_test.go @@ -1,22 +1,21 @@ package sitemap import ( - "crypto/tls" - "net/http/httptest" "os" "path/filepath" "testing" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/compass" ) -// TestBackendURL pins the root URL the sitemap's absolute URLs and the -// frontend_url rewrite start from: app.url when set; else the request's -// Host with https for TLS, or for the last X-Forwarded-Proto hop of a peer -// in http.trusted_proxies, while a client that sends the header directly -// gets http. -func TestBackendURL(t *testing.T) { +// TestSiteURLPrecedence pins the site address every starts with +// (CONTEXT 2026-10-28): the setting, else golem15.sitemap.site_url, else +// app.url, each only when it is an absolute http(s) URL, trailing slashes +// trimmed; nothing configured is "" (the request's Host is never used). +func TestSiteURLPrecedence(t *testing.T) { app := func(t *testing.T, files map[string]string) *backpack.App { t.Helper() dir := t.TempDir() @@ -31,39 +30,52 @@ func TestBackendURL(t *testing.T) { } return backpack.New(cfg) } - trusted := app(t, map[string]string{"http.yaml": "trusted_proxies:\n - 10.0.0.0/8\n"}) + text := func(s string) *string { return &s } + both := app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n", "golem15.yaml": "sitemap:\n site_url: https://config.example.test/\n"}) + appOnly := app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n"}) for _, tc := range []struct { - name string - app *backpack.App - remote string - proto []string - tls bool - want string + name string + app *backpack.App + settings models.Settings + want string }{ - {"app.url wins", app(t, map[string]string{"app.yaml": "url: https://api.example.test/\n"}), "203.0.113.9:1234", []string{"http"}, false, "https://api.example.test"}, - {"plain request", trusted, "203.0.113.9:1234", nil, false, "http://h.test"}, - {"TLS", trusted, "203.0.113.9:1234", nil, true, "https://h.test"}, - {"trusted proxy says https", trusted, "10.1.2.3:5000", []string{"https"}, false, "https://h.test"}, - {"trusted proxy, last hop wins", trusted, "10.1.2.3:5000", []string{"https, http"}, false, "http://h.test"}, - {"trusted proxy, last header wins", trusted, "10.1.2.3:5000", []string{"http", "HTTPS"}, false, "https://h.test"}, - {"untrusted client says https", trusted, "203.0.113.9:1234", []string{"https"}, false, "http://h.test"}, - {"no trusted proxies configured", app(t, nil), "10.1.2.3:5000", []string{"https"}, false, "http://h.test"}, - {"nil app", nil, "10.1.2.3:5000", []string{"https"}, false, "http://h.test"}, - {"IPv4-mapped trusted peer", trusted, "[::ffff:10.1.2.3]:5000", []string{"https"}, false, "https://h.test"}, - {"unparsable peer", trusted, "not-an-address", []string{"https"}, false, "http://h.test"}, + {"setting wins", both, models.Settings{SiteURL: text("https://www.example.test/")}, "https://www.example.test"}, + {"config key", both, models.Settings{}, "https://config.example.test"}, + {"app.url", appOnly, models.Settings{}, "https://api.example.test"}, + {"unsafe setting skipped", appOnly, models.Settings{SiteURL: text("javascript:alert(1)")}, "https://api.example.test"}, + {"nothing configured", app(t, nil), models.Settings{}, ""}, + {"nil app", nil, models.Settings{}, ""}, } { t.Run(tc.name, func(t *testing.T) { - r := httptest.NewRequest("GET", "http://h.test/sitemap.xml", nil) - r.RemoteAddr = tc.remote - for _, v := range tc.proto { - r.Header.Add("X-Forwarded-Proto", v) - } - if tc.tls { - r.TLS = &tls.ConnectionState{} - } - if got := backendURL(tc.app, r); got != tc.want { - t.Fatalf("backendURL = %q, want %q", got, tc.want) + if got := classes.SiteURL(tc.app, tc.settings); got != tc.want { + t.Fatalf("SiteURL = %q, want %q", got, tc.want) } }) } } + +// TestFileNames pins the per-group file names the index lists and the file +// route accepts. +func TestFileNames(t *testing.T) { + if got := classes.FileName("posts", 3); got != "sitemap-posts-3.xml" { + t.Fatalf("FileName = %q", got) + } + for name, want := range map[string]struct { + grp string + n int + ok bool + }{ + "sitemap-posts-1.xml": {"posts", 1, true}, + "sitemap-landing-pages-12.xml": {"landing-pages", 12, true}, + "sitemap-posts-2-1.xml": {"posts-2", 1, true}, + "sitemap-posts-0.xml": {"", 0, false}, + "sitemap-Posts-1.xml": {"", 0, false}, + "sitemap-posts.xml": {"", 0, false}, + "sitemap-../x-1.xml": {"", 0, false}, + } { + grp, n, ok := classes.ParseFileName(name) + if grp != want.grp || n != want.n || ok != want.ok { + t.Errorf("ParseFileName(%q) = %q %d %v, want %+v", name, grp, n, ok, want) + } + } +} diff --git a/sitemap_index_test.go b/sitemap_index_test.go new file mode 100644 index 0000000..1b17476 --- /dev/null +++ b/sitemap_index_test.go @@ -0,0 +1,133 @@ +package sitemap_test + +import ( + "context" + "net/http" + "regexp" + "strings" + "testing" + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/summercms/modules/pact" +) + +// bootIndex boots the fake content plugin and the sitemap on a fresh +// database with an empty content store. +func bootIndex(t *testing.T, extra ...string) *sitemapEnv { + t.Helper() + contentStore.reset(t) + return bootPlugins(t, append([]string{contentPluginID, "golem15.sitemap"}, extra...)) +} + +// rebuild runs the plugin's rebuild job once, as a worker would. +func (e *sitemapEnv) rebuild(t *testing.T) { + t.Helper() + for _, p := range e.plugins { + hj, ok := p.(pact.HasJobs) + if !ok || p.ID() != "golem15.sitemap" { + continue + } + for _, j := range hj.Jobs() { + if err := j.Work(context.Background(), classes.RebuildArgs{}); err != nil { + t.Fatalf("rebuild job: %v", err) + } + return + } + } + t.Fatal("the sitemap registers no rebuild job") +} + +// setFileSize stores the file size setting. +func (e *sitemapEnv) setFileSize(t *testing.T, n int) { + t.Helper() + if err := e.gdb.Exec(`UPDATE golem15_sitemap_settings SET file_size = ? WHERE id = 1`, n).Error; err != nil { + t.Fatal(err) + } +} + +// locs lists the values of a sitemap body in order. +func locs(body string) []string { + var out []string + for _, m := range regexp.MustCompile(`([^<]*)`).FindAllStringSubmatch(body, -1) { + out = append(out, m[1]) + } + return out +} + +// TestSitemapIndex is the tracer of D-30: a content type's published +// records appear, after a rebuild job, in an index at /sitemap.xml that +// links per-group files of file_size addresses with absolute and a +// real ; a draft is absent, no priority or change frequency is +// written, values are XML-escaped (T-33-09) and an unknown file is a 404. +func TestSitemapIndex(t *testing.T) { + e := bootIndex(t) + e.setFileSize(t, 2) + base := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC) + for i, slug := range []string{"a", "b", "c", "d"} { + contentStore.put(pageType, page1(i+1, slug, base.Add(time.Duration(i)*time.Hour))) + } + contentStore.put(pageType, fakeRecord{ID: 5, Published: true, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: `/pages/fish&chips?x=<1>"`, Title: `Fish & "chips" `}}}) + contentStore.put(pageType, fakeRecord{ID: 6, Published: false, LastMod: base, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/draft", Title: "Draft"}}}) + + if rec := e.get(t, "/sitemap.xml"); rec.Code != http.StatusOK || len(locs(rec.Body.String())) != 0 { + t.Fatalf("index before the first rebuild: %d %s", rec.Code, rec.Body.String()) + } + e.rebuild(t) + + rec := e.get(t, "/sitemap.xml") + body := rec.Body.String() + if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" { + t.Fatalf("GET /sitemap.xml: %d %q %s", rec.Code, rec.Header().Get("Content-Type"), body) + } + if !strings.HasPrefix(body, ``) || !strings.Contains(body, ``) { + t.Fatalf("index head:\n%s", body) + } + want := []string{ + "https://api.example.test/sitemap-pages-1.xml", + "https://api.example.test/sitemap-pages-2.xml", + "https://api.example.test/sitemap-pages-3.xml", + } + if got := locs(body); strings.Join(got, " ") != strings.Join(want, " ") { + t.Fatalf("index locs = %v, want %v\n%s", got, want, body) + } + if n := strings.Count(body, ""); n != 3 { + t.Fatalf("index has %d lastmod elements:\n%s", n, body) + } + if !strings.Contains(body, "https://api.example.test/sitemap-pages-1.xml2026-10-01T09:00:00Z") { + t.Fatalf("file 1 lastmod is not the newest of its addresses:\n%s", body) + } + + rec = e.get(t, "/_sitemap/files/sitemap-pages-1.xml") + file := rec.Body.String() + if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/xml" || rec.Header().Get("X-Robots-Tag") != "noindex" { + t.Fatalf("file 1: %d %v %s", rec.Code, rec.Header(), file) + } + if got := locs(file); strings.Join(got, " ") != "https://api.example.test/pages/a https://api.example.test/pages/b" { + t.Fatalf("file 1 locs = %v\n%s", got, file) + } + if !strings.Contains(file, "https://api.example.test/pages/a2026-10-01T08:00:00Z") { + t.Fatalf("file 1 lacks a real lastmod:\n%s", file) + } + file3 := e.get(t, "/_sitemap/files/sitemap-pages-3.xml").Body.String() + if !strings.Contains(file3, "https://api.example.test/pages/fish&chips?x=<1>"") { + t.Fatalf("file 3 does not escape the address:\n%s", file3) + } + all := body + file + e.get(t, "/_sitemap/files/sitemap-pages-2.xml").Body.String() + file3 + for _, banned := range []string{"", "", "/pages/draft", "xml-stylesheet", "xmlns:xhtml"} { + if strings.Contains(all, banned) { + t.Errorf("the sitemap contains %q:\n%s", banned, all) + } + } + for _, path := range []string{ + "/_sitemap/files/sitemap-pages-4.xml", + "/_sitemap/files/sitemap-posts-1.xml", + "/_sitemap/files/sitemap-pages-0.xml", + "/_sitemap/files/other.xml", + "/sitemap.xsl", + } { + if rec := e.get(t, path); rec.Code != http.StatusNotFound { + t.Errorf("GET %s: %d, want 404", path, rec.Code) + } + } +} diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index b1817b3..3824357 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -15,9 +15,7 @@ import ( "testing" "time" - sitemap "git.golem15.com/golem15/sm-sitemap-plugin" - "git.golem15.com/golem15/sm-sitemap-plugin/classes" - "git.golem15.com/golem15/sm-sitemap-plugin/console" + _ "git.golem15.com/golem15/sm-sitemap-plugin" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" @@ -27,7 +25,6 @@ import ( "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" - "git.golem15.com/golem15/summercms/modules/signpost" "git.golem15.com/golem15/summercms/modules/surf" "gorm.io/gorm" ) @@ -101,15 +98,22 @@ func init() { // sitemapEnv is a booted application with the sitemap and the fake types. type sitemapEnv struct { - h http.Handler - app *backpack.App - gdb *gorm.DB + h http.Handler + app *backpack.App + gdb *gorm.DB + plugins []party.Plugin } -// bootSitemap boots the application. The sitemap cache is off -// (golem15.sitemap.cache_seconds 0): the smoke tests change listeners and -// registry content between requests. overrides set further config keys. +// bootSitemap boots the application with the sitemap and the fake types; +// overrides set further config keys. func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { + t.Helper() + return bootPlugins(t, []string{testTypesPluginID, "golem15.sitemap"}, overrides...) +} + +// bootPlugins boots the application with the plugins ids on a fresh +// database; overrides set further config keys. +func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv { t.Helper() sqlDB := freshDB(t) gdb, err := lagoon.Use(t.Context(), sqlDB) @@ -132,7 +136,6 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { "http.body_limits.upload_bytes": 8388608, "mail.driver": "log", "mail.from": "noreply@example.test", - sitemap.CacheConfigKey: 0, } for _, o := range overrides { for key, value := range o { @@ -148,7 +151,7 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { if err := lagoon.Publish(app, sqlDB, gdb); err != nil { t.Fatal(err) } - plugins, err := party.Activate(app, []string{testTypesPluginID, "golem15.sitemap"}) + plugins, err := party.Activate(app, ids) if err != nil { t.Fatal(err) } @@ -159,7 +162,7 @@ func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { if err != nil { t.Fatal(err) } - return &sitemapEnv{h: h, app: app, gdb: gdb} + return &sitemapEnv{h: h, app: app, gdb: gdb, plugins: plugins} } func (e *sitemapEnv) get(t *testing.T, path string) *httptest.ResponseRecorder { @@ -184,100 +187,6 @@ func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem func text(s string) *string { return &s } -// TestSitemapCacheAndThrottle pins WR-05: with the cache on, a repeated GET -// is served from memory (a listener added later does not run), a save of -// the definition is served at once, and the 61st request of a client within -// a minute is a 429. -func TestSitemapCacheAndThrottle(t *testing.T) { - e := bootSitemap(t, map[string]any{sitemap.CacheConfigKey: 60}) - def := e.storeDefinition(t, []models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")}, - {Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")}, - }) - requests := 0 - get := func() *httptest.ResponseRecorder { - requests++ - return e.get(t, "/sitemap.xml") - } - first := get() - if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), "https://api.example.test/") { - t.Fatalf("first GET: %d %s", first.Code, first.Body.String()) - } - e.app.Events.Listen("acme.skipall", func(_ context.Context, ev *classes.BeforeAddItem) error { - ev.Skip = true - return nil - }) - if again := get(); again.Code != http.StatusOK || again.Body.String() != first.Body.String() { - t.Fatalf("a repeated GET was generated again:\n%s", again.Body.String()) - } - def.SetItems(append(def.Items(), models.DefinitionItem{Type: models.ItemTypeURL, URL: text("/x")})) - if err := e.gdb.Save(&def).Error; err != nil { - t.Fatal(err) - } - if saved := get(); saved.Code != http.StatusOK || strings.Contains(saved.Body.String(), "") { - t.Fatalf("a saved definition was served from the cache:\n%s", saved.Body.String()) - } - for requests < 60 { - if rec := get(); rec.Code != http.StatusOK { - t.Fatalf("request %d: %d", requests, rec.Code) - } - } - if rec := get(); rec.Code != http.StatusTooManyRequests { - t.Fatalf("request %d: %d, want 429", requests, rec.Code) - } - if rec := e.get(t, "/sitemap.xsl"); rec.Code != http.StatusOK { - t.Fatalf("/sitemap.xsl is throttled with the sitemap: %d", rec.Code) - } -} - -// TestSitemapTracer stores a definition with a url item and a registry -// item, GETs /sitemap.xml and compares the body with the PHP golden. -func TestSitemapTracer(t *testing.T) { - e := bootSitemap(t) - - if rec := e.get(t, "/sitemap.xml"); rec.Code != http.StatusNotFound { - t.Fatalf("no definition: %d, want 404", rec.Code) - } - - def := e.storeDefinition(t, nil) - rec := e.get(t, "/sitemap.xml") - if rec.Code != http.StatusOK || rec.Body.Len() != 0 || rec.Header().Get("Content-Type") != "application/xml" { - t.Fatalf("empty definition: %d %q %q", rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) - } - - def.SetItems([]models.DefinitionItem{ - {Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")}, - {Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")}, - }) - if err := e.gdb.Save(&def).Error; err != nil { - t.Fatal(err) - } - rec = e.get(t, "/sitemap.xml") - if rec.Code != http.StatusOK { - t.Fatalf("GET /sitemap.xml: %d %s", rec.Code, rec.Body.String()) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/xml" { - t.Fatalf("Content-Type = %q", ct) - } - want, err := os.ReadFile(filepath.Join("classes", "testdata", "tracer.golden")) - if err != nil { - t.Fatal(err) - } - if got := rec.Body.String(); got != string(want) { - t.Fatalf("/sitemap.xml differs from the PHP golden\n got: %q\nwant: %q", got, want) - } - - // The stored data keeps the PHP item keys. - var raw string - if err := e.gdb.Raw(`SELECT data FROM golem15_sitemap_definitions WHERE id = ?`, def.ID).Scan(&raw).Error; err != nil { - t.Fatal(err) - } - const wantData = `[{"changefreq":"always","priority":"1.0","nesting":false,"type":"url","url":"/","reference":null,"cmsPage":null,"urlPattern":null},{"changefreq":"weekly","priority":"0.5","nesting":true,"type":"acme-post","url":null,"reference":"7","cmsPage":null,"urlPattern":""}]` - if raw != wantData { - t.Fatalf("stored data = %s", raw) - } -} - func writeFile(t *testing.T, dir, name, body string) string { t.Helper() path := filepath.Join(dir, name) @@ -292,163 +201,6 @@ func testOutput() (bonfire.Output, *bytes.Buffer) { return bonfire.NewOutput(strings.NewReader(""), &buf, &buf), &buf } -// TestSitemapImportSmoke copies a Winter export through sitemap:import-winter, -// then checks the frontend_url rewrite, the stylesheet route, a -// BeforeAddItem listener and sitemap:import. -func TestSitemapImportSmoke(t *testing.T) { - e := bootSitemap(t) - ctx := context.Background() - reg, ok := e.app.Lookup[*signpost.Registry]() - if !ok { - t.Fatal("signpost registry is not published") - } - dir := t.TempDir() - data := `[{"changefreq":"always","priority":"1.0","nesting":"0","type":"url","url":"/","code":null,"reference":null,"cmsPage":null,"urlPattern":null,"exists":false,"fillable":["changefreq"]},` + - `{"changefreq":"monthly","priority":"0.5","nesting":"0","type":"static-page","url":null,"reference":"about","cmsPage":null,"urlPattern":null},` + - `{"changefreq":"weekly","priority":"0.4","nesting":"1","type":"blog-posts","url":null,"reference":null,"cmsPage":"blog/post","urlPattern":"/news/:slug"},` + - `{"changefreq":"weekly","priority":"0.4","nesting":"0","type":"acme-post","url":null,"reference":"99","cmsPage":null,"urlPattern":null},` + - `{"changefreq":"daily","priority":"0.7","nesting":"false","type":"cms-page","url":"","reference":"news","cmsPage":"","urlPattern":""}]` - quotedRaw, _ := json.Marshal(data) - quoted := string(quotedRaw) - defs := writeFile(t, dir, "definitions.jsonl", `{"id": 3, "theme": "site-theme", "data": `+quoted+`, "created_at": "2026-01-01 10:00:00", "updated_at": "2026-01-02 10:00:00"}`+"\n") - settings := writeFile(t, dir, "settings.json", `{"frontend_url":"https://www.example.test/","frontend_default_locale":"pl","enable_priority_tiers":"0","priority_homepage":"0.9","legacy_key":"x"}`) - - out, buf := testOutput() - report, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{ - DefinitionsFile: defs, SettingsFile: settings, Maps: map[string]string{"blog-posts": "acme-list"}, - PageURLs: map[string]string{"news": "/news"}, - }, out) - if err != nil { - t.Fatalf("import-winter: %v\n%s", err, buf.String()) - } - if report.Kept != 3 || report.Dropped != 2 || report.Remapped != 2 || !report.Settings { - t.Fatalf("report = %+v\n%s", report, buf.String()) - } - for _, want := range []string{ - "item 1 (type 'static-page', reference 'about'): dropped (static and CMS page types are not supported)", - "item 2 (type 'blog-posts'): type 'blog-posts' mapped to 'acme-list'", - "item 2 (type 'blog-posts'): cmsPage 'blog/post' dropped", - "item 3 (type 'acme-post', reference '99'): dropped (reference '99' is not among the current references of 'acme-post')", - "item 4 (type 'cms-page', reference 'news'): copied as type 'url' with url '/news'", - "setting 'legacy_key' ignored", - "theme 'site-theme'", - } { - if !strings.Contains(buf.String(), want) { - t.Errorf("report lacks %q:\n%s", want, buf.String()) - } - } - var def models.Definition - if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil { - t.Fatal(err) - } - if items := def.Items(); len(items) != 3 || items[0].Type != "url" || items[1].Type != "acme-list" || models.Text(items[1].URLPattern) != "/news/:slug" || - items[2].Type != "url" || models.Text(items[2].URL) != "/news" || items[2].Reference != nil { - t.Fatalf("copied items = %+v", items) - } - var st models.Settings - if err := e.gdb.First(&st, 1).Error; err != nil { - t.Fatal(err) - } - if models.Text(st.FrontendURL) != "https://www.example.test/" || st.EnablePriorityTiers || st.Priority(models.PriorityHomepage) != "0.9" || st.Priority(models.PriorityBlog) != "0.6" { - t.Fatalf("copied settings = %+v", st) - } - - // A second copy without --force is refused. - out, buf = testOutput() - if _, err := console.ImportWinterFiles(ctx, e.gdb, reg, console.WinterOptions{DefinitionsFile: defs}, out); err == nil || !strings.Contains(err.Error(), "--force") { - t.Fatalf("second import: %v\n%s", err, buf.String()) - } - - // The frontend URL replaces the backend URL in loc, hreflang and the - // stylesheet href; the list type expands per alternate. - rec := e.get(t, "/sitemap.xml") - body := rec.Body.String() - if rec.Code != http.StatusOK || rec.Header().Get("Cache-Control") != "no-cache, private" { - t.Fatalf("GET /sitemap.xml: %d %v", rec.Code, rec.Header()) - } - for _, want := range []string{ - `href="https://www.example.test/sitemap.xsl"`, - `https://www.example.test/`, - `https://www.example.test/news`, - `https://www.example.test/news/a`, - `https://www.example.test/en/news/a`, - `hreflang="en" href="https://www.example.test/en/news/a"/>`, - } { - if !strings.Contains(body, want) { - t.Errorf("sitemap lacks %s:\n%s", want, body) - } - } - if strings.Contains(body, "api.example.test") { - t.Errorf("the backend URL is left in the sitemap:\n%s", body) - } - - // /sitemap.xsl answers the PHP file. - rec = e.get(t, "/sitemap.xsl") - want, err := os.ReadFile(filepath.Join("assets", "sitemap.xsl")) - if err != nil { - t.Fatal(err) - } - if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "text/xsl; charset=UTF-8" || !bytes.Equal(rec.Body.Bytes(), want) { - t.Fatalf("GET /sitemap.xsl: %d %q (%d bytes)", rec.Code, rec.Header().Get("Content-Type"), rec.Body.Len()) - } - - // A BeforeAddItem listener that sets Skip drops its URL. - e.app.Events.Listen("acme.skip", func(_ context.Context, ev *classes.BeforeAddItem) error { - if ev.Info.URL == "/en/news/a" { - ev.Skip = true - } - return nil - }) - body = e.get(t, "/sitemap.xml").Body.String() - if strings.Contains(body, "https://www.example.test/en/news/a") || !strings.Contains(body, "https://www.example.test/news/a") { - t.Fatalf("the skipped URL is still written:\n%s", body) - } - - // sitemap:import replaces the url items and keeps the dynamic ones first. - manifest := writeFile(t, dir, "urls.json", `[{"url":"/x","changefreq":"daily"},{"url":"/y","priority":"0.9"}]`) - out, buf = testOutput() - if err := console.ImportFile(ctx, e.gdb, manifest, "", false, out); err != nil { - t.Fatalf("sitemap:import: %v", err) - } - if err := e.gdb.Where("theme = ?", models.DefinitionTheme).First(&def).Error; err != nil { - t.Fatal(err) - } - items := def.Items() - if len(items) != 3 || items[0].Type != "acme-list" || models.Text(items[1].URL) != "/x" || models.Text(items[1].Priority) != "0.5" || models.Text(items[2].Changefreq) != "monthly" { - t.Fatalf("imported items = %+v", items) - } - if !strings.Contains(buf.String(), "Imported 2 URLs into sitemap definition for theme 'default'.") || !strings.Contains(buf.String(), "Kept 1 existing non-URL (dynamic) items.") { - t.Fatalf("sitemap:import output:\n%s", buf.String()) - } - bad := writeFile(t, dir, "bad.json", `[{"url":"/x"},{"changefreq":"daily"}]`) - if err := console.ImportFile(ctx, e.gdb, bad, "", true, out); err == nil || err.Error() != "Entry 1 is missing a 'url' field" { - t.Fatalf("missing url: %v", err) - } - - // T-17-15: a frontend URL written straight into the row (one stored - // before the save check existed) is not applied; the sitemap keeps the - // backend URL and none of the value reaches it. - for _, value := range []string{"javascript:alert(document.domain)//", `https://x"/>https://api.example.test/x") { - t.Errorf("frontend_url %q: the sitemap is not on the backend URL:\n%s", value, body) - } - if strings.Contains(body, "www.example.test") { - t.Errorf("frontend_url %q: the earlier frontend URL is left in the sitemap:\n%s", value, body) - } - } -} - var adminSeq atomic.Uint64 // call sends an admin API request with an optional bearer token and JSON diff --git a/updates/202610280001_create_golem15_sitemap_urls.go b/updates/202610280001_create_golem15_sitemap_urls.go new file mode 100644 index 0000000..6d7b73f --- /dev/null +++ b/updates/202610280001_create_golem15_sitemap_urls.go @@ -0,0 +1,52 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// createURLs creates golem15_sitemap_urls, the slot table the sitemap files +// are served from (D-30): one row per address (content type, record and +// locale; an additional address has type_key extra). grp is the file group, +// file_no the file within it (NULL until the address is numbered), so a +// rebuild keeps every listed address in its file. An address left out keeps +// its row with excluded_reason set and leaves a hole in its file. +var createURLs = []*gormigrate.Migration{ + { + ID: "202610280001_create_golem15_sitemap_urls", + Migrate: func(tx *gorm.DB) error { + for _, stmt := range []string{ + `CREATE TABLE golem15_sitemap_urls ( + id BIGSERIAL PRIMARY KEY, + grp TEXT NOT NULL, + type_key TEXT NOT NULL, + record_id TEXT NOT NULL, + locale TEXT NOT NULL DEFAULT '', + url TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + file_no INTEGER NULL, + first_seen TIMESTAMPTZ NOT NULL, + lastmod TIMESTAMPTZ NULL, + alternates JSONB NOT NULL DEFAULT '[]', + images JSONB NOT NULL DEFAULT '[]', + excluded_reason TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL +)`, + `CREATE UNIQUE INDEX golem15_sitemap_urls_record_unique ON golem15_sitemap_urls (type_key, record_id, locale)`, + `CREATE INDEX golem15_sitemap_urls_file_index ON golem15_sitemap_urls (grp, file_no)`, + } { + if err := tx.Exec(stmt).Error; err != nil { + return err + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`DROP TABLE IF EXISTS golem15_sitemap_urls`).Error + }, + }, +} + +func init() { + Register(createURLs...) +} diff --git a/updates/202610280002_update_golem15_sitemap_settings.go b/updates/202610280002_update_golem15_sitemap_settings.go new file mode 100644 index 0000000..f82aab7 --- /dev/null +++ b/updates/202610280002_update_golem15_sitemap_settings.go @@ -0,0 +1,51 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// settingsColumns are the columns the per-type sitemap format adds to the +// settings singleton: the site address, the file size, the default locale, +// the additional addresses, the debounce time of the next rebuild +// (rebuild_pending_until), the file size the files are numbered for +// (numbered_size; a different file_size renumbers them), when the last +// rebuild finished (built_at) and when one was last requested from the +// admin (rebuild_requested_at). The Phase 17 columns stay in the table, +// unread. +var settingsColumns = []string{ + "site_url TEXT NULL", + "file_size INTEGER NOT NULL DEFAULT 1000", + "default_locale TEXT NULL", + "extra_urls JSONB NULL DEFAULT '[]'", + "rebuild_pending_until TIMESTAMPTZ NULL", + "numbered_size INTEGER NULL", + "built_at TIMESTAMPTZ NULL", + "rebuild_requested_at TIMESTAMPTZ NULL", +} + +var updateSettings = []*gormigrate.Migration{ + { + ID: "202610280002_update_golem15_sitemap_settings", + Migrate: func(tx *gorm.DB) error { + for _, column := range settingsColumns { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings ADD COLUMN ` + column).Error; err != nil { + return err + } + } + return tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error + }, + Rollback: func(tx *gorm.DB) error { + for _, name := range []string{"site_url", "file_size", "default_locale", "extra_urls", "rebuild_pending_until", "numbered_size", "built_at", "rebuild_requested_at"} { + if err := tx.Exec(`ALTER TABLE golem15_sitemap_settings DROP COLUMN IF EXISTS ` + name).Error; err != nil { + return err + } + } + return nil + }, + }, +} + +func init() { + Register(updateSettings...) +}