fix(17): WR-05 throttle /sitemap.xml and cache its output

The anonymous route resolved every registry item on each request with no throttle and no cache. - surf bucket sitemap-xml: 60 requests a minute per client IP. - The generated XML is cached for golem15.sitemap.cache_seconds (default 60, 0 off), keyed by root URL, definition and settings so a save shows at once; one generation runs at a time.
2026-10-12 21:38:39 +02:00
parent 433202bbc8
commit cf5c40e442
5 changed files with 235 additions and 23 deletions
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ The output keeps the PHP plugin's bytes: element order, escaping, whitespace, da

 ## Features

- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, outside any throttle group and without the backend middleware.
+- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, without the backend middleware. `/sitemap.xml` is throttled per client IP and its output is cached for a short time.
 - **One definition.** Table `golem15_sitemap_definitions` keeps the PHP columns (`id`, `theme` nullable and indexed, `data` text, timestamps). The `theme` column holds the fixed key `default` (`models.DefinitionTheme`); `/sitemap.xml` serves that row. `models.Definition.Fillable` is exactly `theme` and `data`, the PHP `$fillable` with `$guarded = ['*']`.
 - **Items.** An item is a URL (`type: url`) or a registry menu item type. Winter.Pages static pages, all static pages and the CMS page field are not ported. Stored item JSON in the PHP shape decodes tolerantly: numbers, strings and booleans are coerced, and unknown keys (`cmsPage`, `code`, `exists`, `fillable` and any other) are ignored. A save writes the PHP `toArray` key set (`changefreq`, `priority`, `nesting`, `type`, `url`, `reference`, `cmsPage` as null, `urlPattern`).
 - **Settings.** A singleton row (`golem15_sitemap_settings`, id 1) holds the frontend URL, the frontend default locale, the updates manifest path and the priority tiers, with the PHP defaults.
@@ -48,6 +48,11 @@ The plugin's own `go.mod` and `go.work` replace the framework with `../summercms
 - `200` with an empty body when the definition has no items;
 - otherwise the sitemap, `Content-Type: application/xml`, `Cache-Control: no-cache, private`.

+Generating the sitemap resolves every registry item (all published posts, all categories, their translations), so the route is guarded:
+
+- **Throttle.** The surf bucket `sitemap-xml` (`sitemap.BucketSitemap`) allows 60 requests a minute per client IP, read behind the host's trusted proxies. The 429 body is surf's `{"message":"Too Many Attempts."}`. `/sitemap.xsl` is not throttled.
+- **Cache.** The generated document is kept in memory for `golem15.sitemap.cache_seconds` seconds (`sitemap.CacheConfigKey`, default `60`; `0` turns it off, a negative value fails the boot). The cache key covers the root URL and the stored definition and settings, so a save of either is served at once; a change of registry content (a new or unpublished post) shows once the entry expires. One generation runs at a time; concurrent requests for a missing entry wait for it.
+
 The document is the PHP `DOMDocument::saveXML()` format: the XML declaration and the `xml-stylesheet` processing instruction each followed by a newline, everything else on one line ending with `</urlset>` and a newline. Items that all resolve to nothing produce a self-closing `<urlset .../>`. Text is escaped like libxml (`&`, `<`, `>` and carriage return; attributes also `"`, tab and newline); single quotes and UTF-8 stay literal.

 Each URL is written in the PHP order (`classes.Generator`):
@@ -169,9 +174,12 @@ The WinterCMS forms are adapted to the admin's YAML:
 | Identifier | Description |
 |------------|-------------|
 | `sitemap.Plugin` | Compiled plugin; `ID` is `golem15.sitemap`. |
-| `sitemap.Plugin.Routes` | `GET /sitemap.xml` and `GET /sitemap.xsl`. |
-| `sitemap.Plugin.Boot` | Compiles the tier rules; a bad rule fails the boot. |
-| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (tier rules). |
+| `sitemap.Plugin.Routes` | `GET /sitemap.xml` (throttled, cached) and `GET /sitemap.xsl`. |
+| `sitemap.Plugin.Buckets` | Registers the `sitemap-xml` throttle bucket: 60 requests a minute per client IP. |
+| `sitemap.BucketSitemap` | `sitemap-xml`, the throttle bucket of `/sitemap.xml`. |
+| `sitemap.CacheConfigKey` | `golem15.sitemap.cache_seconds`, the seconds a generated sitemap is cached. |
+| `sitemap.Plugin.Boot` | Compiles the tier rules and starts an empty sitemap cache; a bad rule or a negative `cache_seconds` fails the boot. |
+| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (cache time and tier rules). |
 | `sitemap.Plugin.Commands` | Registers `sitemap:import` and `sitemap:import-winter`. |
 | `sitemap.Plugin.AdminControllers` | The definitions controller. |
 | `sitemap.Plugin.Settings` | The Sitemap link and the Sitemap settings form. |
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -1,5 +1,11 @@
 # golem15.sitemap defaults, read as golem15.sitemap.<key>.
 #
+# cache_seconds is how long a generated /sitemap.xml is served from memory.
+# A save of the definition or the settings shows at once; registry content
+# (a new or changed post) shows once the entry expires. 0 turns the cache
+# off.
+cache_seconds: 60
+#
 # The priority tier rules assign a URL the value of one of the five
 # priority settings when Enable priority tiers is on. A URL's path, without
 # trailing slashes ("" for the homepage), is matched against the rules in
--- a/plugin.go
+++ b/plugin.go
@@ -8,6 +8,7 @@ import (
 	"embed"
 	"fmt"
 	"io/fs"
+	"time"

 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/console"
@@ -17,16 +18,18 @@ import (
 	"git.golem15.com/golem15/summercms/modules/bonfire"
 	"git.golem15.com/golem15/summercms/modules/pact"
 	"git.golem15.com/golem15/summercms/modules/party"
+	"git.golem15.com/golem15/summercms/modules/surf"
 	"github.com/go-gormigrate/gormigrate/v2"
 )

 var (
-	_ party.Plugin       = (*Plugin)(nil)
-	_ pact.HasConfig     = (*Plugin)(nil)
-	_ pact.HasMigrations = (*Plugin)(nil)
-	_ pact.HasModels     = (*Plugin)(nil)
-	_ pact.HasRoutes     = (*Plugin)(nil)
-	_ pact.HasCommands   = (*Plugin)(nil)
+	_ party.Plugin        = (*Plugin)(nil)
+	_ pact.HasConfig      = (*Plugin)(nil)
+	_ pact.HasMigrations  = (*Plugin)(nil)
+	_ pact.HasModels      = (*Plugin)(nil)
+	_ pact.HasRoutes      = (*Plugin)(nil)
+	_ pact.HasCommands    = (*Plugin)(nil)
+	_ surf.BucketProvider = (*Plugin)(nil)
 )

 //go:embed config
@@ -41,6 +44,7 @@ var sitemapXSL []byte
 type Plugin struct {
 	app   *backpack.App
 	tiers classes.Tiers
+	cache *sitemapCache
 }

 func (p *Plugin) ID() string { return "golem15.sitemap" }
@@ -56,17 +60,27 @@ func (p *Plugin) Register(app *backpack.App) error {

 // Boot compiles the priority tier rules (classes.TiersConfigKey); a rule
 // with an unknown setting or a pattern that does not compile fails the boot.
+// It also starts an empty sitemap cache for CacheConfigKey seconds; a
+// negative value fails the boot.
 func (p *Plugin) Boot(app *backpack.App) error {
 	p.app = app
 	var tiers classes.Tiers
 	var err error
+	seconds := 0
 	if app != nil {
 		tiers, err = classes.LoadTiers(app.Config)
+		if app.Config != nil {
+			seconds = app.Config.Int(CacheConfigKey)
+		}
 	}
 	if err != nil {
 		return fmt.Errorf("golem15.sitemap: %w", err)
 	}
+	if seconds < 0 {
+		return fmt.Errorf("golem15.sitemap: %s must not be negative, got %d", CacheConfigKey, seconds)
+	}
 	p.tiers = tiers
+	p.cache = newSitemapCache(time.Duration(seconds) * time.Second)
 	return nil
 }

--- a/routes.go
+++ b/routes.go
@@ -1,12 +1,17 @@
 package sitemap

 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"log/slog"
 	"net"
 	"net/http"
 	"net/netip"
 	"strings"
+	"sync"
+	"time"

 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
@@ -24,15 +29,113 @@ const (
 	cacheControl   = "no-cache, private"
 )

+// BucketSitemap is the throttle bucket of GET /sitemap.xml.
+const BucketSitemap = "sitemap-xml"
+
+// CacheConfigKey is the config key of the number of seconds a generated
+// sitemap is served from memory; 0 turns the cache off.
+const CacheConfigKey = "golem15.sitemap.cache_seconds"
+
+// cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the
+// request's scheme and Host when app.url is not set) and stored definition
+// and settings. A full cache is emptied.
+const cacheMaxEntries = 16
+
 // Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application
-// root, outside any throttle group and without the backend middleware:
-// search engines and browsers fetch them anonymously.
+// root without the backend middleware: search engines and browsers fetch
+// them anonymously. /sitemap.xml resolves every registry item, so it is
+// throttled per client IP (BucketSitemap) and its output is cached.
 func (p *Plugin) Routes(r pact.Router) error {
-	r.Get("/sitemap.xml", p.sitemapXML)
+	r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap)
 	r.Get("/sitemap.xsl", p.sitemapXSL)
 	return nil
 }

+// Buckets registers sitemap-xml: 60 requests a minute per client IP (behind
+// the configured trusted proxies). 429 bodies are surf's
+// {"message":"Too Many Attempts."}.
+func (p *Plugin) Buckets() map[string]surf.Bucket {
+	trusted := surf.TrustedProxies(nil)
+	if p.app != nil {
+		trusted = surf.TrustedProxies(p.app.Config)
+	}
+	return map[string]surf.Bucket{
+		BucketSitemap: {
+			Name:  BucketSitemap,
+			Max:   60,
+			Decay: time.Minute,
+			Key: func(r *http.Request) string {
+				return "sitemap-xml|" + surf.ClientIP(r, trusted)
+			},
+		},
+	}
+}
+
+// sitemapCache holds generated sitemaps for the configured time. The key
+// covers the root URL and the stored definition and settings, so a save of
+// either is served at once; registry content (a new post) shows after the
+// entry expires. gen lets one generation run at a time, so concurrent misses
+// wait for it instead of each resolving every registry item.
+type sitemapCache struct {
+	ttl     time.Duration
+	gen     sync.Mutex
+	mu      sync.Mutex
+	entries map[string]cachedSitemap
+}
+
+type cachedSitemap struct {
+	xml     string
+	expires time.Time
+}
+
+func newSitemapCache(ttl time.Duration) *sitemapCache {
+	return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}}
+}
+
+func (c *sitemapCache) get(key string) (string, bool) {
+	if c == nil || c.ttl <= 0 || key == "" {
+		return "", false
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.entries[key]
+	if !ok || time.Now().After(e.expires) {
+		return "", false
+	}
+	return e.xml, true
+}
+
+func (c *sitemapCache) put(key, xml string) {
+	if c == nil || c.ttl <= 0 || key == "" {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if len(c.entries) >= cacheMaxEntries {
+		clear(c.entries)
+	}
+	c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)}
+}
+
+// cacheKey is a digest of the root URL, the definition's items and the
+// settings; "" (no caching) when either does not encode.
+func cacheKey(root string, def models.Definition, settings models.Settings) string {
+	items, err := json.Marshal(def.Items())
+	if err != nil {
+		return ""
+	}
+	set, err := json.Marshal(settings)
+	if err != nil {
+		return ""
+	}
+	h := sha256.New()
+	for _, part := range [][]byte{[]byte(root), items, set} {
+		h.Write(part)
+		h.Write([]byte{0})
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
 // sitemapXSL serves the embedded stylesheet, the PHP file byte for byte.
 func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Content-Type", contentTypeXSL)
@@ -69,23 +172,48 @@ func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	root := backendURL(app, r)
-	gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events}
-	if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
-		gen.Resolver = reg
-	}
-	xml, err := gen.Generate(r.Context(), defs[0].Items())
+	xml, err := p.generate(r, root, defs[0], settings)
 	if err != nil {
 		slog.Default().Error("golem15.sitemap: generate", "error", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
-	xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL))
 	w.Header().Set("Content-Type", contentTypeXML)
 	w.Header().Set("Cache-Control", cacheControl)
 	w.WriteHeader(http.StatusOK)
 	_, _ = w.Write([]byte(xml))
 }

+// generate returns the sitemap of def with the frontend_url rewrite
+// applied, from the cache when it holds it.
+func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) {
+	cache := p.cache
+	key := ""
+	if cache != nil && cache.ttl > 0 {
+		key = cacheKey(root, def, settings)
+		if xml, ok := cache.get(key); ok {
+			return xml, nil
+		}
+		cache.gen.Lock()
+		defer cache.gen.Unlock()
+		if xml, ok := cache.get(key); ok {
+			return xml, nil
+		}
+	}
+	app := p.app
+	gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events}
+	if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
+		gen.Resolver = reg
+	}
+	xml, err := gen.Generate(r.Context(), def.Items())
+	if err != nil {
+		return "", err
+	}
+	xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL))
+	cache.put(key, xml)
+	return xml, nil
+}
+
 // loadSettings reads the settings row; without one the PHP defaults apply.
 func loadSettings(db *gorm.DB) (models.Settings, error) {
 	var rows []models.Settings
--- a/sitemap_smoke_test.go
+++ b/sitemap_smoke_test.go
@@ -15,7 +15,7 @@ import (
 	"testing"
 	"time"

-	_ "git.golem15.com/golem15/sm-sitemap-plugin"
+	sitemap "git.golem15.com/golem15/sm-sitemap-plugin"
 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/sm-sitemap-plugin/console"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
@@ -106,7 +106,10 @@ type sitemapEnv struct {
 	gdb *gorm.DB
 }

-func bootSitemap(t *testing.T) *sitemapEnv {
+// bootSitemap boots the application. The sitemap cache is off
+// (golem15.sitemap.cache_seconds 0): the smoke tests change listeners and
+// registry content between requests. overrides set further config keys.
+func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
 	t.Helper()
 	sqlDB := freshDB(t)
 	gdb, err := lagoon.Use(t.Context(), sqlDB)
@@ -124,12 +127,19 @@ func bootSitemap(t *testing.T) *sitemapEnv {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for key, value := range map[string]any{
+	settings := map[string]any{
 		"http.body_limits.default_bytes": 1048576,
 		"http.body_limits.upload_bytes":  8388608,
 		"mail.driver":                    "log",
 		"mail.from":                      "noreply@example.test",
-	} {
+		sitemap.CacheConfigKey:           0,
+	}
+	for _, o := range overrides {
+		for key, value := range o {
+			settings[key] = value
+		}
+	}
+	for key, value := range settings {
 		if err := cfg.Set(key, value); err != nil {
 			t.Fatal(err)
 		}
@@ -174,6 +184,52 @@ func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem

 func text(s string) *string { return &s }

+// TestSitemapCacheAndThrottle pins WR-05: with the cache on, a repeated GET
+// is served from memory (a listener added later does not run), a save of
+// the definition is served at once, and the 61st request of a client within
+// a minute is a 429.
+func TestSitemapCacheAndThrottle(t *testing.T) {
+	e := bootSitemap(t, map[string]any{sitemap.CacheConfigKey: 60})
+	def := e.storeDefinition(t, []models.DefinitionItem{
+		{Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")},
+		{Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")},
+	})
+	requests := 0
+	get := func() *httptest.ResponseRecorder {
+		requests++
+		return e.get(t, "/sitemap.xml")
+	}
+	first := get()
+	if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), "<loc>https://api.example.test/</loc>") {
+		t.Fatalf("first GET: %d %s", first.Code, first.Body.String())
+	}
+	e.app.Events.Listen("acme.skipall", func(_ context.Context, ev *classes.BeforeAddItem) error {
+		ev.Skip = true
+		return nil
+	})
+	if again := get(); again.Code != http.StatusOK || again.Body.String() != first.Body.String() {
+		t.Fatalf("a repeated GET was generated again:\n%s", again.Body.String())
+	}
+	def.SetItems(append(def.Items(), models.DefinitionItem{Type: models.ItemTypeURL, URL: text("/x")}))
+	if err := e.gdb.Save(&def).Error; err != nil {
+		t.Fatal(err)
+	}
+	if saved := get(); saved.Code != http.StatusOK || strings.Contains(saved.Body.String(), "<url>") {
+		t.Fatalf("a saved definition was served from the cache:\n%s", saved.Body.String())
+	}
+	for requests < 60 {
+		if rec := get(); rec.Code != http.StatusOK {
+			t.Fatalf("request %d: %d", requests, rec.Code)
+		}
+	}
+	if rec := get(); rec.Code != http.StatusTooManyRequests {
+		t.Fatalf("request %d: %d, want 429", requests, rec.Code)
+	}
+	if rec := e.get(t, "/sitemap.xsl"); rec.Code != http.StatusOK {
+		t.Fatalf("/sitemap.xsl is throttled with the sitemap: %d", rec.Code)
+	}
+}
+
 // TestSitemapTracer stores a definition with a url item and a registry
 // item, GETs /sitemap.xml and compares the body with the PHP golden.
 func TestSitemapTracer(t *testing.T) {