diff --git a/README.md b/README.md index a462a29..5e1688b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The output keeps the PHP plugin's bytes: element order, escaping, whitespace, da ## Features -- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, outside any throttle group and without the backend middleware. +- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, without the backend middleware. `/sitemap.xml` is throttled per client IP and its output is cached for a short time. - **One definition.** Table `golem15_sitemap_definitions` keeps the PHP columns (`id`, `theme` nullable and indexed, `data` text, timestamps). The `theme` column holds the fixed key `default` (`models.DefinitionTheme`); `/sitemap.xml` serves that row. `models.Definition.Fillable` is exactly `theme` and `data`, the PHP `$fillable` with `$guarded = ['*']`. - **Items.** An item is a URL (`type: url`) or a registry menu item type. Winter.Pages static pages, all static pages and the CMS page field are not ported. Stored item JSON in the PHP shape decodes tolerantly: numbers, strings and booleans are coerced, and unknown keys (`cmsPage`, `code`, `exists`, `fillable` and any other) are ignored. A save writes the PHP `toArray` key set (`changefreq`, `priority`, `nesting`, `type`, `url`, `reference`, `cmsPage` as null, `urlPattern`). - **Settings.** A singleton row (`golem15_sitemap_settings`, id 1) holds the frontend URL, the frontend default locale, the updates manifest path and the priority tiers, with the PHP defaults. @@ -48,6 +48,11 @@ The plugin's own `go.mod` and `go.work` replace the framework with `../summercms - `200` with an empty body when the definition has no items; - otherwise the sitemap, `Content-Type: application/xml`, `Cache-Control: no-cache, private`. +Generating the sitemap resolves every registry item (all published posts, all categories, their translations), so the route is guarded: + +- **Throttle.** The surf bucket `sitemap-xml` (`sitemap.BucketSitemap`) allows 60 requests a minute per client IP, read behind the host's trusted proxies. The 429 body is surf's `{"message":"Too Many Attempts."}`. `/sitemap.xsl` is not throttled. +- **Cache.** The generated document is kept in memory for `golem15.sitemap.cache_seconds` seconds (`sitemap.CacheConfigKey`, default `60`; `0` turns it off, a negative value fails the boot). The cache key covers the root URL and the stored definition and settings, so a save of either is served at once; a change of registry content (a new or unpublished post) shows once the entry expires. One generation runs at a time; concurrent requests for a missing entry wait for it. + The document is the PHP `DOMDocument::saveXML()` format: the XML declaration and the `xml-stylesheet` processing instruction each followed by a newline, everything else on one line ending with `` and a newline. Items that all resolve to nothing produce a self-closing ``. Text is escaped like libxml (`&`, `<`, `>` and carriage return; attributes also `"`, tab and newline); single quotes and UTF-8 stay literal. Each URL is written in the PHP order (`classes.Generator`): @@ -169,9 +174,12 @@ The WinterCMS forms are adapted to the admin's YAML: | Identifier | Description | |------------|-------------| | `sitemap.Plugin` | Compiled plugin; `ID` is `golem15.sitemap`. | -| `sitemap.Plugin.Routes` | `GET /sitemap.xml` and `GET /sitemap.xsl`. | -| `sitemap.Plugin.Boot` | Compiles the tier rules; a bad rule fails the boot. | -| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (tier rules). | +| `sitemap.Plugin.Routes` | `GET /sitemap.xml` (throttled, cached) and `GET /sitemap.xsl`. | +| `sitemap.Plugin.Buckets` | Registers the `sitemap-xml` throttle bucket: 60 requests a minute per client IP. | +| `sitemap.BucketSitemap` | `sitemap-xml`, the throttle bucket of `/sitemap.xml`. | +| `sitemap.CacheConfigKey` | `golem15.sitemap.cache_seconds`, the seconds a generated sitemap is cached. | +| `sitemap.Plugin.Boot` | Compiles the tier rules and starts an empty sitemap cache; a bad rule or a negative `cache_seconds` fails the boot. | +| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (cache time and tier rules). | | `sitemap.Plugin.Commands` | Registers `sitemap:import` and `sitemap:import-winter`. | | `sitemap.Plugin.AdminControllers` | The definitions controller. | | `sitemap.Plugin.Settings` | The Sitemap link and the Sitemap settings form. | diff --git a/config/config.yaml b/config/config.yaml index 354b890..6d346a7 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,5 +1,11 @@ # golem15.sitemap defaults, read as golem15.sitemap.. # +# cache_seconds is how long a generated /sitemap.xml is served from memory. +# A save of the definition or the settings shows at once; registry content +# (a new or changed post) shows once the entry expires. 0 turns the cache +# off. +cache_seconds: 60 +# # The priority tier rules assign a URL the value of one of the five # priority settings when Enable priority tiers is on. A URL's path, without # trailing slashes ("" for the homepage), is matched against the rules in diff --git a/plugin.go b/plugin.go index fc942bd..48eb548 100644 --- a/plugin.go +++ b/plugin.go @@ -8,6 +8,7 @@ import ( "embed" "fmt" "io/fs" + "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/console" @@ -17,16 +18,18 @@ import ( "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" + "git.golem15.com/golem15/summercms/modules/surf" "github.com/go-gormigrate/gormigrate/v2" ) var ( - _ party.Plugin = (*Plugin)(nil) - _ pact.HasConfig = (*Plugin)(nil) - _ pact.HasMigrations = (*Plugin)(nil) - _ pact.HasModels = (*Plugin)(nil) - _ pact.HasRoutes = (*Plugin)(nil) - _ pact.HasCommands = (*Plugin)(nil) + _ party.Plugin = (*Plugin)(nil) + _ pact.HasConfig = (*Plugin)(nil) + _ pact.HasMigrations = (*Plugin)(nil) + _ pact.HasModels = (*Plugin)(nil) + _ pact.HasRoutes = (*Plugin)(nil) + _ pact.HasCommands = (*Plugin)(nil) + _ surf.BucketProvider = (*Plugin)(nil) ) //go:embed config @@ -41,6 +44,7 @@ var sitemapXSL []byte type Plugin struct { app *backpack.App tiers classes.Tiers + cache *sitemapCache } func (p *Plugin) ID() string { return "golem15.sitemap" } @@ -56,17 +60,27 @@ func (p *Plugin) Register(app *backpack.App) error { // Boot compiles the priority tier rules (classes.TiersConfigKey); a rule // with an unknown setting or a pattern that does not compile fails the boot. +// It also starts an empty sitemap cache for CacheConfigKey seconds; a +// negative value fails the boot. func (p *Plugin) Boot(app *backpack.App) error { p.app = app var tiers classes.Tiers var err error + seconds := 0 if app != nil { tiers, err = classes.LoadTiers(app.Config) + if app.Config != nil { + seconds = app.Config.Int(CacheConfigKey) + } } if err != nil { return fmt.Errorf("golem15.sitemap: %w", err) } + if seconds < 0 { + return fmt.Errorf("golem15.sitemap: %s must not be negative, got %d", CacheConfigKey, seconds) + } p.tiers = tiers + p.cache = newSitemapCache(time.Duration(seconds) * time.Second) return nil } diff --git a/routes.go b/routes.go index f9ac4eb..9e6316e 100644 --- a/routes.go +++ b/routes.go @@ -1,12 +1,17 @@ package sitemap import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "log/slog" "net" "net/http" "net/netip" "strings" + "sync" + "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" @@ -24,15 +29,113 @@ const ( cacheControl = "no-cache, private" ) +// BucketSitemap is the throttle bucket of GET /sitemap.xml. +const BucketSitemap = "sitemap-xml" + +// CacheConfigKey is the config key of the number of seconds a generated +// sitemap is served from memory; 0 turns the cache off. +const CacheConfigKey = "golem15.sitemap.cache_seconds" + +// cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the +// request's scheme and Host when app.url is not set) and stored definition +// and settings. A full cache is emptied. +const cacheMaxEntries = 16 + // Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application -// root, outside any throttle group and without the backend middleware: -// search engines and browsers fetch them anonymously. +// root without the backend middleware: search engines and browsers fetch +// them anonymously. /sitemap.xml resolves every registry item, so it is +// throttled per client IP (BucketSitemap) and its output is cached. func (p *Plugin) Routes(r pact.Router) error { - r.Get("/sitemap.xml", p.sitemapXML) + r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap) r.Get("/sitemap.xsl", p.sitemapXSL) return nil } +// Buckets registers sitemap-xml: 60 requests a minute per client IP (behind +// the configured trusted proxies). 429 bodies are surf's +// {"message":"Too Many Attempts."}. +func (p *Plugin) Buckets() map[string]surf.Bucket { + trusted := surf.TrustedProxies(nil) + if p.app != nil { + trusted = surf.TrustedProxies(p.app.Config) + } + return map[string]surf.Bucket{ + BucketSitemap: { + Name: BucketSitemap, + Max: 60, + Decay: time.Minute, + Key: func(r *http.Request) string { + return "sitemap-xml|" + surf.ClientIP(r, trusted) + }, + }, + } +} + +// sitemapCache holds generated sitemaps for the configured time. The key +// covers the root URL and the stored definition and settings, so a save of +// either is served at once; registry content (a new post) shows after the +// entry expires. gen lets one generation run at a time, so concurrent misses +// wait for it instead of each resolving every registry item. +type sitemapCache struct { + ttl time.Duration + gen sync.Mutex + mu sync.Mutex + entries map[string]cachedSitemap +} + +type cachedSitemap struct { + xml string + expires time.Time +} + +func newSitemapCache(ttl time.Duration) *sitemapCache { + return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}} +} + +func (c *sitemapCache) get(key string) (string, bool) { + if c == nil || c.ttl <= 0 || key == "" { + return "", false + } + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok || time.Now().After(e.expires) { + return "", false + } + return e.xml, true +} + +func (c *sitemapCache) put(key, xml string) { + if c == nil || c.ttl <= 0 || key == "" { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.entries) >= cacheMaxEntries { + clear(c.entries) + } + c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)} +} + +// cacheKey is a digest of the root URL, the definition's items and the +// settings; "" (no caching) when either does not encode. +func cacheKey(root string, def models.Definition, settings models.Settings) string { + items, err := json.Marshal(def.Items()) + if err != nil { + return "" + } + set, err := json.Marshal(settings) + if err != nil { + return "" + } + h := sha256.New() + for _, part := range [][]byte{[]byte(root), items, set} { + h.Write(part) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + // sitemapXSL serves the embedded stylesheet, the PHP file byte for byte. func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", contentTypeXSL) @@ -69,23 +172,48 @@ func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) { return } root := backendURL(app, r) - gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events} - if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { - gen.Resolver = reg - } - xml, err := gen.Generate(r.Context(), defs[0].Items()) + xml, err := p.generate(r, root, defs[0], settings) if err != nil { slog.Default().Error("golem15.sitemap: generate", "error", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } - xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL)) w.Header().Set("Content-Type", contentTypeXML) w.Header().Set("Cache-Control", cacheControl) w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(xml)) } +// generate returns the sitemap of def with the frontend_url rewrite +// applied, from the cache when it holds it. +func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) { + cache := p.cache + key := "" + if cache != nil && cache.ttl > 0 { + key = cacheKey(root, def, settings) + if xml, ok := cache.get(key); ok { + return xml, nil + } + cache.gen.Lock() + defer cache.gen.Unlock() + if xml, ok := cache.get(key); ok { + return xml, nil + } + } + app := p.app + gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events} + if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil { + gen.Resolver = reg + } + xml, err := gen.Generate(r.Context(), def.Items()) + if err != nil { + return "", err + } + xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL)) + cache.put(key, xml) + return xml, nil +} + // loadSettings reads the settings row; without one the PHP defaults apply. func loadSettings(db *gorm.DB) (models.Settings, error) { var rows []models.Settings diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index 79b9243..76a9216 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - _ "git.golem15.com/golem15/sm-sitemap-plugin" + sitemap "git.golem15.com/golem15/sm-sitemap-plugin" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/console" "git.golem15.com/golem15/sm-sitemap-plugin/models" @@ -106,7 +106,10 @@ type sitemapEnv struct { gdb *gorm.DB } -func bootSitemap(t *testing.T) *sitemapEnv { +// bootSitemap boots the application. The sitemap cache is off +// (golem15.sitemap.cache_seconds 0): the smoke tests change listeners and +// registry content between requests. overrides set further config keys. +func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv { t.Helper() sqlDB := freshDB(t) gdb, err := lagoon.Use(t.Context(), sqlDB) @@ -124,12 +127,19 @@ func bootSitemap(t *testing.T) *sitemapEnv { if err != nil { t.Fatal(err) } - for key, value := range map[string]any{ + settings := map[string]any{ "http.body_limits.default_bytes": 1048576, "http.body_limits.upload_bytes": 8388608, "mail.driver": "log", "mail.from": "noreply@example.test", - } { + sitemap.CacheConfigKey: 0, + } + for _, o := range overrides { + for key, value := range o { + settings[key] = value + } + } + for key, value := range settings { if err := cfg.Set(key, value); err != nil { t.Fatal(err) } @@ -174,6 +184,52 @@ func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem func text(s string) *string { return &s } +// TestSitemapCacheAndThrottle pins WR-05: with the cache on, a repeated GET +// is served from memory (a listener added later does not run), a save of +// the definition is served at once, and the 61st request of a client within +// a minute is a 429. +func TestSitemapCacheAndThrottle(t *testing.T) { + e := bootSitemap(t, map[string]any{sitemap.CacheConfigKey: 60}) + def := e.storeDefinition(t, []models.DefinitionItem{ + {Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")}, + {Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")}, + }) + requests := 0 + get := func() *httptest.ResponseRecorder { + requests++ + return e.get(t, "/sitemap.xml") + } + first := get() + if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), "https://api.example.test/") { + t.Fatalf("first GET: %d %s", first.Code, first.Body.String()) + } + e.app.Events.Listen("acme.skipall", func(_ context.Context, ev *classes.BeforeAddItem) error { + ev.Skip = true + return nil + }) + if again := get(); again.Code != http.StatusOK || again.Body.String() != first.Body.String() { + t.Fatalf("a repeated GET was generated again:\n%s", again.Body.String()) + } + def.SetItems(append(def.Items(), models.DefinitionItem{Type: models.ItemTypeURL, URL: text("/x")})) + if err := e.gdb.Save(&def).Error; err != nil { + t.Fatal(err) + } + if saved := get(); saved.Code != http.StatusOK || strings.Contains(saved.Body.String(), "") { + t.Fatalf("a saved definition was served from the cache:\n%s", saved.Body.String()) + } + for requests < 60 { + if rec := get(); rec.Code != http.StatusOK { + t.Fatalf("request %d: %d", requests, rec.Code) + } + } + if rec := get(); rec.Code != http.StatusTooManyRequests { + t.Fatalf("request %d: %d, want 429", requests, rec.Code) + } + if rec := e.get(t, "/sitemap.xsl"); rec.Code != http.StatusOK { + t.Fatalf("/sitemap.xsl is throttled with the sitemap: %d", rec.Code) + } +} + // TestSitemapTracer stores a definition with a url item and a registry // item, GETs /sitemap.xml and compares the body with the PHP golden. func TestSitemapTracer(t *testing.T) {