diff --git a/README.md b/README.md
index a462a29..5e1688b 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ The output keeps the PHP plugin's bytes: element order, escaping, whitespace, da
## Features
-- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, outside any throttle group and without the backend middleware.
+- **Routes.** Anonymous `GET /sitemap.xml` and `GET /sitemap.xsl` at the application root, without the backend middleware. `/sitemap.xml` is throttled per client IP and its output is cached for a short time.
- **One definition.** Table `golem15_sitemap_definitions` keeps the PHP columns (`id`, `theme` nullable and indexed, `data` text, timestamps). The `theme` column holds the fixed key `default` (`models.DefinitionTheme`); `/sitemap.xml` serves that row. `models.Definition.Fillable` is exactly `theme` and `data`, the PHP `$fillable` with `$guarded = ['*']`.
- **Items.** An item is a URL (`type: url`) or a registry menu item type. Winter.Pages static pages, all static pages and the CMS page field are not ported. Stored item JSON in the PHP shape decodes tolerantly: numbers, strings and booleans are coerced, and unknown keys (`cmsPage`, `code`, `exists`, `fillable` and any other) are ignored. A save writes the PHP `toArray` key set (`changefreq`, `priority`, `nesting`, `type`, `url`, `reference`, `cmsPage` as null, `urlPattern`).
- **Settings.** A singleton row (`golem15_sitemap_settings`, id 1) holds the frontend URL, the frontend default locale, the updates manifest path and the priority tiers, with the PHP defaults.
@@ -48,6 +48,11 @@ The plugin's own `go.mod` and `go.work` replace the framework with `../summercms
- `200` with an empty body when the definition has no items;
- otherwise the sitemap, `Content-Type: application/xml`, `Cache-Control: no-cache, private`.
+Generating the sitemap resolves every registry item (all published posts, all categories, their translations), so the route is guarded:
+
+- **Throttle.** The surf bucket `sitemap-xml` (`sitemap.BucketSitemap`) allows 60 requests a minute per client IP, read behind the host's trusted proxies. The 429 body is surf's `{"message":"Too Many Attempts."}`. `/sitemap.xsl` is not throttled.
+- **Cache.** The generated document is kept in memory for `golem15.sitemap.cache_seconds` seconds (`sitemap.CacheConfigKey`, default `60`; `0` turns it off, a negative value fails the boot). The cache key covers the root URL and the stored definition and settings, so a save of either is served at once; a change of registry content (a new or unpublished post) shows once the entry expires. One generation runs at a time; concurrent requests for a missing entry wait for it.
+
The document is the PHP `DOMDocument::saveXML()` format: the XML declaration and the `xml-stylesheet` processing instruction each followed by a newline, everything else on one line ending with `` and a newline. Items that all resolve to nothing produce a self-closing ``. Text is escaped like libxml (`&`, `<`, `>` and carriage return; attributes also `"`, tab and newline); single quotes and UTF-8 stay literal.
Each URL is written in the PHP order (`classes.Generator`):
@@ -169,9 +174,12 @@ The WinterCMS forms are adapted to the admin's YAML:
| Identifier | Description |
|------------|-------------|
| `sitemap.Plugin` | Compiled plugin; `ID` is `golem15.sitemap`. |
-| `sitemap.Plugin.Routes` | `GET /sitemap.xml` and `GET /sitemap.xsl`. |
-| `sitemap.Plugin.Boot` | Compiles the tier rules; a bad rule fails the boot. |
-| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (tier rules). |
+| `sitemap.Plugin.Routes` | `GET /sitemap.xml` (throttled, cached) and `GET /sitemap.xsl`. |
+| `sitemap.Plugin.Buckets` | Registers the `sitemap-xml` throttle bucket: 60 requests a minute per client IP. |
+| `sitemap.BucketSitemap` | `sitemap-xml`, the throttle bucket of `/sitemap.xml`. |
+| `sitemap.CacheConfigKey` | `golem15.sitemap.cache_seconds`, the seconds a generated sitemap is cached. |
+| `sitemap.Plugin.Boot` | Compiles the tier rules and starts an empty sitemap cache; a bad rule or a negative `cache_seconds` fails the boot. |
+| `sitemap.Plugin.ConfigFS` | The default `config/config.yaml` (cache time and tier rules). |
| `sitemap.Plugin.Commands` | Registers `sitemap:import` and `sitemap:import-winter`. |
| `sitemap.Plugin.AdminControllers` | The definitions controller. |
| `sitemap.Plugin.Settings` | The Sitemap link and the Sitemap settings form. |
diff --git a/config/config.yaml b/config/config.yaml
index 354b890..6d346a7 100644
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -1,5 +1,11 @@
# golem15.sitemap defaults, read as golem15.sitemap..
#
+# cache_seconds is how long a generated /sitemap.xml is served from memory.
+# A save of the definition or the settings shows at once; registry content
+# (a new or changed post) shows once the entry expires. 0 turns the cache
+# off.
+cache_seconds: 60
+#
# The priority tier rules assign a URL the value of one of the five
# priority settings when Enable priority tiers is on. A URL's path, without
# trailing slashes ("" for the homepage), is matched against the rules in
diff --git a/plugin.go b/plugin.go
index fc942bd..48eb548 100644
--- a/plugin.go
+++ b/plugin.go
@@ -8,6 +8,7 @@ import (
"embed"
"fmt"
"io/fs"
+ "time"
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
"git.golem15.com/golem15/sm-sitemap-plugin/console"
@@ -17,16 +18,18 @@ import (
"git.golem15.com/golem15/summercms/modules/bonfire"
"git.golem15.com/golem15/summercms/modules/pact"
"git.golem15.com/golem15/summercms/modules/party"
+ "git.golem15.com/golem15/summercms/modules/surf"
"github.com/go-gormigrate/gormigrate/v2"
)
var (
- _ party.Plugin = (*Plugin)(nil)
- _ pact.HasConfig = (*Plugin)(nil)
- _ pact.HasMigrations = (*Plugin)(nil)
- _ pact.HasModels = (*Plugin)(nil)
- _ pact.HasRoutes = (*Plugin)(nil)
- _ pact.HasCommands = (*Plugin)(nil)
+ _ party.Plugin = (*Plugin)(nil)
+ _ pact.HasConfig = (*Plugin)(nil)
+ _ pact.HasMigrations = (*Plugin)(nil)
+ _ pact.HasModels = (*Plugin)(nil)
+ _ pact.HasRoutes = (*Plugin)(nil)
+ _ pact.HasCommands = (*Plugin)(nil)
+ _ surf.BucketProvider = (*Plugin)(nil)
)
//go:embed config
@@ -41,6 +44,7 @@ var sitemapXSL []byte
type Plugin struct {
app *backpack.App
tiers classes.Tiers
+ cache *sitemapCache
}
func (p *Plugin) ID() string { return "golem15.sitemap" }
@@ -56,17 +60,27 @@ func (p *Plugin) Register(app *backpack.App) error {
// Boot compiles the priority tier rules (classes.TiersConfigKey); a rule
// with an unknown setting or a pattern that does not compile fails the boot.
+// It also starts an empty sitemap cache for CacheConfigKey seconds; a
+// negative value fails the boot.
func (p *Plugin) Boot(app *backpack.App) error {
p.app = app
var tiers classes.Tiers
var err error
+ seconds := 0
if app != nil {
tiers, err = classes.LoadTiers(app.Config)
+ if app.Config != nil {
+ seconds = app.Config.Int(CacheConfigKey)
+ }
}
if err != nil {
return fmt.Errorf("golem15.sitemap: %w", err)
}
+ if seconds < 0 {
+ return fmt.Errorf("golem15.sitemap: %s must not be negative, got %d", CacheConfigKey, seconds)
+ }
p.tiers = tiers
+ p.cache = newSitemapCache(time.Duration(seconds) * time.Second)
return nil
}
diff --git a/routes.go b/routes.go
index f9ac4eb..9e6316e 100644
--- a/routes.go
+++ b/routes.go
@@ -1,12 +1,17 @@
package sitemap
import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
"errors"
"log/slog"
"net"
"net/http"
"net/netip"
"strings"
+ "sync"
+ "time"
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
@@ -24,15 +29,113 @@ const (
cacheControl = "no-cache, private"
)
+// BucketSitemap is the throttle bucket of GET /sitemap.xml.
+const BucketSitemap = "sitemap-xml"
+
+// CacheConfigKey is the config key of the number of seconds a generated
+// sitemap is served from memory; 0 turns the cache off.
+const CacheConfigKey = "golem15.sitemap.cache_seconds"
+
+// cacheMaxEntries bounds the cache: one entry per root URL (app.url, or the
+// request's scheme and Host when app.url is not set) and stored definition
+// and settings. A full cache is emptied.
+const cacheMaxEntries = 16
+
// Routes mounts GET /sitemap.xml and GET /sitemap.xsl at the application
-// root, outside any throttle group and without the backend middleware:
-// search engines and browsers fetch them anonymously.
+// root without the backend middleware: search engines and browsers fetch
+// them anonymously. /sitemap.xml resolves every registry item, so it is
+// throttled per client IP (BucketSitemap) and its output is cached.
func (p *Plugin) Routes(r pact.Router) error {
- r.Get("/sitemap.xml", p.sitemapXML)
+ r.Get("/sitemap.xml", p.sitemapXML, "throttle:"+BucketSitemap)
r.Get("/sitemap.xsl", p.sitemapXSL)
return nil
}
+// Buckets registers sitemap-xml: 60 requests a minute per client IP (behind
+// the configured trusted proxies). 429 bodies are surf's
+// {"message":"Too Many Attempts."}.
+func (p *Plugin) Buckets() map[string]surf.Bucket {
+ trusted := surf.TrustedProxies(nil)
+ if p.app != nil {
+ trusted = surf.TrustedProxies(p.app.Config)
+ }
+ return map[string]surf.Bucket{
+ BucketSitemap: {
+ Name: BucketSitemap,
+ Max: 60,
+ Decay: time.Minute,
+ Key: func(r *http.Request) string {
+ return "sitemap-xml|" + surf.ClientIP(r, trusted)
+ },
+ },
+ }
+}
+
+// sitemapCache holds generated sitemaps for the configured time. The key
+// covers the root URL and the stored definition and settings, so a save of
+// either is served at once; registry content (a new post) shows after the
+// entry expires. gen lets one generation run at a time, so concurrent misses
+// wait for it instead of each resolving every registry item.
+type sitemapCache struct {
+ ttl time.Duration
+ gen sync.Mutex
+ mu sync.Mutex
+ entries map[string]cachedSitemap
+}
+
+type cachedSitemap struct {
+ xml string
+ expires time.Time
+}
+
+func newSitemapCache(ttl time.Duration) *sitemapCache {
+ return &sitemapCache{ttl: ttl, entries: map[string]cachedSitemap{}}
+}
+
+func (c *sitemapCache) get(key string) (string, bool) {
+ if c == nil || c.ttl <= 0 || key == "" {
+ return "", false
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ e, ok := c.entries[key]
+ if !ok || time.Now().After(e.expires) {
+ return "", false
+ }
+ return e.xml, true
+}
+
+func (c *sitemapCache) put(key, xml string) {
+ if c == nil || c.ttl <= 0 || key == "" {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if len(c.entries) >= cacheMaxEntries {
+ clear(c.entries)
+ }
+ c.entries[key] = cachedSitemap{xml: xml, expires: time.Now().Add(c.ttl)}
+}
+
+// cacheKey is a digest of the root URL, the definition's items and the
+// settings; "" (no caching) when either does not encode.
+func cacheKey(root string, def models.Definition, settings models.Settings) string {
+ items, err := json.Marshal(def.Items())
+ if err != nil {
+ return ""
+ }
+ set, err := json.Marshal(settings)
+ if err != nil {
+ return ""
+ }
+ h := sha256.New()
+ for _, part := range [][]byte{[]byte(root), items, set} {
+ h.Write(part)
+ h.Write([]byte{0})
+ }
+ return hex.EncodeToString(h.Sum(nil))
+}
+
// sitemapXSL serves the embedded stylesheet, the PHP file byte for byte.
func (p *Plugin) sitemapXSL(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", contentTypeXSL)
@@ -69,23 +172,48 @@ func (p *Plugin) sitemapXML(w http.ResponseWriter, r *http.Request) {
return
}
root := backendURL(app, r)
- gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events}
- if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
- gen.Resolver = reg
- }
- xml, err := gen.Generate(r.Context(), defs[0].Items())
+ xml, err := p.generate(r, root, defs[0], settings)
if err != nil {
slog.Default().Error("golem15.sitemap: generate", "error", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
- xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL))
w.Header().Set("Content-Type", contentTypeXML)
w.Header().Set("Cache-Control", cacheControl)
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(xml))
}
+// generate returns the sitemap of def with the frontend_url rewrite
+// applied, from the cache when it holds it.
+func (p *Plugin) generate(r *http.Request, root string, def models.Definition, settings models.Settings) (string, error) {
+ cache := p.cache
+ key := ""
+ if cache != nil && cache.ttl > 0 {
+ key = cacheKey(root, def, settings)
+ if xml, ok := cache.get(key); ok {
+ return xml, nil
+ }
+ cache.gen.Lock()
+ defer cache.gen.Unlock()
+ if xml, ok := cache.get(key); ok {
+ return xml, nil
+ }
+ }
+ app := p.app
+ gen := &classes.Generator{Root: root, Settings: settings, Tiers: p.tiers, Events: app.Events}
+ if reg, ok := app.Lookup[*signpost.Registry](); ok && reg != nil {
+ gen.Resolver = reg
+ }
+ xml, err := gen.Generate(r.Context(), def.Items())
+ if err != nil {
+ return "", err
+ }
+ xml = classes.Rewrite(xml, classes.BackendURL(root), models.Text(settings.FrontendURL))
+ cache.put(key, xml)
+ return xml, nil
+}
+
// loadSettings reads the settings row; without one the PHP defaults apply.
func loadSettings(db *gorm.DB) (models.Settings, error) {
var rows []models.Settings
diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go
index 79b9243..76a9216 100644
--- a/sitemap_smoke_test.go
+++ b/sitemap_smoke_test.go
@@ -15,7 +15,7 @@ import (
"testing"
"time"
- _ "git.golem15.com/golem15/sm-sitemap-plugin"
+ sitemap "git.golem15.com/golem15/sm-sitemap-plugin"
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
"git.golem15.com/golem15/sm-sitemap-plugin/console"
"git.golem15.com/golem15/sm-sitemap-plugin/models"
@@ -106,7 +106,10 @@ type sitemapEnv struct {
gdb *gorm.DB
}
-func bootSitemap(t *testing.T) *sitemapEnv {
+// bootSitemap boots the application. The sitemap cache is off
+// (golem15.sitemap.cache_seconds 0): the smoke tests change listeners and
+// registry content between requests. overrides set further config keys.
+func bootSitemap(t *testing.T, overrides ...map[string]any) *sitemapEnv {
t.Helper()
sqlDB := freshDB(t)
gdb, err := lagoon.Use(t.Context(), sqlDB)
@@ -124,12 +127,19 @@ func bootSitemap(t *testing.T) *sitemapEnv {
if err != nil {
t.Fatal(err)
}
- for key, value := range map[string]any{
+ settings := map[string]any{
"http.body_limits.default_bytes": 1048576,
"http.body_limits.upload_bytes": 8388608,
"mail.driver": "log",
"mail.from": "noreply@example.test",
- } {
+ sitemap.CacheConfigKey: 0,
+ }
+ for _, o := range overrides {
+ for key, value := range o {
+ settings[key] = value
+ }
+ }
+ for key, value := range settings {
if err := cfg.Set(key, value); err != nil {
t.Fatal(err)
}
@@ -174,6 +184,52 @@ func (e *sitemapEnv) storeDefinition(t *testing.T, items []models.DefinitionItem
func text(s string) *string { return &s }
+// TestSitemapCacheAndThrottle pins WR-05: with the cache on, a repeated GET
+// is served from memory (a listener added later does not run), a save of
+// the definition is served at once, and the 61st request of a client within
+// a minute is a 429.
+func TestSitemapCacheAndThrottle(t *testing.T) {
+ e := bootSitemap(t, map[string]any{sitemap.CacheConfigKey: 60})
+ def := e.storeDefinition(t, []models.DefinitionItem{
+ {Type: models.ItemTypeURL, URL: text("/"), Changefreq: text("always"), Priority: text("1.0")},
+ {Type: "acme-post", Reference: text("7"), URLPattern: text(""), Nesting: true, Changefreq: text("weekly"), Priority: text("0.5")},
+ })
+ requests := 0
+ get := func() *httptest.ResponseRecorder {
+ requests++
+ return e.get(t, "/sitemap.xml")
+ }
+ first := get()
+ if first.Code != http.StatusOK || !strings.Contains(first.Body.String(), "https://api.example.test/") {
+ t.Fatalf("first GET: %d %s", first.Code, first.Body.String())
+ }
+ e.app.Events.Listen("acme.skipall", func(_ context.Context, ev *classes.BeforeAddItem) error {
+ ev.Skip = true
+ return nil
+ })
+ if again := get(); again.Code != http.StatusOK || again.Body.String() != first.Body.String() {
+ t.Fatalf("a repeated GET was generated again:\n%s", again.Body.String())
+ }
+ def.SetItems(append(def.Items(), models.DefinitionItem{Type: models.ItemTypeURL, URL: text("/x")}))
+ if err := e.gdb.Save(&def).Error; err != nil {
+ t.Fatal(err)
+ }
+ if saved := get(); saved.Code != http.StatusOK || strings.Contains(saved.Body.String(), "") {
+ t.Fatalf("a saved definition was served from the cache:\n%s", saved.Body.String())
+ }
+ for requests < 60 {
+ if rec := get(); rec.Code != http.StatusOK {
+ t.Fatalf("request %d: %d", requests, rec.Code)
+ }
+ }
+ if rec := get(); rec.Code != http.StatusTooManyRequests {
+ t.Fatalf("request %d: %d, want 429", requests, rec.Code)
+ }
+ if rec := e.get(t, "/sitemap.xsl"); rec.Code != http.StatusOK {
+ t.Fatalf("/sitemap.xsl is throttled with the sitemap: %d", rec.Code)
+ }
+}
+
// TestSitemapTracer stores a definition with a url item and a registry
// item, GETs /sitemap.xml and compares the body with the PHP golden.
func TestSitemapTracer(t *testing.T) {