diff --git a/classes/indexnow.go b/classes/indexnow.go new file mode 100644 index 0000000..eb58955 --- /dev/null +++ b/classes/indexnow.go @@ -0,0 +1,294 @@ +package classes + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "git.golem15.com/golem15/sm-sitemap-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/conga" + "git.golem15.com/golem15/summercms/modules/fetchguard" + "gorm.io/gorm" +) + +// IndexNow (D-31, P33-14): after a rebuild the added, changed and removed +// addresses are posted to the shared IndexNow endpoint, which hands them to +// every participating engine. The endpoint is fixed, not configurable: the +// outbound client may reach that one host only (T-33-13). +const ( + IndexNowKind = "golem15.sitemap.indexnow" + IndexNowHost = "api.indexnow.org" + IndexNowEndpoint = "https://" + IndexNowHost + "/indexnow" + // MaxIndexNowURLs is the most addresses one request carries. + MaxIndexNowURLs = 10000 + // indexNowTimeout bounds one request. + indexNowTimeout = 10 * time.Second + // IndexNowAttempts is the attempt limit of the job: an answer of 429 or + // 5xx, or a network error, is retried by the queue up to this many times. + IndexNowAttempts = 5 +) + +// IndexNowArgs are the arguments of the IndexNow job: the absolute +// addresses of one request (at most MaxIndexNowURLs). +type IndexNowArgs struct { + URLs []string `json:"urls"` +} + +// Kind is IndexNowKind. +func (IndexNowArgs) Kind() string { return IndexNowKind } + +// keyPattern is the form of an IndexNow key: 8 to 128 letters, digits and +// hyphens (indexnow.org documentation). +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,128}$`) + +// ValidKey reports whether k may be an IndexNow key. +func ValidKey(k string) bool { return keyPattern.MatchString(k) } + +// GenerateKey returns a new IndexNow key: 32 lowercase hex characters from +// the system random source. +func GenerateKey() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("golem15.sitemap: indexnow key: %w", err) + } + return hex.EncodeToString(b[:]), nil +} + +// KeyFileName is the name of the key file of key: .txt, served at the +// site root by the SEO layer (from GET /_sitemap/indexnow/.txt); "" +// when key is not a ValidKey. +func KeyFileName(key string) string { + if !ValidKey(key) { + return "" + } + return key + ".txt" +} + +// ActiveKey is the key of settings while IndexNow is on, "" otherwise. +func ActiveKey(s models.Settings) string { + if !s.IndexNowEnabled { + return "" + } + if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) { + return k + } + return "" +} + +// Production reports whether app runs in the production environment; no +// IndexNow request is sent anywhere else (T-33-15). +func Production(app *backpack.App) bool { + return app != nil && app.Config != nil && app.Config.Environment() == "production" +} + +// hostOf is the lowercase host of an absolute http(s) URL, "" otherwise. +func hostOf(u string) string { + parsed, err := url.Parse(u) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return "" + } + return strings.ToLower(parsed.Hostname()) +} + +// SiteURLs keeps the addresses on the host of site, each once, in order: +// IndexNow refuses (422) a request with an address of another host. +func SiteURLs(site string, urls []string) []string { + host := hostOf(site) + if host == "" { + return nil + } + seen := map[string]bool{} + out := []string{} + for _, u := range urls { + if u == "" || seen[u] || hostOf(u) != host { + continue + } + seen[u] = true + out = append(out, u) + } + return out +} + +// NotifyIndexNow queues the IndexNow job for what a rebuild changed: its +// added, changed and removed addresses on the site's host, in requests of +// at most MaxIndexNowURLs. Nothing is queued outside production, while +// IndexNow is off or has no key, without a site address or when nothing +// changed. +func NotifyIndexNow(ctx context.Context, app *backpack.App, db *gorm.DB, res RebuildResult) error { + if !Production(app) { + return nil + } + settings, err := LoadSettings(db) + if err != nil { + return err + } + if ActiveKey(settings) == "" { + return nil + } + all := make([]string, 0, len(res.Added)+len(res.Changed)+len(res.Removed)) + all = append(append(append(all, res.Added...), res.Changed...), res.Removed...) + urls := SiteURLs(SiteURL(app, settings), all) + if len(urls) == 0 { + return nil + } + m, err := conga.From(app) + if err != nil { + return err + } + for start := 0; start < len(urls); start += MaxIndexNowURLs { + end := min(start+MaxIndexNowURLs, len(urls)) + if err := m.Enqueue(ctx, db, IndexNowArgs{URLs: urls[start:end]}, conga.EnqueueOpts{}); err != nil { + return err + } + } + return nil +} + +// indexNowBody is the JSON of an IndexNow request. keyLocation is not sent: +// the key file is served at the site root, which covers every address of +// the host (P33-14). +type indexNowBody struct { + Host string `json:"host"` + Key string `json:"key"` + URLList []string `json:"urlList"` +} + +// errIndexNowRetry is a retryable IndexNow answer. +type errIndexNowRetry struct{ status int } + +func (e errIndexNowRetry) Error() string { + return fmt.Sprintf("golem15.sitemap: indexnow answered %d; retrying", e.status) +} + +// SendIndexNow runs one IndexNow job: it posts {host, key, urlList} to +// IndexNowEndpoint through a fetchguard client that reaches that host only +// (HTTPS, no redirects) and records the answer (indexnow_last_at, _count, +// _status). 200 and 202 are success; 400, 403 and 422 are recorded and not +// retried; 429 and 5xx are recorded and returned as an error, so the queue +// retries the job. Outside production, while IndexNow is off or without a +// site address nothing is sent; addresses of another host never are. +func SendIndexNow(ctx context.Context, app *backpack.App, args IndexNowArgs) error { + if !Production(app) { + pluginLog.Info("golem15.sitemap: indexnow skipped outside production", "urls", len(args.URLs)) + return nil + } + db, ok := app.Lookup[*gorm.DB]() + if !ok || db == nil { + return ErrNoDatabase + } + db = db.WithContext(ctx) + settings, err := LoadSettings(db) + if err != nil { + return err + } + key := ActiveKey(settings) + site := SiteURL(app, settings) + urls := SiteURLs(site, args.URLs) + if key == "" || len(urls) == 0 { + return nil + } + if len(urls) > MaxIndexNowURLs { + urls = urls[:MaxIndexNowURLs] + } + client, err := fetchguard.NewClient(fetchguard.Policy{ + Mode: fetchguard.AllowHostsMode, + AllowHosts: []string{IndexNowHost}, + Timeout: indexNowTimeout, + }, app.Config) + if err != nil { + return err + } + res, err := client.PostJSON(ctx, IndexNowEndpoint, nil, indexNowBody{Host: hostOf(site), Key: key, URLList: urls}) + if err != nil { + return err + } + status := res.StatusCode + if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_last_at = ?, indexnow_last_count = ?, indexnow_last_status = ? WHERE id = 1`, + time.Now().UTC(), len(urls), status).Error; err != nil { + return err + } + switch { + case status == http.StatusOK || status == http.StatusAccepted: + pluginLog.Info("golem15.sitemap: indexnow notified", "urls", len(urls), "status", status) + return nil + case status == http.StatusTooManyRequests || status >= 500: + pluginLog.Warn("golem15.sitemap: indexnow will retry", "urls", len(urls), "status", status) + return errIndexNowRetry{status: status} + default: + pluginLog.Warn("golem15.sitemap: indexnow refused", "urls", len(urls), "status", status) + return nil + } +} + +// EnsureKey gives settings row 1 an IndexNow key on save: while enabled, +// a new key is generated when none is stored or rotate is set (the +// indexnow_new_key checkbox), so the old key file answers 404 at once. It +// returns the stored key ("" while disabled without one). +func EnsureKey(db *gorm.DB, enabled, rotate bool) (string, error) { + settings, err := LoadSettings(db) + if err != nil { + return "", err + } + current := strings.TrimSpace(models.Text(settings.IndexNowKey)) + if !enabled || (ValidKey(current) && !rotate) { + return current, nil + } + key, err := GenerateKey() + if err != nil { + return "", err + } + if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_key = ? WHERE id = 1`, key).Error; err != nil { + return "", err + } + return key, nil +} + +// IndexNowRetryAt is when the queue next tries a pending IndexNow job +// (after a 429 or 5xx answer), nil when none is pending. +func IndexNowRetryAt(db *gorm.DB) *time.Time { + var at []time.Time + if err := db.Raw(`SELECT scheduled_at FROM river_job WHERE kind = ? AND state IN ('retryable', 'scheduled', 'available') ORDER BY scheduled_at ASC LIMIT 1`, IndexNowKind).Scan(&at).Error; err != nil || len(at) == 0 { + return nil + } + t := at[0].UTC() + return &t +} + +// IndexNowStatus is the state the S2 partial and the management API show: +// the key and its public URL (empty before a key exists), the last +// notification (nil before the first) and, after a 429, the next attempt. +type IndexNowStatus struct { + Enabled bool `json:"enabled"` + Key string `json:"key"` + KeyURL string `json:"key_url"` + LastAt *time.Time `json:"last_at"` + LastCount int `json:"last_count"` + LastStatus int `json:"last_status"` + RetryAt *time.Time `json:"retry_at"` +} + +// LoadIndexNowStatus reads the IndexNow state of settings. +func LoadIndexNowStatus(db *gorm.DB, app *backpack.App, s models.Settings) IndexNowStatus { + out := IndexNowStatus{Enabled: s.IndexNowEnabled, LastAt: s.IndexNowLastAt} + if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) { + out.Key = k + out.KeyURL = SiteURL(app, s) + "/" + KeyFileName(k) + } + if s.IndexNowLastCount != nil { + out.LastCount = *s.IndexNowLastCount + } + if s.IndexNowLastStatus != nil { + out.LastStatus = *s.IndexNowLastStatus + } + if out.LastStatus == http.StatusTooManyRequests { + out.RetryAt = IndexNowRetryAt(db) + } + return out +} diff --git a/classes/rebuild.go b/classes/rebuild.go index 601f764..f27f157 100644 --- a/classes/rebuild.go +++ b/classes/rebuild.go @@ -117,7 +117,9 @@ type collected struct { // numbers the files, in one transaction that holds the settings row, so // two rebuilds never interleave. A record that is gone or unpublished // leaves its rows in the table with a reason, as a hole; only a changed -// file size renumbers the files (D-30). +// file size renumbers the files (D-30). In production, with IndexNow on, +// the added, changed and removed addresses are queued for IndexNow after +// the commit (NotifyIndexNow). func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { var res RebuildResult if app == nil { @@ -154,7 +156,15 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { res, err = apply(tx, found, locked, site) return err }) - return res, err + if err != nil { + return res, err + } + // IndexNow hears of the change once it is committed (D-31); a queue + // error is logged and never fails the rebuild. + if err := NotifyIndexNow(ctx, app, db, res); err != nil { + pluginLog.Warn("golem15.sitemap: queue indexnow", "error", err) + } + return res, nil } // collect walks the content types page by page and adds the additional diff --git a/commands_documented_test.go b/commands_documented_test.go index 3889fc3..9fed217 100644 --- a/commands_documented_test.go +++ b/commands_documented_test.go @@ -24,8 +24,8 @@ func TestCommandsDocumented(t *testing.T) { } // TestScheduleAndJobs pins the nightly sitemap:rebuild at 03:00, which must -// name a registered command, and the rebuild job (kind -// golem15.sitemap.rebuild, three attempts). +// name a registered command, the rebuild job (kind golem15.sitemap.rebuild, +// three attempts) and the IndexNow job (golem15.sitemap.indexnow, five). func TestScheduleAndJobs(t *testing.T) { p := &Plugin{} entries := p.Schedule() @@ -40,11 +40,15 @@ func TestScheduleAndJobs(t *testing.T) { t.Fatal("the scheduled command is not registered") } jobs := p.Jobs() - if len(jobs) != 1 { + if len(jobs) != 2 { t.Fatalf("%d jobs", len(jobs)) } info, ok := conga.Describe(jobs[0]) if !ok || info.Kind != classes.RebuildKind || info.MaxAttempts != 3 { t.Fatalf("job = %+v %v", info, ok) } + info, ok = conga.Describe(jobs[1]) + if !ok || info.Kind != classes.IndexNowKind || info.MaxAttempts != 5 { + t.Fatalf("job = %+v %v", info, ok) + } } diff --git a/config/config.yaml b/config/config.yaml index 89018c6..b75bee0 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -18,3 +18,9 @@ lastmod_ignore: # rebuild_delay_seconds is how long after a record change the queued # rebuild runs; further changes in that time join the same rebuild. rebuild_delay_seconds: 60 +# +# IndexNow has no config key: the endpoint is fixed at +# https://api.indexnow.org/indexnow (the only host the plugin's outbound +# client may reach), the key is generated in the settings and stored in the +# database, and requests are sent only when the app environment is +# production. diff --git a/controllers/api/indexnow.go b/controllers/api/indexnow.go new file mode 100644 index 0000000..120caf5 --- /dev/null +++ b/controllers/api/indexnow.go @@ -0,0 +1,49 @@ +// Package api holds the sitemap plugin's HTTP handlers outside the admin: +// the IndexNow key file, the HTML sitemap API and the management API. +package api + +import ( + "net/http" + + "git.golem15.com/golem15/sm-sitemap-plugin/classes" + "git.golem15.com/golem15/summercms/modules/backpack" + "gorm.io/gorm" +) + +// KeyFilePattern is the {file} constraint of the key file route: an +// IndexNow key file name, .txt. +const KeyFilePattern = `[A-Za-z0-9-]{8,128}\.txt` + +// KeyFile serves GET /_sitemap/indexnow/{file}: the IndexNow key as +// text/plain when IndexNow is on and file is .txt; any other name, a +// replaced key or IndexNow off answers 404 (D-31). The SEO layer serves it +// at the site root. +func KeyFile(app func() *backpack.App) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + a := app() + if a == nil { + http.NotFound(w, r) + return + } + db, ok := a.Lookup[*gorm.DB]() + if !ok || db == nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + settings, err := classes.LoadSettings(db.WithContext(r.Context())) + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + key := classes.ActiveKey(settings) + if key == "" || r.PathValue("file") != classes.KeyFileName(key) { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache, private") + w.Header().Set("X-Robots-Tag", "noindex") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(key)) + } +} diff --git a/controllers/settings.go b/controllers/settings.go index f2b8601..876fec6 100644 --- a/controllers/settings.go +++ b/controllers/settings.go @@ -3,6 +3,9 @@ package controllers import ( "context" "errors" + "net/http" + "strconv" + "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" @@ -84,18 +87,44 @@ func (c settingsAdminController) FormBeforeCreate(ctx context.Context, model any return c.FormBeforeUpdate(ctx, model) } -// FormAfterUpdate dispatches a rebuild once the save commits (C26: saving -// the form rebuilds the files now). A queue error is logged and never fails -// the save. -func (c settingsAdminController) FormAfterUpdate(ctx context.Context, _ any) error { +// FormVirtualFields names indexnow_new_key, the "generate a new key on +// save" checkbox, which is no column. +func (settingsAdminController) FormVirtualFields() []string { + return []string{"indexnow_new_key"} +} + +// FormAfterUpdate generates the IndexNow key (on the first save with +// IndexNow on, or a new one when indexnow_new_key is checked) and +// dispatches a rebuild once the save commits (C26: saving the form rebuilds +// the files now). A queue error is logged and never fails the save. +func (c settingsAdminController) FormAfterUpdate(ctx context.Context, model any) error { app := appOf(c.app) tx, err := database(ctx, c.app) if err != nil || app == nil { return nil } + if s, ok := model.(*models.Settings); ok && s != nil { + values, _ := cabana.VirtualFieldsFromContext(ctx) + if _, err := classes.EnsureKey(tx, s.IndexNowEnabled, boolOf(values["indexnow_new_key"])); err != nil { + return err + } + } return classes.RequestRebuild(ctx, app, tx) } +// boolOf reads a submitted checkbox value. +func boolOf(v any) bool { + switch b := v.(type) { + case bool: + return b + case string: + return b == "1" || strings.EqualFold(b, "true") || strings.EqualFold(b, "on") + case float64: + return b != 0 + } + return false +} + // FormAfterCreate dispatches a rebuild like an update. func (c settingsAdminController) FormAfterCreate(ctx context.Context, model any) error { return c.FormAfterUpdate(ctx, model) @@ -135,10 +164,62 @@ func (c settingsAdminController) PartialData(ctx context.Context, name string, _ return c.filesData(ctx) case "news_sources": return c.newsSourcesData(ctx) + case "indexnow_status": + return c.indexNowData(ctx) } return nil, nil } +// indexNowView is the view model of the S2 partial indexnow_status. +type indexNowView struct { + Key string + Last string + Callout string + Severity string +} + +// indexNowData serves S2 (UI-SPEC C26): indexnow.key or indexnow.no_key, +// indexnow.last or indexnow.never, and the callout of the last answer: 403 +// err_key (warning), 422 err_host (warning), 429 err_rate with the next +// attempt (plain) while one is pending. +func (c settingsAdminController) indexNowData(ctx context.Context) (any, error) { + app := appOf(c.app) + db, err := database(ctx, c.app) + if err != nil { + return nil, err + } + settings, err := classes.LoadSettings(db) + if err != nil { + return nil, err + } + st := classes.LoadIndexNowStatus(db, app, settings) + tr := translator(c.app) + view := indexNowView{} + if st.Key == "" { + view.Key = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.no_key", nil) + } else { + view.Key = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.key", map[string]string{"key": st.Key, "url": st.KeyURL}) + } + if st.LastAt == nil { + view.Last = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.never", nil) + } else { + view.Last = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.last", map[string]string{ + "time": formatTime(st.LastAt), "count": strconv.Itoa(st.LastCount), "status": strconv.Itoa(st.LastStatus), + }) + } + switch st.LastStatus { + case http.StatusForbidden: + view.Callout, view.Severity = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_key", map[string]string{"url": st.KeyURL}), "warning" + case http.StatusUnprocessableEntity: + view.Callout, view.Severity = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_host", nil), "warning" + case http.StatusTooManyRequests: + if st.RetryAt != nil { + view.Callout = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_rate", map[string]string{"time": formatTime(st.RetryAt)}) + } + } + return view, nil +} + // sourceRow is one row of the news_sources partial: the value to enter and // what it lists. type sourceRow struct { @@ -337,4 +418,5 @@ var ( _ pact.FormBeforeUpdate = settingsAdminController{} _ pact.FormAfterCreate = settingsAdminController{} _ pact.FormAfterUpdate = settingsAdminController{} + _ pact.FormVirtualFields = settingsAdminController{} ) diff --git a/controllers/settings/_indexnow_status.htm b/controllers/settings/_indexnow_status.htm new file mode 100644 index 0000000..c1d9f57 --- /dev/null +++ b/controllers/settings/_indexnow_status.htm @@ -0,0 +1,15 @@ +
+

{{ .Data.Key }}

+

{{ .Data.Last }}

+
+{{- if .Data.Callout -}} +{{- if eq .Data.Severity "warning" -}} + +{{- else -}} +
+

{{ .Data.Callout }}

+
+{{- end -}} +{{- end -}} diff --git a/jobs.go b/jobs.go index 19750de..b61ca5f 100644 --- a/jobs.go +++ b/jobs.go @@ -8,13 +8,21 @@ import ( "git.golem15.com/golem15/summercms/modules/pact" ) -// Jobs registers the rebuild job (classes.RebuildKind) on conga. +// Jobs registers the rebuild job (classes.RebuildKind) and the IndexNow +// job (classes.IndexNowKind, five attempts: 429 and 5xx answers retry) on +// conga. func (p *Plugin) Jobs() []pact.Job { return []pact.Job{ conga.Job(p.rebuild, conga.MaxAttempts(3)), + conga.Job(p.indexNow, conga.MaxAttempts(classes.IndexNowAttempts)), } } +// indexNow sends one IndexNow request (classes.SendIndexNow). +func (p *Plugin) indexNow(ctx context.Context, args classes.IndexNowArgs) error { + return classes.SendIndexNow(ctx, p.app, args) +} + // rebuild runs one rebuild of the slot table. It ends the debounce window // first, so record events during the rebuild queue the next one. func (p *Plugin) rebuild(ctx context.Context, _ classes.RebuildArgs) error { diff --git a/models/settings.go b/models/settings.go index 5cfb453..5a6cfae 100644 --- a/models/settings.go +++ b/models/settings.go @@ -92,7 +92,7 @@ func (Settings) TableName() string { return "golem15_sitemap_settings" } // Fillable lists the columns the settings form edits. func (Settings) Fillable() []string { - return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources"} + return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled"} } // Rules is empty: the settings controller runs Validate before a save. diff --git a/models/settings/fields.yaml b/models/settings/fields.yaml index 1af62f1..6592787 100644 --- a/models/settings/fields.yaml +++ b/models/settings/fields.yaml @@ -99,3 +99,31 @@ fields: action: hide field: news_enabled condition: unchecked + + indexnow_enabled: + label: golem15.sitemap::lang.settings.indexnow_enabled + comment: golem15.sitemap::lang.settings.indexnow_enabled_comment + type: switch + default: false + span: full + tab: golem15.sitemap::lang.settings.tab_indexnow + + _indexnow_status: + type: partial + path: indexnow_status + span: full + tab: golem15.sitemap::lang.settings.tab_indexnow + + # No column: the controller lists it in FormVirtualFields and generates + # a new key on save when it is checked. + indexnow_new_key: + label: golem15.sitemap::lang.settings.indexnow_new_key + comment: golem15.sitemap::lang.settings.indexnow_new_key_comment + type: checkbox + default: false + span: full + tab: golem15.sitemap::lang.settings.tab_indexnow + trigger: + action: hide + field: indexnow_enabled + condition: unchecked diff --git a/routes.go b/routes.go index e5c047b..1ccfeb5 100644 --- a/routes.go +++ b/routes.go @@ -7,6 +7,7 @@ import ( "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" + apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/pact" @@ -28,17 +29,23 @@ const BucketSitemap = "sitemap-xml" // maps the root names to this route. const FilesPath = "/_sitemap/files/" -// Routes mounts GET /sitemap.xml (the index) and GET -// /_sitemap/files/{file} (one file) without the backend middleware: search -// engines fetch them anonymously. Both are served from the slot table, never -// generated per request, and throttled per client IP (BucketSitemap). +// Routes mounts GET /sitemap.xml (the index), GET /_sitemap/files/{file} +// (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file) +// without the backend middleware: search engines fetch them anonymously. +// The sitemaps are served from the slot table, never generated per +// request, and every route is throttled per client IP (BucketSitemap). func (p *Plugin) Routes(r pact.Router) error { r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap) r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap) r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`) + r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap) + r.Where("file", apicontrollers.KeyFilePattern) return nil } +// booted returns the booted application. +func (p *Plugin) booted() *backpack.App { return p.app } + // Buckets registers sitemap-xml: 60 requests a minute per client IP (behind // the configured trusted proxies). 429 bodies are surf's // {"message":"Too Many Attempts."}. diff --git a/sitemap_extras_test.go b/sitemap_extras_test.go index a84a391..00324d6 100644 --- a/sitemap_extras_test.go +++ b/sitemap_extras_test.go @@ -2,14 +2,20 @@ package sitemap_test import ( "context" + "encoding/json" "fmt" + "io" "net/http" "regexp" "strings" + "sync" "testing" "time" + "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/conga" + "git.golem15.com/golem15/summercms/modules/fetchguard" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" ) @@ -290,3 +296,243 @@ func TestNewsSitemap(t *testing.T) { t.Fatalf("News file after turning it off: %d", rec.Code) } } + +// job returns the sitemap's registered job of kind. +func (e *sitemapEnv) job(t *testing.T, kind string) pact.Job { + t.Helper() + for _, p := range e.plugins { + hj, ok := p.(pact.HasJobs) + if !ok || p.ID() != "golem15.sitemap" { + continue + } + for _, j := range hj.Jobs() { + if info, ok := conga.Describe(j); ok && info.Kind == kind { + return j + } + } + } + t.Fatalf("the sitemap registers no job %s", kind) + return nil +} + +// fakeIndexNow is the IndexNow engine of a test: it records each request +// and answers status. +type fakeIndexNow struct { + mu sync.Mutex + status int + requests []*http.Request + bodies []map[string]any +} + +func (f *fakeIndexNow) RoundTrip(r *http.Request) (*http.Response, error) { + raw, _ := io.ReadAll(r.Body) + body := map[string]any{} + _ = json.Unmarshal(raw, &body) + f.mu.Lock() + f.requests = append(f.requests, r) + f.bodies = append(f.bodies, body) + status := f.status + f.mu.Unlock() + return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil +} + +// queuedIndexNow decodes the URL lists of the queued IndexNow jobs, oldest +// first. +func (e *sitemapEnv) queuedIndexNow(t *testing.T) [][]string { + t.Helper() + var rows []string + if err := e.gdb.Raw(`SELECT args::text FROM river_job WHERE kind = ? ORDER BY id`, classes.IndexNowKind).Scan(&rows).Error; err != nil { + t.Fatal(err) + } + out := make([][]string, 0, len(rows)) + for _, raw := range rows { + var args classes.IndexNowArgs + if err := json.Unmarshal([]byte(raw), &args); err != nil { + t.Fatal(err) + } + out = append(out, args.URLs) + } + return out +} + +// indexNowKey reads the stored key. +func (e *sitemapEnv) indexNowKey(t *testing.T) string { + t.Helper() + var key *string + if err := e.gdb.Raw(`SELECT indexnow_key FROM golem15_sitemap_settings WHERE id = 1`).Scan(&key).Error; err != nil { + t.Fatal(err) + } + if key == nil { + return "" + } + return *key +} + +// TestIndexNow drives D-31's IndexNow in production: enabling and saving +// generates a 32-hex key whose file answers text/plain (another name 404); +// a rebuild with a new and a removed address queues one job with both; the +// job posts {host, key, urlList} to api.indexnow.org and records the +// answer (202); 403 and 422 are recorded without retry, 429 fails the job +// so it retries; S2 shows each state; an address of another host is never +// sent; a new key makes the old file 404. Outside production nothing is +// queued or sent. +func TestIndexNow(t *testing.T) { + e := bootJournal(t, map[string]any{envOverride: "production"}) + token := e.admin(t, permSitemap) + const base = "/golem15/sitemap/settings" + s2 := func() string { + t.Helper() + rec := e.call(t, http.MethodGet, base+"/partials/indexnow_status", token, nil) + if rec.Code != http.StatusOK { + t.Fatalf("S2: %d %s", rec.Code, rec.Body.String()) + } + return rec.Body.String() + } + if got := s2(); !strings.Contains(got, "A key is generated when you turn IndexNow on and save.") || !strings.Contains(got, "No notification sent yet.") { + t.Fatalf("S2 before a key:\n%s", got) + } + schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String() + for _, want := range []string{`"indexnow_enabled"`, `"indexnow_new_key"`, `"indexnow_status"`, "IndexNow"} { + if !strings.Contains(schema, want) { + t.Errorf("form schema lacks %s", want) + } + } + + // A rebuild before IndexNow is on queues nothing. + at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC) + contentStore.put(postType, post(1, "a", "en", at)) + e.rebuild(t) + if q := e.queuedIndexNow(t); len(q) != 0 { + t.Fatalf("queued while off: %v", q) + } + + if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true}); rec.Code != http.StatusOK { + t.Fatalf("enable IndexNow: %d %s", rec.Code, rec.Body.String()) + } + key := e.indexNowKey(t) + if !regexp.MustCompile(`^[0-9a-f]{32}$`).MatchString(key) { + t.Fatalf("key = %q", key) + } + rec := e.get(t, "/_sitemap/indexnow/"+key+".txt") + if rec.Code != http.StatusOK || rec.Body.String() != key || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") { + t.Fatalf("key file: %d %q %v", rec.Code, rec.Body.String(), rec.Header()) + } + for _, name := range []string{"0123456789abcdef0123456789abcdef.txt", "short.txt", key + ".xml"} { + if rec := e.get(t, "/_sitemap/indexnow/"+name); rec.Code != http.StatusNotFound { + t.Errorf("key file %s: %d", name, rec.Code) + } + } + if got := s2(); !strings.Contains(got, "Key: "+key+", served at https://api.example.test/"+key+".txt") { + t.Fatalf("S2 with a key:\n%s", got) + } + + // One new and one removed address: one job with both. + contentStore.put(postType, post(2, "b", "en", at)) + contentStore.remove(postType, 1) + e.rebuild(t) + q := e.queuedIndexNow(t) + if len(q) != 1 || strings.Join(q[0], " ") != "https://api.example.test/blog/b https://api.example.test/blog/a" { + t.Fatalf("queued = %v", q) + } + + engine := &fakeIndexNow{status: http.StatusAccepted} + ctx := fetchguard.WithTransport(t.Context(), engine) + job := e.job(t, classes.IndexNowKind) + args := classes.IndexNowArgs{URLs: append(q[0], "https://evil.test/x", "http://api.example.test.evil.test/y")} + if err := job.Work(ctx, args); err != nil { + t.Fatalf("IndexNow job: %v", err) + } + if len(engine.requests) != 1 { + t.Fatalf("%d requests", len(engine.requests)) + } + req, body := engine.requests[0], engine.bodies[0] + if req.Method != http.MethodPost || req.URL.String() != "https://api.indexnow.org/indexnow" || !strings.HasPrefix(req.Header.Get("Content-Type"), "application/json") { + t.Fatalf("request = %s %s %v", req.Method, req.URL, req.Header) + } + list, _ := body["urlList"].([]any) + if body["host"] != "api.example.test" || body["key"] != key || len(list) != 2 || body["keyLocation"] != nil { + t.Fatalf("body = %v", body) + } + var last struct { + Status int `gorm:"column:indexnow_last_status"` + Count int `gorm:"column:indexnow_last_count"` + } + e.gdb.Raw(`SELECT indexnow_last_status, indexnow_last_count FROM golem15_sitemap_settings WHERE id = 1`).Scan(&last) + if last.Status != 202 || last.Count != 2 { + t.Fatalf("recorded = %+v", last) + } + if got := s2(); !strings.Contains(got, ", 2 addresses, answer 202.") || strings.Contains(got, "summer-callout--warning") { + t.Fatalf("S2 after 202:\n%s", got) + } + + for _, c := range []struct { + status int + retry bool + want string + }{ + {http.StatusForbidden, false, "The engines could not read the key file at https://api.example.test/" + key + ".txt."}, + {http.StatusUnprocessableEntity, false, "The engines refused the addresses because they do not match the site address."}, + {http.StatusBadRequest, false, "answer 400."}, + {http.StatusTooManyRequests, true, "Too many notifications; the next attempt is at "}, + } { + engine.status = c.status + err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]}) + if (err != nil) != c.retry { + t.Errorf("status %d: job error %v, want retry %v", c.status, err, c.retry) + } + if got := s2(); !strings.Contains(got, c.want) { + t.Errorf("S2 after %d lacks %q:\n%s", c.status, c.want, got) + } + } + + // A new key: the old file answers 404 at once. + if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true, "indexnow_new_key": true}); rec.Code != http.StatusOK { + t.Fatalf("new key: %d %s", rec.Code, rec.Body.String()) + } + fresh := e.indexNowKey(t) + if fresh == key || len(fresh) != 32 { + t.Fatalf("new key = %q (old %q)", fresh, key) + } + if rec := e.get(t, "/_sitemap/indexnow/"+key+".txt"); rec.Code != http.StatusNotFound { + t.Fatalf("old key file: %d", rec.Code) + } + if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusOK { + t.Fatalf("new key file: %d", rec.Code) + } + // A save without the checkbox keeps the key; turning IndexNow off + // takes the file down. + e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true}) + if got := e.indexNowKey(t); got != fresh { + t.Fatalf("a plain save changed the key to %q", got) + } + e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": false}) + if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusNotFound { + t.Fatalf("key file while off: %d", rec.Code) + } + before := len(engine.requests) + if err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]}); err != nil || len(engine.requests) != before { + t.Fatalf("job while off: %v, %d requests", err, len(engine.requests)-before) + } +} + +// TestIndexNowStaging pins T-33-15: outside production a rebuild queues no +// IndexNow job and the job sends nothing. +func TestIndexNowStaging(t *testing.T) { + e := bootJournal(t) + e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`) + contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC))) + e.rebuild(t) + if q := e.queuedIndexNow(t); len(q) != 0 { + t.Fatalf("queued outside production: %v", q) + } + engine := &fakeIndexNow{status: http.StatusOK} + if err := e.job(t, classes.IndexNowKind).Work(fetchguard.WithTransport(t.Context(), engine), classes.IndexNowArgs{URLs: []string{"https://api.example.test/blog/a"}}); err != nil { + t.Fatal(err) + } + if len(engine.requests) != 0 { + t.Fatalf("%d requests outside production", len(engine.requests)) + } + if rec := e.get(t, "/_sitemap/indexnow/0123456789abcdef0123456789abcdef.txt"); rec.Code != http.StatusOK { + t.Fatalf("key file outside production: %d", rec.Code) + } +} diff --git a/sitemap_smoke_test.go b/sitemap_smoke_test.go index 99075e0..3591d59 100644 --- a/sitemap_smoke_test.go +++ b/sitemap_smoke_test.go @@ -35,8 +35,13 @@ type sitemapEnv struct { plugins []party.Plugin } +// envOverride is the override key that sets the app environment +// (SUMMER_ENV) instead of a config key; development by default. +const envOverride = "__env" + // bootPlugins boots the application with the plugins ids on a fresh -// database; overrides set further config keys. +// database; overrides set further config keys (envOverride the +// environment). func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv { t.Helper() sqlDB := freshDB(t) @@ -48,9 +53,15 @@ func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitem if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: sitemap-smoke\nlocale: en\nfallback_locale: en\nurl: https://api.example.test\n"), 0o644); err != nil { t.Fatal(err) } + env := "development" + for _, o := range overrides { + if v, ok := o[envOverride].(string); ok { + env = v + } + } cfg, err := compass.Open(compass.Options{ Dir: dir, - Environ: []string{"SUMMER_ENV=development", "SUMMER_ADMIN__JWT__SECRET=sitemap-smoke-test-only-hs256-secret-key"}, + Environ: []string{"SUMMER_ENV=" + env, "SUMMER_ADMIN__JWT__SECRET=sitemap-smoke-test-only-hs256-secret-key"}, }) if err != nil { t.Fatal(err) @@ -63,7 +74,9 @@ func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitem } for _, o := range overrides { for key, value := range o { - settings[key] = value + if key != envOverride { + settings[key] = value + } } } for key, value := range settings {