feat: IndexNow key, key file and pings

- saving with IndexNow on generates a 32-hex key (crypto/rand); indexnow_new_key generates a new one, so the old key file answers 404 - GET /_sitemap/indexnow/{file} serves <key>.txt as text/plain while IndexNow is on; the SEO layer maps it to the site root - after each rebuild in production the added, changed and removed addresses on the site host are queued as golem15.sitemap.indexnow jobs (at most 10 000 per request, five attempts) - the job posts {host, key, urlList} to api.indexnow.org through a fetchguard client limited to that host and records the answer; 429 and 5xx retry, 400, 403 and 422 do not - the IndexNow tab shows the S2 status partial: key or no key, last notification or never, and the 403, 422 and 429 callouts
2026-11-04 12:26:47 +01:00
parent a10ce2d9de
commit e02d0b173f
13 changed files with 780 additions and 18 deletions
--- a/classes/indexnow.go
+++ b/classes/indexnow.go
@@ -0,0 +1,294 @@
+package classes
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/conga"
+	"git.golem15.com/golem15/summercms/modules/fetchguard"
+	"gorm.io/gorm"
+)
+
+// IndexNow (D-31, P33-14): after a rebuild the added, changed and removed
+// addresses are posted to the shared IndexNow endpoint, which hands them to
+// every participating engine. The endpoint is fixed, not configurable: the
+// outbound client may reach that one host only (T-33-13).
+const (
+	IndexNowKind     = "golem15.sitemap.indexnow"
+	IndexNowHost     = "api.indexnow.org"
+	IndexNowEndpoint = "https://" + IndexNowHost + "/indexnow"
+	// MaxIndexNowURLs is the most addresses one request carries.
+	MaxIndexNowURLs = 10000
+	// indexNowTimeout bounds one request.
+	indexNowTimeout = 10 * time.Second
+	// IndexNowAttempts is the attempt limit of the job: an answer of 429 or
+	// 5xx, or a network error, is retried by the queue up to this many times.
+	IndexNowAttempts = 5
+)
+
+// IndexNowArgs are the arguments of the IndexNow job: the absolute
+// addresses of one request (at most MaxIndexNowURLs).
+type IndexNowArgs struct {
+	URLs []string `json:"urls"`
+}
+
+// Kind is IndexNowKind.
+func (IndexNowArgs) Kind() string { return IndexNowKind }
+
+// keyPattern is the form of an IndexNow key: 8 to 128 letters, digits and
+// hyphens (indexnow.org documentation).
+var keyPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,128}$`)
+
+// ValidKey reports whether k may be an IndexNow key.
+func ValidKey(k string) bool { return keyPattern.MatchString(k) }
+
+// GenerateKey returns a new IndexNow key: 32 lowercase hex characters from
+// the system random source.
+func GenerateKey() (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("golem15.sitemap: indexnow key: %w", err)
+	}
+	return hex.EncodeToString(b[:]), nil
+}
+
+// KeyFileName is the name of the key file of key: <key>.txt, served at the
+// site root by the SEO layer (from GET /_sitemap/indexnow/<key>.txt); ""
+// when key is not a ValidKey.
+func KeyFileName(key string) string {
+	if !ValidKey(key) {
+		return ""
+	}
+	return key + ".txt"
+}
+
+// ActiveKey is the key of settings while IndexNow is on, "" otherwise.
+func ActiveKey(s models.Settings) string {
+	if !s.IndexNowEnabled {
+		return ""
+	}
+	if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
+		return k
+	}
+	return ""
+}
+
+// Production reports whether app runs in the production environment; no
+// IndexNow request is sent anywhere else (T-33-15).
+func Production(app *backpack.App) bool {
+	return app != nil && app.Config != nil && app.Config.Environment() == "production"
+}
+
+// hostOf is the lowercase host of an absolute http(s) URL, "" otherwise.
+func hostOf(u string) string {
+	parsed, err := url.Parse(u)
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		return ""
+	}
+	return strings.ToLower(parsed.Hostname())
+}
+
+// SiteURLs keeps the addresses on the host of site, each once, in order:
+// IndexNow refuses (422) a request with an address of another host.
+func SiteURLs(site string, urls []string) []string {
+	host := hostOf(site)
+	if host == "" {
+		return nil
+	}
+	seen := map[string]bool{}
+	out := []string{}
+	for _, u := range urls {
+		if u == "" || seen[u] || hostOf(u) != host {
+			continue
+		}
+		seen[u] = true
+		out = append(out, u)
+	}
+	return out
+}
+
+// NotifyIndexNow queues the IndexNow job for what a rebuild changed: its
+// added, changed and removed addresses on the site's host, in requests of
+// at most MaxIndexNowURLs. Nothing is queued outside production, while
+// IndexNow is off or has no key, without a site address or when nothing
+// changed.
+func NotifyIndexNow(ctx context.Context, app *backpack.App, db *gorm.DB, res RebuildResult) error {
+	if !Production(app) {
+		return nil
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return err
+	}
+	if ActiveKey(settings) == "" {
+		return nil
+	}
+	all := make([]string, 0, len(res.Added)+len(res.Changed)+len(res.Removed))
+	all = append(append(append(all, res.Added...), res.Changed...), res.Removed...)
+	urls := SiteURLs(SiteURL(app, settings), all)
+	if len(urls) == 0 {
+		return nil
+	}
+	m, err := conga.From(app)
+	if err != nil {
+		return err
+	}
+	for start := 0; start < len(urls); start += MaxIndexNowURLs {
+		end := min(start+MaxIndexNowURLs, len(urls))
+		if err := m.Enqueue(ctx, db, IndexNowArgs{URLs: urls[start:end]}, conga.EnqueueOpts{}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// indexNowBody is the JSON of an IndexNow request. keyLocation is not sent:
+// the key file is served at the site root, which covers every address of
+// the host (P33-14).
+type indexNowBody struct {
+	Host    string   `json:"host"`
+	Key     string   `json:"key"`
+	URLList []string `json:"urlList"`
+}
+
+// errIndexNowRetry is a retryable IndexNow answer.
+type errIndexNowRetry struct{ status int }
+
+func (e errIndexNowRetry) Error() string {
+	return fmt.Sprintf("golem15.sitemap: indexnow answered %d; retrying", e.status)
+}
+
+// SendIndexNow runs one IndexNow job: it posts {host, key, urlList} to
+// IndexNowEndpoint through a fetchguard client that reaches that host only
+// (HTTPS, no redirects) and records the answer (indexnow_last_at, _count,
+// _status). 200 and 202 are success; 400, 403 and 422 are recorded and not
+// retried; 429 and 5xx are recorded and returned as an error, so the queue
+// retries the job. Outside production, while IndexNow is off or without a
+// site address nothing is sent; addresses of another host never are.
+func SendIndexNow(ctx context.Context, app *backpack.App, args IndexNowArgs) error {
+	if !Production(app) {
+		pluginLog.Info("golem15.sitemap: indexnow skipped outside production", "urls", len(args.URLs))
+		return nil
+	}
+	db, ok := app.Lookup[*gorm.DB]()
+	if !ok || db == nil {
+		return ErrNoDatabase
+	}
+	db = db.WithContext(ctx)
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return err
+	}
+	key := ActiveKey(settings)
+	site := SiteURL(app, settings)
+	urls := SiteURLs(site, args.URLs)
+	if key == "" || len(urls) == 0 {
+		return nil
+	}
+	if len(urls) > MaxIndexNowURLs {
+		urls = urls[:MaxIndexNowURLs]
+	}
+	client, err := fetchguard.NewClient(fetchguard.Policy{
+		Mode:       fetchguard.AllowHostsMode,
+		AllowHosts: []string{IndexNowHost},
+		Timeout:    indexNowTimeout,
+	}, app.Config)
+	if err != nil {
+		return err
+	}
+	res, err := client.PostJSON(ctx, IndexNowEndpoint, nil, indexNowBody{Host: hostOf(site), Key: key, URLList: urls})
+	if err != nil {
+		return err
+	}
+	status := res.StatusCode
+	if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_last_at = ?, indexnow_last_count = ?, indexnow_last_status = ? WHERE id = 1`,
+		time.Now().UTC(), len(urls), status).Error; err != nil {
+		return err
+	}
+	switch {
+	case status == http.StatusOK || status == http.StatusAccepted:
+		pluginLog.Info("golem15.sitemap: indexnow notified", "urls", len(urls), "status", status)
+		return nil
+	case status == http.StatusTooManyRequests || status >= 500:
+		pluginLog.Warn("golem15.sitemap: indexnow will retry", "urls", len(urls), "status", status)
+		return errIndexNowRetry{status: status}
+	default:
+		pluginLog.Warn("golem15.sitemap: indexnow refused", "urls", len(urls), "status", status)
+		return nil
+	}
+}
+
+// EnsureKey gives settings row 1 an IndexNow key on save: while enabled,
+// a new key is generated when none is stored or rotate is set (the
+// indexnow_new_key checkbox), so the old key file answers 404 at once. It
+// returns the stored key ("" while disabled without one).
+func EnsureKey(db *gorm.DB, enabled, rotate bool) (string, error) {
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return "", err
+	}
+	current := strings.TrimSpace(models.Text(settings.IndexNowKey))
+	if !enabled || (ValidKey(current) && !rotate) {
+		return current, nil
+	}
+	key, err := GenerateKey()
+	if err != nil {
+		return "", err
+	}
+	if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_key = ? WHERE id = 1`, key).Error; err != nil {
+		return "", err
+	}
+	return key, nil
+}
+
+// IndexNowRetryAt is when the queue next tries a pending IndexNow job
+// (after a 429 or 5xx answer), nil when none is pending.
+func IndexNowRetryAt(db *gorm.DB) *time.Time {
+	var at []time.Time
+	if err := db.Raw(`SELECT scheduled_at FROM river_job WHERE kind = ? AND state IN ('retryable', 'scheduled', 'available') ORDER BY scheduled_at ASC LIMIT 1`, IndexNowKind).Scan(&at).Error; err != nil || len(at) == 0 {
+		return nil
+	}
+	t := at[0].UTC()
+	return &t
+}
+
+// IndexNowStatus is the state the S2 partial and the management API show:
+// the key and its public URL (empty before a key exists), the last
+// notification (nil before the first) and, after a 429, the next attempt.
+type IndexNowStatus struct {
+	Enabled    bool       `json:"enabled"`
+	Key        string     `json:"key"`
+	KeyURL     string     `json:"key_url"`
+	LastAt     *time.Time `json:"last_at"`
+	LastCount  int        `json:"last_count"`
+	LastStatus int        `json:"last_status"`
+	RetryAt    *time.Time `json:"retry_at"`
+}
+
+// LoadIndexNowStatus reads the IndexNow state of settings.
+func LoadIndexNowStatus(db *gorm.DB, app *backpack.App, s models.Settings) IndexNowStatus {
+	out := IndexNowStatus{Enabled: s.IndexNowEnabled, LastAt: s.IndexNowLastAt}
+	if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
+		out.Key = k
+		out.KeyURL = SiteURL(app, s) + "/" + KeyFileName(k)
+	}
+	if s.IndexNowLastCount != nil {
+		out.LastCount = *s.IndexNowLastCount
+	}
+	if s.IndexNowLastStatus != nil {
+		out.LastStatus = *s.IndexNowLastStatus
+	}
+	if out.LastStatus == http.StatusTooManyRequests {
+		out.RetryAt = IndexNowRetryAt(db)
+	}
+	return out
+}
--- a/classes/rebuild.go
+++ b/classes/rebuild.go
@@ -117,7 +117,9 @@ type collected struct {
 // numbers the files, in one transaction that holds the settings row, so
 // two rebuilds never interleave. A record that is gone or unpublished
 // leaves its rows in the table with a reason, as a hole; only a changed
-// file size renumbers the files (D-30).
+// file size renumbers the files (D-30). In production, with IndexNow on,
+// the added, changed and removed addresses are queued for IndexNow after
+// the commit (NotifyIndexNow).
 func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
 	var res RebuildResult
 	if app == nil {
@@ -154,7 +156,15 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
 		res, err = apply(tx, found, locked, site)
 		return err
 	})
-	return res, err
+	if err != nil {
+		return res, err
+	}
+	// IndexNow hears of the change once it is committed (D-31); a queue
+	// error is logged and never fails the rebuild.
+	if err := NotifyIndexNow(ctx, app, db, res); err != nil {
+		pluginLog.Warn("golem15.sitemap: queue indexnow", "error", err)
+	}
+	return res, nil
 }

 // collect walks the content types page by page and adds the additional
--- a/commands_documented_test.go
+++ b/commands_documented_test.go
@@ -24,8 +24,8 @@ func TestCommandsDocumented(t *testing.T) {
 }

 // TestScheduleAndJobs pins the nightly sitemap:rebuild at 03:00, which must
-// name a registered command, and the rebuild job (kind
-// golem15.sitemap.rebuild, three attempts).
+// name a registered command, the rebuild job (kind golem15.sitemap.rebuild,
+// three attempts) and the IndexNow job (golem15.sitemap.indexnow, five).
 func TestScheduleAndJobs(t *testing.T) {
 	p := &Plugin{}
 	entries := p.Schedule()
@@ -40,11 +40,15 @@ func TestScheduleAndJobs(t *testing.T) {
 		t.Fatal("the scheduled command is not registered")
 	}
 	jobs := p.Jobs()
-	if len(jobs) != 1 {
+	if len(jobs) != 2 {
 		t.Fatalf("%d jobs", len(jobs))
 	}
 	info, ok := conga.Describe(jobs[0])
 	if !ok || info.Kind != classes.RebuildKind || info.MaxAttempts != 3 {
 		t.Fatalf("job = %+v %v", info, ok)
 	}
+	info, ok = conga.Describe(jobs[1])
+	if !ok || info.Kind != classes.IndexNowKind || info.MaxAttempts != 5 {
+		t.Fatalf("job = %+v %v", info, ok)
+	}
 }
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -18,3 +18,9 @@ lastmod_ignore:
 # rebuild_delay_seconds is how long after a record change the queued
 # rebuild runs; further changes in that time join the same rebuild.
 rebuild_delay_seconds: 60
+#
+# IndexNow has no config key: the endpoint is fixed at
+# https://api.indexnow.org/indexnow (the only host the plugin's outbound
+# client may reach), the key is generated in the settings and stored in the
+# database, and requests are sent only when the app environment is
+# production.
--- a/controllers/api/indexnow.go
+++ b/controllers/api/indexnow.go
@@ -0,0 +1,49 @@
+// Package api holds the sitemap plugin's HTTP handlers outside the admin:
+// the IndexNow key file, the HTML sitemap API and the management API.
+package api
+
+import (
+	"net/http"
+
+	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"gorm.io/gorm"
+)
+
+// KeyFilePattern is the {file} constraint of the key file route: an
+// IndexNow key file name, <key>.txt.
+const KeyFilePattern = `[A-Za-z0-9-]{8,128}\.txt`
+
+// KeyFile serves GET /_sitemap/indexnow/{file}: the IndexNow key as
+// text/plain when IndexNow is on and file is <key>.txt; any other name, a
+// replaced key or IndexNow off answers 404 (D-31). The SEO layer serves it
+// at the site root.
+func KeyFile(app func() *backpack.App) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		a := app()
+		if a == nil {
+			http.NotFound(w, r)
+			return
+		}
+		db, ok := a.Lookup[*gorm.DB]()
+		if !ok || db == nil {
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		settings, err := classes.LoadSettings(db.WithContext(r.Context()))
+		if err != nil {
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		key := classes.ActiveKey(settings)
+		if key == "" || r.PathValue("file") != classes.KeyFileName(key) {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-cache, private")
+		w.Header().Set("X-Robots-Tag", "noindex")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(key))
+	}
+}
--- a/controllers/settings.go
+++ b/controllers/settings.go
@@ -3,6 +3,9 @@ package controllers
 import (
 	"context"
 	"errors"
+	"net/http"
+	"strconv"
+	"strings"
 	"time"

 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
@@ -84,18 +87,44 @@ func (c settingsAdminController) FormBeforeCreate(ctx context.Context, model any
 	return c.FormBeforeUpdate(ctx, model)
 }

-// FormAfterUpdate dispatches a rebuild once the save commits (C26: saving
-// the form rebuilds the files now). A queue error is logged and never fails
-// the save.
-func (c settingsAdminController) FormAfterUpdate(ctx context.Context, _ any) error {
+// FormVirtualFields names indexnow_new_key, the "generate a new key on
+// save" checkbox, which is no column.
+func (settingsAdminController) FormVirtualFields() []string {
+	return []string{"indexnow_new_key"}
+}
+
+// FormAfterUpdate generates the IndexNow key (on the first save with
+// IndexNow on, or a new one when indexnow_new_key is checked) and
+// dispatches a rebuild once the save commits (C26: saving the form rebuilds
+// the files now). A queue error is logged and never fails the save.
+func (c settingsAdminController) FormAfterUpdate(ctx context.Context, model any) error {
 	app := appOf(c.app)
 	tx, err := database(ctx, c.app)
 	if err != nil || app == nil {
 		return nil
 	}
+	if s, ok := model.(*models.Settings); ok && s != nil {
+		values, _ := cabana.VirtualFieldsFromContext(ctx)
+		if _, err := classes.EnsureKey(tx, s.IndexNowEnabled, boolOf(values["indexnow_new_key"])); err != nil {
+			return err
+		}
+	}
 	return classes.RequestRebuild(ctx, app, tx)
 }

+// boolOf reads a submitted checkbox value.
+func boolOf(v any) bool {
+	switch b := v.(type) {
+	case bool:
+		return b
+	case string:
+		return b == "1" || strings.EqualFold(b, "true") || strings.EqualFold(b, "on")
+	case float64:
+		return b != 0
+	}
+	return false
+}
+
 // FormAfterCreate dispatches a rebuild like an update.
 func (c settingsAdminController) FormAfterCreate(ctx context.Context, model any) error {
 	return c.FormAfterUpdate(ctx, model)
@@ -135,10 +164,62 @@ func (c settingsAdminController) PartialData(ctx context.Context, name string, _
 		return c.filesData(ctx)
 	case "news_sources":
 		return c.newsSourcesData(ctx)
+	case "indexnow_status":
+		return c.indexNowData(ctx)
 	}
 	return nil, nil
 }

+// indexNowView is the view model of the S2 partial indexnow_status.
+type indexNowView struct {
+	Key      string
+	Last     string
+	Callout  string
+	Severity string
+}
+
+// indexNowData serves S2 (UI-SPEC C26): indexnow.key or indexnow.no_key,
+// indexnow.last or indexnow.never, and the callout of the last answer: 403
+// err_key (warning), 422 err_host (warning), 429 err_rate with the next
+// attempt (plain) while one is pending.
+func (c settingsAdminController) indexNowData(ctx context.Context) (any, error) {
+	app := appOf(c.app)
+	db, err := database(ctx, c.app)
+	if err != nil {
+		return nil, err
+	}
+	settings, err := classes.LoadSettings(db)
+	if err != nil {
+		return nil, err
+	}
+	st := classes.LoadIndexNowStatus(db, app, settings)
+	tr := translator(c.app)
+	view := indexNowView{}
+	if st.Key == "" {
+		view.Key = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.no_key", nil)
+	} else {
+		view.Key = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.key", map[string]string{"key": st.Key, "url": st.KeyURL})
+	}
+	if st.LastAt == nil {
+		view.Last = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.never", nil)
+	} else {
+		view.Last = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.last", map[string]string{
+			"time": formatTime(st.LastAt), "count": strconv.Itoa(st.LastCount), "status": strconv.Itoa(st.LastStatus),
+		})
+	}
+	switch st.LastStatus {
+	case http.StatusForbidden:
+		view.Callout, view.Severity = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_key", map[string]string{"url": st.KeyURL}), "warning"
+	case http.StatusUnprocessableEntity:
+		view.Callout, view.Severity = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_host", nil), "warning"
+	case http.StatusTooManyRequests:
+		if st.RetryAt != nil {
+			view.Callout = Phrase(ctx, tr, "golem15.sitemap::lang.indexnow.err_rate", map[string]string{"time": formatTime(st.RetryAt)})
+		}
+	}
+	return view, nil
+}
+
 // sourceRow is one row of the news_sources partial: the value to enter and
 // what it lists.
 type sourceRow struct {
@@ -337,4 +418,5 @@ var (
 	_ pact.FormBeforeUpdate  = settingsAdminController{}
 	_ pact.FormAfterCreate   = settingsAdminController{}
 	_ pact.FormAfterUpdate   = settingsAdminController{}
+	_ pact.FormVirtualFields = settingsAdminController{}
 )
--- a/controllers/settings/_indexnow_status.htm
+++ b/controllers/settings/_indexnow_status.htm
@@ -0,0 +1,15 @@
+<div class="summer-callout" role="status">
+<p class="summer-callout__text">{{ .Data.Key }}</p>
+<p class="summer-callout__text">{{ .Data.Last }}</p>
+</div>
+{{- if .Data.Callout -}}
+{{- if eq .Data.Severity "warning" -}}
+<div class="summer-callout summer-callout--warning" role="alert">
+<p class="summer-callout__text">{{ .Data.Callout }}</p>
+</div>
+{{- else -}}
+<div class="summer-callout" role="status">
+<p class="summer-callout__text">{{ .Data.Callout }}</p>
+</div>
+{{- end -}}
+{{- end -}}
--- a/jobs.go
+++ b/jobs.go
@@ -8,13 +8,21 @@ import (
 	"git.golem15.com/golem15/summercms/modules/pact"
 )

-// Jobs registers the rebuild job (classes.RebuildKind) on conga.
+// Jobs registers the rebuild job (classes.RebuildKind) and the IndexNow
+// job (classes.IndexNowKind, five attempts: 429 and 5xx answers retry) on
+// conga.
 func (p *Plugin) Jobs() []pact.Job {
 	return []pact.Job{
 		conga.Job(p.rebuild, conga.MaxAttempts(3)),
+		conga.Job(p.indexNow, conga.MaxAttempts(classes.IndexNowAttempts)),
 	}
 }

+// indexNow sends one IndexNow request (classes.SendIndexNow).
+func (p *Plugin) indexNow(ctx context.Context, args classes.IndexNowArgs) error {
+	return classes.SendIndexNow(ctx, p.app, args)
+}
+
 // rebuild runs one rebuild of the slot table. It ends the debounce window
 // first, so record events during the rebuild queue the next one.
 func (p *Plugin) rebuild(ctx context.Context, _ classes.RebuildArgs) error {
--- a/models/settings.go
+++ b/models/settings.go
@@ -92,7 +92,7 @@ func (Settings) TableName() string { return "golem15_sitemap_settings" }

 // Fillable lists the columns the settings form edits.
 func (Settings) Fillable() []string {
-	return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources"}
+	return []string{"site_url", "file_size", "extra_urls", "images_enabled", "news_enabled", "news_publication", "news_sources", "indexnow_enabled"}
 }

 // Rules is empty: the settings controller runs Validate before a save.
--- a/models/settings/fields.yaml
+++ b/models/settings/fields.yaml
@@ -99,3 +99,31 @@ fields:
            action: hide
            field: news_enabled
            condition: unchecked
+
+    indexnow_enabled:
+        label: golem15.sitemap::lang.settings.indexnow_enabled
+        comment: golem15.sitemap::lang.settings.indexnow_enabled_comment
+        type: switch
+        default: false
+        span: full
+        tab: golem15.sitemap::lang.settings.tab_indexnow
+
+    _indexnow_status:
+        type: partial
+        path: indexnow_status
+        span: full
+        tab: golem15.sitemap::lang.settings.tab_indexnow
+
+    # No column: the controller lists it in FormVirtualFields and generates
+    # a new key on save when it is checked.
+    indexnow_new_key:
+        label: golem15.sitemap::lang.settings.indexnow_new_key
+        comment: golem15.sitemap::lang.settings.indexnow_new_key_comment
+        type: checkbox
+        default: false
+        span: full
+        tab: golem15.sitemap::lang.settings.tab_indexnow
+        trigger:
+            action: hide
+            field: indexnow_enabled
+            condition: unchecked
--- a/routes.go
+++ b/routes.go
@@ -7,6 +7,7 @@ import (
 	"time"

 	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
+	apicontrollers "git.golem15.com/golem15/sm-sitemap-plugin/controllers/api"
 	"git.golem15.com/golem15/sm-sitemap-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/pact"
@@ -28,17 +29,23 @@ const BucketSitemap = "sitemap-xml"
 // maps the root names to this route.
 const FilesPath = "/_sitemap/files/"

-// Routes mounts GET /sitemap.xml (the index) and GET
-// /_sitemap/files/{file} (one file) without the backend middleware: search
-// engines fetch them anonymously. Both are served from the slot table, never
-// generated per request, and throttled per client IP (BucketSitemap).
+// Routes mounts GET /sitemap.xml (the index), GET /_sitemap/files/{file}
+// (one file) and GET /_sitemap/indexnow/{file} (the IndexNow key file)
+// without the backend middleware: search engines fetch them anonymously.
+// The sitemaps are served from the slot table, never generated per
+// request, and every route is throttled per client IP (BucketSitemap).
 func (p *Plugin) Routes(r pact.Router) error {
 	r.Get("/sitemap.xml", p.sitemapIndex, "throttle:"+BucketSitemap)
 	r.Get("/_sitemap/files/{file}", p.sitemapFile, "throttle:"+BucketSitemap)
 	r.Where("file", `sitemap-(news|[a-z0-9-]+-[0-9]+)\.xml`)
+	r.Get("/_sitemap/indexnow/{file}", apicontrollers.KeyFile(p.booted), "throttle:"+BucketSitemap)
+	r.Where("file", apicontrollers.KeyFilePattern)
 	return nil
 }

+// booted returns the booted application.
+func (p *Plugin) booted() *backpack.App { return p.app }
+
 // Buckets registers sitemap-xml: 60 requests a minute per client IP (behind
 // the configured trusted proxies). 429 bodies are surf's
 // {"message":"Too Many Attempts."}.
--- a/sitemap_extras_test.go
+++ b/sitemap_extras_test.go
@@ -2,14 +2,20 @@ package sitemap_test

 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"

+	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
 	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/conga"
+	"git.golem15.com/golem15/summercms/modules/fetchguard"
 	"git.golem15.com/golem15/summercms/modules/pact"
 	"git.golem15.com/golem15/summercms/modules/party"
 )
@@ -290,3 +296,243 @@ func TestNewsSitemap(t *testing.T) {
 		t.Fatalf("News file after turning it off: %d", rec.Code)
 	}
 }
+
+// job returns the sitemap's registered job of kind.
+func (e *sitemapEnv) job(t *testing.T, kind string) pact.Job {
+	t.Helper()
+	for _, p := range e.plugins {
+		hj, ok := p.(pact.HasJobs)
+		if !ok || p.ID() != "golem15.sitemap" {
+			continue
+		}
+		for _, j := range hj.Jobs() {
+			if info, ok := conga.Describe(j); ok && info.Kind == kind {
+				return j
+			}
+		}
+	}
+	t.Fatalf("the sitemap registers no job %s", kind)
+	return nil
+}
+
+// fakeIndexNow is the IndexNow engine of a test: it records each request
+// and answers status.
+type fakeIndexNow struct {
+	mu       sync.Mutex
+	status   int
+	requests []*http.Request
+	bodies   []map[string]any
+}
+
+func (f *fakeIndexNow) RoundTrip(r *http.Request) (*http.Response, error) {
+	raw, _ := io.ReadAll(r.Body)
+	body := map[string]any{}
+	_ = json.Unmarshal(raw, &body)
+	f.mu.Lock()
+	f.requests = append(f.requests, r)
+	f.bodies = append(f.bodies, body)
+	status := f.status
+	f.mu.Unlock()
+	return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+}
+
+// queuedIndexNow decodes the URL lists of the queued IndexNow jobs, oldest
+// first.
+func (e *sitemapEnv) queuedIndexNow(t *testing.T) [][]string {
+	t.Helper()
+	var rows []string
+	if err := e.gdb.Raw(`SELECT args::text FROM river_job WHERE kind = ? ORDER BY id`, classes.IndexNowKind).Scan(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	out := make([][]string, 0, len(rows))
+	for _, raw := range rows {
+		var args classes.IndexNowArgs
+		if err := json.Unmarshal([]byte(raw), &args); err != nil {
+			t.Fatal(err)
+		}
+		out = append(out, args.URLs)
+	}
+	return out
+}
+
+// indexNowKey reads the stored key.
+func (e *sitemapEnv) indexNowKey(t *testing.T) string {
+	t.Helper()
+	var key *string
+	if err := e.gdb.Raw(`SELECT indexnow_key FROM golem15_sitemap_settings WHERE id = 1`).Scan(&key).Error; err != nil {
+		t.Fatal(err)
+	}
+	if key == nil {
+		return ""
+	}
+	return *key
+}
+
+// TestIndexNow drives D-31's IndexNow in production: enabling and saving
+// generates a 32-hex key whose file answers text/plain (another name 404);
+// a rebuild with a new and a removed address queues one job with both; the
+// job posts {host, key, urlList} to api.indexnow.org and records the
+// answer (202); 403 and 422 are recorded without retry, 429 fails the job
+// so it retries; S2 shows each state; an address of another host is never
+// sent; a new key makes the old file 404. Outside production nothing is
+// queued or sent.
+func TestIndexNow(t *testing.T) {
+	e := bootJournal(t, map[string]any{envOverride: "production"})
+	token := e.admin(t, permSitemap)
+	const base = "/golem15/sitemap/settings"
+	s2 := func() string {
+		t.Helper()
+		rec := e.call(t, http.MethodGet, base+"/partials/indexnow_status", token, nil)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("S2: %d %s", rec.Code, rec.Body.String())
+		}
+		return rec.Body.String()
+	}
+	if got := s2(); !strings.Contains(got, "A key is generated when you turn IndexNow on and save.") || !strings.Contains(got, "No notification sent yet.") {
+		t.Fatalf("S2 before a key:\n%s", got)
+	}
+	schema := e.call(t, http.MethodGet, base+"/schema/form", token, nil).Body.String()
+	for _, want := range []string{`"indexnow_enabled"`, `"indexnow_new_key"`, `"indexnow_status"`, "IndexNow"} {
+		if !strings.Contains(schema, want) {
+			t.Errorf("form schema lacks %s", want)
+		}
+	}
+
+	// A rebuild before IndexNow is on queues nothing.
+	at := time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)
+	contentStore.put(postType, post(1, "a", "en", at))
+	e.rebuild(t)
+	if q := e.queuedIndexNow(t); len(q) != 0 {
+		t.Fatalf("queued while off: %v", q)
+	}
+
+	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true}); rec.Code != http.StatusOK {
+		t.Fatalf("enable IndexNow: %d %s", rec.Code, rec.Body.String())
+	}
+	key := e.indexNowKey(t)
+	if !regexp.MustCompile(`^[0-9a-f]{32}$`).MatchString(key) {
+		t.Fatalf("key = %q", key)
+	}
+	rec := e.get(t, "/_sitemap/indexnow/"+key+".txt")
+	if rec.Code != http.StatusOK || rec.Body.String() != key || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
+		t.Fatalf("key file: %d %q %v", rec.Code, rec.Body.String(), rec.Header())
+	}
+	for _, name := range []string{"0123456789abcdef0123456789abcdef.txt", "short.txt", key + ".xml"} {
+		if rec := e.get(t, "/_sitemap/indexnow/"+name); rec.Code != http.StatusNotFound {
+			t.Errorf("key file %s: %d", name, rec.Code)
+		}
+	}
+	if got := s2(); !strings.Contains(got, "Key: "+key+", served at https://api.example.test/"+key+".txt") {
+		t.Fatalf("S2 with a key:\n%s", got)
+	}
+
+	// One new and one removed address: one job with both.
+	contentStore.put(postType, post(2, "b", "en", at))
+	contentStore.remove(postType, 1)
+	e.rebuild(t)
+	q := e.queuedIndexNow(t)
+	if len(q) != 1 || strings.Join(q[0], " ") != "https://api.example.test/blog/b https://api.example.test/blog/a" {
+		t.Fatalf("queued = %v", q)
+	}
+
+	engine := &fakeIndexNow{status: http.StatusAccepted}
+	ctx := fetchguard.WithTransport(t.Context(), engine)
+	job := e.job(t, classes.IndexNowKind)
+	args := classes.IndexNowArgs{URLs: append(q[0], "https://evil.test/x", "http://api.example.test.evil.test/y")}
+	if err := job.Work(ctx, args); err != nil {
+		t.Fatalf("IndexNow job: %v", err)
+	}
+	if len(engine.requests) != 1 {
+		t.Fatalf("%d requests", len(engine.requests))
+	}
+	req, body := engine.requests[0], engine.bodies[0]
+	if req.Method != http.MethodPost || req.URL.String() != "https://api.indexnow.org/indexnow" || !strings.HasPrefix(req.Header.Get("Content-Type"), "application/json") {
+		t.Fatalf("request = %s %s %v", req.Method, req.URL, req.Header)
+	}
+	list, _ := body["urlList"].([]any)
+	if body["host"] != "api.example.test" || body["key"] != key || len(list) != 2 || body["keyLocation"] != nil {
+		t.Fatalf("body = %v", body)
+	}
+	var last struct {
+		Status int `gorm:"column:indexnow_last_status"`
+		Count  int `gorm:"column:indexnow_last_count"`
+	}
+	e.gdb.Raw(`SELECT indexnow_last_status, indexnow_last_count FROM golem15_sitemap_settings WHERE id = 1`).Scan(&last)
+	if last.Status != 202 || last.Count != 2 {
+		t.Fatalf("recorded = %+v", last)
+	}
+	if got := s2(); !strings.Contains(got, ", 2 addresses, answer 202.") || strings.Contains(got, "summer-callout--warning") {
+		t.Fatalf("S2 after 202:\n%s", got)
+	}
+
+	for _, c := range []struct {
+		status int
+		retry  bool
+		want   string
+	}{
+		{http.StatusForbidden, false, "The engines could not read the key file at https://api.example.test/" + key + ".txt."},
+		{http.StatusUnprocessableEntity, false, "The engines refused the addresses because they do not match the site address."},
+		{http.StatusBadRequest, false, "answer 400."},
+		{http.StatusTooManyRequests, true, "Too many notifications; the next attempt is at "},
+	} {
+		engine.status = c.status
+		err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]})
+		if (err != nil) != c.retry {
+			t.Errorf("status %d: job error %v, want retry %v", c.status, err, c.retry)
+		}
+		if got := s2(); !strings.Contains(got, c.want) {
+			t.Errorf("S2 after %d lacks %q:\n%s", c.status, c.want, got)
+		}
+	}
+
+	// A new key: the old file answers 404 at once.
+	if rec := e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true, "indexnow_new_key": true}); rec.Code != http.StatusOK {
+		t.Fatalf("new key: %d %s", rec.Code, rec.Body.String())
+	}
+	fresh := e.indexNowKey(t)
+	if fresh == key || len(fresh) != 32 {
+		t.Fatalf("new key = %q (old %q)", fresh, key)
+	}
+	if rec := e.get(t, "/_sitemap/indexnow/"+key+".txt"); rec.Code != http.StatusNotFound {
+		t.Fatalf("old key file: %d", rec.Code)
+	}
+	if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusOK {
+		t.Fatalf("new key file: %d", rec.Code)
+	}
+	// A save without the checkbox keeps the key; turning IndexNow off
+	// takes the file down.
+	e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": true})
+	if got := e.indexNowKey(t); got != fresh {
+		t.Fatalf("a plain save changed the key to %q", got)
+	}
+	e.call(t, http.MethodPut, base+"/1", token, map[string]any{"indexnow_enabled": false})
+	if rec := e.get(t, "/_sitemap/indexnow/"+fresh+".txt"); rec.Code != http.StatusNotFound {
+		t.Fatalf("key file while off: %d", rec.Code)
+	}
+	before := len(engine.requests)
+	if err := job.Work(ctx, classes.IndexNowArgs{URLs: q[0]}); err != nil || len(engine.requests) != before {
+		t.Fatalf("job while off: %v, %d requests", err, len(engine.requests)-before)
+	}
+}
+
+// TestIndexNowStaging pins T-33-15: outside production a rebuild queues no
+// IndexNow job and the job sends nothing.
+func TestIndexNowStaging(t *testing.T) {
+	e := bootJournal(t)
+	e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`)
+	contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)))
+	e.rebuild(t)
+	if q := e.queuedIndexNow(t); len(q) != 0 {
+		t.Fatalf("queued outside production: %v", q)
+	}
+	engine := &fakeIndexNow{status: http.StatusOK}
+	if err := e.job(t, classes.IndexNowKind).Work(fetchguard.WithTransport(t.Context(), engine), classes.IndexNowArgs{URLs: []string{"https://api.example.test/blog/a"}}); err != nil {
+		t.Fatal(err)
+	}
+	if len(engine.requests) != 0 {
+		t.Fatalf("%d requests outside production", len(engine.requests))
+	}
+	if rec := e.get(t, "/_sitemap/indexnow/0123456789abcdef0123456789abcdef.txt"); rec.Code != http.StatusOK {
+		t.Fatalf("key file outside production: %d", rec.Code)
+	}
+}
--- a/sitemap_smoke_test.go
+++ b/sitemap_smoke_test.go
@@ -35,8 +35,13 @@ type sitemapEnv struct {
 	plugins []party.Plugin
 }

+// envOverride is the override key that sets the app environment
+// (SUMMER_ENV) instead of a config key; development by default.
+const envOverride = "__env"
+
 // bootPlugins boots the application with the plugins ids on a fresh
-// database; overrides set further config keys.
+// database; overrides set further config keys (envOverride the
+// environment).
 func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitemapEnv {
 	t.Helper()
 	sqlDB := freshDB(t)
@@ -48,9 +53,15 @@ func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitem
 	if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: sitemap-smoke\nlocale: en\nfallback_locale: en\nurl: https://api.example.test\n"), 0o644); err != nil {
 		t.Fatal(err)
 	}
+	env := "development"
+	for _, o := range overrides {
+		if v, ok := o[envOverride].(string); ok {
+			env = v
+		}
+	}
 	cfg, err := compass.Open(compass.Options{
 		Dir:     dir,
-		Environ: []string{"SUMMER_ENV=development", "SUMMER_ADMIN__JWT__SECRET=sitemap-smoke-test-only-hs256-secret-key"},
+		Environ: []string{"SUMMER_ENV=" + env, "SUMMER_ADMIN__JWT__SECRET=sitemap-smoke-test-only-hs256-secret-key"},
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -63,7 +74,9 @@ func bootPlugins(t *testing.T, ids []string, overrides ...map[string]any) *sitem
 	}
 	for _, o := range overrides {
 		for key, value := range o {
-			settings[key] = value
+			if key != envOverride {
+				settings[key] = value
+			}
 		}
 	}
 	for key, value := range settings {