feat: IndexNow key, key file and pings

- saving with IndexNow on generates a 32-hex key (crypto/rand);
  indexnow_new_key generates a new one, so the old key file answers 404
- GET /_sitemap/indexnow/{file} serves <key>.txt as text/plain while
  IndexNow is on; the SEO layer maps it to the site root
- after each rebuild in production the added, changed and removed
  addresses on the site host are queued as golem15.sitemap.indexnow jobs
  (at most 10 000 per request, five attempts)
- the job posts {host, key, urlList} to api.indexnow.org through a
  fetchguard client limited to that host and records the answer; 429 and
  5xx retry, 400, 403 and 422 do not
- the IndexNow tab shows the S2 status partial: key or no key, last
  notification or never, and the 403, 422 and 429 callouts
This commit is contained in:
Jakub Zych
2026-11-04 12:26:47 +01:00
parent a10ce2d9de
commit e02d0b173f
13 changed files with 780 additions and 18 deletions

View File

@@ -0,0 +1,49 @@
// Package api holds the sitemap plugin's HTTP handlers outside the admin:
// the IndexNow key file, the HTML sitemap API and the management API.
package api
import (
"net/http"
"git.golem15.com/golem15/sm-sitemap-plugin/classes"
"git.golem15.com/golem15/summercms/modules/backpack"
"gorm.io/gorm"
)
// KeyFilePattern is the {file} constraint of the key file route: an
// IndexNow key file name, <key>.txt.
const KeyFilePattern = `[A-Za-z0-9-]{8,128}\.txt`
// KeyFile serves GET /_sitemap/indexnow/{file}: the IndexNow key as
// text/plain when IndexNow is on and file is <key>.txt; any other name, a
// replaced key or IndexNow off answers 404 (D-31). The SEO layer serves it
// at the site root.
func KeyFile(app func() *backpack.App) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
a := app()
if a == nil {
http.NotFound(w, r)
return
}
db, ok := a.Lookup[*gorm.DB]()
if !ok || db == nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
settings, err := classes.LoadSettings(db.WithContext(r.Context()))
if err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
key := classes.ActiveKey(settings)
if key == "" || r.PathValue("file") != classes.KeyFileName(key) {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Cache-Control", "no-cache, private")
w.Header().Set("X-Robots-Tag", "noindex")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(key))
}
}