feat: IndexNow key, key file and pings
- saving with IndexNow on generates a 32-hex key (crypto/rand);
indexnow_new_key generates a new one, so the old key file answers 404
- GET /_sitemap/indexnow/{file} serves <key>.txt as text/plain while
IndexNow is on; the SEO layer maps it to the site root
- after each rebuild in production the added, changed and removed
addresses on the site host are queued as golem15.sitemap.indexnow jobs
(at most 10 000 per request, five attempts)
- the job posts {host, key, urlList} to api.indexnow.org through a
fetchguard client limited to that host and records the answer; 429 and
5xx retry, 400, 403 and 422 do not
- the IndexNow tab shows the S2 status partial: key or no key, last
notification or never, and the 403, 422 and 429 callouts
This commit is contained in:
Jakub Zych
294
classes/indexnow.go
Normal file
294
classes/indexnow.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// IndexNow (D-31, P33-14): after a rebuild the added, changed and removed
|
||||
// addresses are posted to the shared IndexNow endpoint, which hands them to
|
||||
// every participating engine. The endpoint is fixed, not configurable: the
|
||||
// outbound client may reach that one host only (T-33-13).
|
||||
const (
|
||||
IndexNowKind = "golem15.sitemap.indexnow"
|
||||
IndexNowHost = "api.indexnow.org"
|
||||
IndexNowEndpoint = "https://" + IndexNowHost + "/indexnow"
|
||||
// MaxIndexNowURLs is the most addresses one request carries.
|
||||
MaxIndexNowURLs = 10000
|
||||
// indexNowTimeout bounds one request.
|
||||
indexNowTimeout = 10 * time.Second
|
||||
// IndexNowAttempts is the attempt limit of the job: an answer of 429 or
|
||||
// 5xx, or a network error, is retried by the queue up to this many times.
|
||||
IndexNowAttempts = 5
|
||||
)
|
||||
|
||||
// IndexNowArgs are the arguments of the IndexNow job: the absolute
|
||||
// addresses of one request (at most MaxIndexNowURLs).
|
||||
type IndexNowArgs struct {
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
|
||||
// Kind is IndexNowKind.
|
||||
func (IndexNowArgs) Kind() string { return IndexNowKind }
|
||||
|
||||
// keyPattern is the form of an IndexNow key: 8 to 128 letters, digits and
|
||||
// hyphens (indexnow.org documentation).
|
||||
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,128}$`)
|
||||
|
||||
// ValidKey reports whether k may be an IndexNow key.
|
||||
func ValidKey(k string) bool { return keyPattern.MatchString(k) }
|
||||
|
||||
// GenerateKey returns a new IndexNow key: 32 lowercase hex characters from
|
||||
// the system random source.
|
||||
func GenerateKey() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", fmt.Errorf("golem15.sitemap: indexnow key: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
// KeyFileName is the name of the key file of key: <key>.txt, served at the
|
||||
// site root by the SEO layer (from GET /_sitemap/indexnow/<key>.txt); ""
|
||||
// when key is not a ValidKey.
|
||||
func KeyFileName(key string) string {
|
||||
if !ValidKey(key) {
|
||||
return ""
|
||||
}
|
||||
return key + ".txt"
|
||||
}
|
||||
|
||||
// ActiveKey is the key of settings while IndexNow is on, "" otherwise.
|
||||
func ActiveKey(s models.Settings) string {
|
||||
if !s.IndexNowEnabled {
|
||||
return ""
|
||||
}
|
||||
if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
|
||||
return k
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Production reports whether app runs in the production environment; no
|
||||
// IndexNow request is sent anywhere else (T-33-15).
|
||||
func Production(app *backpack.App) bool {
|
||||
return app != nil && app.Config != nil && app.Config.Environment() == "production"
|
||||
}
|
||||
|
||||
// hostOf is the lowercase host of an absolute http(s) URL, "" otherwise.
|
||||
func hostOf(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(parsed.Hostname())
|
||||
}
|
||||
|
||||
// SiteURLs keeps the addresses on the host of site, each once, in order:
|
||||
// IndexNow refuses (422) a request with an address of another host.
|
||||
func SiteURLs(site string, urls []string) []string {
|
||||
host := hostOf(site)
|
||||
if host == "" {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for _, u := range urls {
|
||||
if u == "" || seen[u] || hostOf(u) != host {
|
||||
continue
|
||||
}
|
||||
seen[u] = true
|
||||
out = append(out, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NotifyIndexNow queues the IndexNow job for what a rebuild changed: its
|
||||
// added, changed and removed addresses on the site's host, in requests of
|
||||
// at most MaxIndexNowURLs. Nothing is queued outside production, while
|
||||
// IndexNow is off or has no key, without a site address or when nothing
|
||||
// changed.
|
||||
func NotifyIndexNow(ctx context.Context, app *backpack.App, db *gorm.DB, res RebuildResult) error {
|
||||
if !Production(app) {
|
||||
return nil
|
||||
}
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ActiveKey(settings) == "" {
|
||||
return nil
|
||||
}
|
||||
all := make([]string, 0, len(res.Added)+len(res.Changed)+len(res.Removed))
|
||||
all = append(append(append(all, res.Added...), res.Changed...), res.Removed...)
|
||||
urls := SiteURLs(SiteURL(app, settings), all)
|
||||
if len(urls) == 0 {
|
||||
return nil
|
||||
}
|
||||
m, err := conga.From(app)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for start := 0; start < len(urls); start += MaxIndexNowURLs {
|
||||
end := min(start+MaxIndexNowURLs, len(urls))
|
||||
if err := m.Enqueue(ctx, db, IndexNowArgs{URLs: urls[start:end]}, conga.EnqueueOpts{}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// indexNowBody is the JSON of an IndexNow request. keyLocation is not sent:
|
||||
// the key file is served at the site root, which covers every address of
|
||||
// the host (P33-14).
|
||||
type indexNowBody struct {
|
||||
Host string `json:"host"`
|
||||
Key string `json:"key"`
|
||||
URLList []string `json:"urlList"`
|
||||
}
|
||||
|
||||
// errIndexNowRetry is a retryable IndexNow answer.
|
||||
type errIndexNowRetry struct{ status int }
|
||||
|
||||
func (e errIndexNowRetry) Error() string {
|
||||
return fmt.Sprintf("golem15.sitemap: indexnow answered %d; retrying", e.status)
|
||||
}
|
||||
|
||||
// SendIndexNow runs one IndexNow job: it posts {host, key, urlList} to
|
||||
// IndexNowEndpoint through a fetchguard client that reaches that host only
|
||||
// (HTTPS, no redirects) and records the answer (indexnow_last_at, _count,
|
||||
// _status). 200 and 202 are success; 400, 403 and 422 are recorded and not
|
||||
// retried; 429 and 5xx are recorded and returned as an error, so the queue
|
||||
// retries the job. Outside production, while IndexNow is off or without a
|
||||
// site address nothing is sent; addresses of another host never are.
|
||||
func SendIndexNow(ctx context.Context, app *backpack.App, args IndexNowArgs) error {
|
||||
if !Production(app) {
|
||||
pluginLog.Info("golem15.sitemap: indexnow skipped outside production", "urls", len(args.URLs))
|
||||
return nil
|
||||
}
|
||||
db, ok := app.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
return ErrNoDatabase
|
||||
}
|
||||
db = db.WithContext(ctx)
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := ActiveKey(settings)
|
||||
site := SiteURL(app, settings)
|
||||
urls := SiteURLs(site, args.URLs)
|
||||
if key == "" || len(urls) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(urls) > MaxIndexNowURLs {
|
||||
urls = urls[:MaxIndexNowURLs]
|
||||
}
|
||||
client, err := fetchguard.NewClient(fetchguard.Policy{
|
||||
Mode: fetchguard.AllowHostsMode,
|
||||
AllowHosts: []string{IndexNowHost},
|
||||
Timeout: indexNowTimeout,
|
||||
}, app.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := client.PostJSON(ctx, IndexNowEndpoint, nil, indexNowBody{Host: hostOf(site), Key: key, URLList: urls})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := res.StatusCode
|
||||
if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_last_at = ?, indexnow_last_count = ?, indexnow_last_status = ? WHERE id = 1`,
|
||||
time.Now().UTC(), len(urls), status).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case status == http.StatusOK || status == http.StatusAccepted:
|
||||
pluginLog.Info("golem15.sitemap: indexnow notified", "urls", len(urls), "status", status)
|
||||
return nil
|
||||
case status == http.StatusTooManyRequests || status >= 500:
|
||||
pluginLog.Warn("golem15.sitemap: indexnow will retry", "urls", len(urls), "status", status)
|
||||
return errIndexNowRetry{status: status}
|
||||
default:
|
||||
pluginLog.Warn("golem15.sitemap: indexnow refused", "urls", len(urls), "status", status)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureKey gives settings row 1 an IndexNow key on save: while enabled,
|
||||
// a new key is generated when none is stored or rotate is set (the
|
||||
// indexnow_new_key checkbox), so the old key file answers 404 at once. It
|
||||
// returns the stored key ("" while disabled without one).
|
||||
func EnsureKey(db *gorm.DB, enabled, rotate bool) (string, error) {
|
||||
settings, err := LoadSettings(db)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
current := strings.TrimSpace(models.Text(settings.IndexNowKey))
|
||||
if !enabled || (ValidKey(current) && !rotate) {
|
||||
return current, nil
|
||||
}
|
||||
key, err := GenerateKey()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_key = ? WHERE id = 1`, key).Error; err != nil {
|
||||
return "", err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// IndexNowRetryAt is when the queue next tries a pending IndexNow job
|
||||
// (after a 429 or 5xx answer), nil when none is pending.
|
||||
func IndexNowRetryAt(db *gorm.DB) *time.Time {
|
||||
var at []time.Time
|
||||
if err := db.Raw(`SELECT scheduled_at FROM river_job WHERE kind = ? AND state IN ('retryable', 'scheduled', 'available') ORDER BY scheduled_at ASC LIMIT 1`, IndexNowKind).Scan(&at).Error; err != nil || len(at) == 0 {
|
||||
return nil
|
||||
}
|
||||
t := at[0].UTC()
|
||||
return &t
|
||||
}
|
||||
|
||||
// IndexNowStatus is the state the S2 partial and the management API show:
|
||||
// the key and its public URL (empty before a key exists), the last
|
||||
// notification (nil before the first) and, after a 429, the next attempt.
|
||||
type IndexNowStatus struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Key string `json:"key"`
|
||||
KeyURL string `json:"key_url"`
|
||||
LastAt *time.Time `json:"last_at"`
|
||||
LastCount int `json:"last_count"`
|
||||
LastStatus int `json:"last_status"`
|
||||
RetryAt *time.Time `json:"retry_at"`
|
||||
}
|
||||
|
||||
// LoadIndexNowStatus reads the IndexNow state of settings.
|
||||
func LoadIndexNowStatus(db *gorm.DB, app *backpack.App, s models.Settings) IndexNowStatus {
|
||||
out := IndexNowStatus{Enabled: s.IndexNowEnabled, LastAt: s.IndexNowLastAt}
|
||||
if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
|
||||
out.Key = k
|
||||
out.KeyURL = SiteURL(app, s) + "/" + KeyFileName(k)
|
||||
}
|
||||
if s.IndexNowLastCount != nil {
|
||||
out.LastCount = *s.IndexNowLastCount
|
||||
}
|
||||
if s.IndexNowLastStatus != nil {
|
||||
out.LastStatus = *s.IndexNowLastStatus
|
||||
}
|
||||
if out.LastStatus == http.StatusTooManyRequests {
|
||||
out.RetryAt = IndexNowRetryAt(db)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -117,7 +117,9 @@ type collected struct {
|
||||
// numbers the files, in one transaction that holds the settings row, so
|
||||
// two rebuilds never interleave. A record that is gone or unpublished
|
||||
// leaves its rows in the table with a reason, as a hole; only a changed
|
||||
// file size renumbers the files (D-30).
|
||||
// file size renumbers the files (D-30). In production, with IndexNow on,
|
||||
// the added, changed and removed addresses are queued for IndexNow after
|
||||
// the commit (NotifyIndexNow).
|
||||
func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
|
||||
var res RebuildResult
|
||||
if app == nil {
|
||||
@@ -154,7 +156,15 @@ func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
|
||||
res, err = apply(tx, found, locked, site)
|
||||
return err
|
||||
})
|
||||
return res, err
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
// IndexNow hears of the change once it is committed (D-31); a queue
|
||||
// error is logged and never fails the rebuild.
|
||||
if err := NotifyIndexNow(ctx, app, db, res); err != nil {
|
||||
pluginLog.Warn("golem15.sitemap: queue indexnow", "error", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// collect walks the content types page by page and adds the additional
|
||||
|
||||
Reference in New Issue
Block a user