package classes import ( "context" "crypto/rand" "encoding/hex" "fmt" "net/http" "net/url" "regexp" "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/conga" "git.golem15.com/golem15/summercms/modules/fetchguard" "gorm.io/gorm" ) // IndexNow (D-31, P33-14): after a rebuild the added, changed and removed // addresses are posted to the shared IndexNow endpoint, which hands them to // every participating engine. The endpoint is fixed, not configurable: the // outbound client may reach that one host only (T-33-13). const ( IndexNowKind = "golem15.sitemap.indexnow" IndexNowHost = "api.indexnow.org" IndexNowEndpoint = "https://" + IndexNowHost + "/indexnow" // MaxIndexNowURLs is the most addresses one request carries. MaxIndexNowURLs = 10000 // indexNowTimeout bounds one request. indexNowTimeout = 10 * time.Second // IndexNowAttempts is the attempt limit of the job: an answer of 429 or // 5xx, or a network error, is retried by the queue up to this many times. IndexNowAttempts = 5 ) // IndexNowArgs are the arguments of the IndexNow job: the absolute // addresses of one request (at most MaxIndexNowURLs). type IndexNowArgs struct { URLs []string `json:"urls"` } // Kind is IndexNowKind. func (IndexNowArgs) Kind() string { return IndexNowKind } // keyPattern is the form of an IndexNow key: 8 to 128 letters, digits and // hyphens (indexnow.org documentation). var keyPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,128}$`) // ValidKey reports whether k may be an IndexNow key. func ValidKey(k string) bool { return keyPattern.MatchString(k) } // GenerateKey returns a new IndexNow key: 32 lowercase hex characters from // the system random source. func GenerateKey() (string, error) { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "", fmt.Errorf("golem15.sitemap: indexnow key: %w", err) } return hex.EncodeToString(b[:]), nil } // KeyFileName is the name of the key file of key: .txt, served at the // site root by the SEO layer (from GET /_sitemap/indexnow/.txt); "" // when key is not a ValidKey. func KeyFileName(key string) string { if !ValidKey(key) { return "" } return key + ".txt" } // ActiveKey is the key of settings while IndexNow is on, "" otherwise. func ActiveKey(s models.Settings) string { if !s.IndexNowEnabled { return "" } if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) { return k } return "" } // Production reports whether app runs in the production environment; no // IndexNow request is sent anywhere else (T-33-15). func Production(app *backpack.App) bool { return app != nil && app.Config != nil && app.Config.Environment() == "production" } // hostOf is the lowercase host of an absolute http(s) URL, "" otherwise. func hostOf(u string) string { parsed, err := url.Parse(u) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { return "" } return strings.ToLower(parsed.Hostname()) } // SiteURLs keeps the addresses on the host of site, each once, in order: // IndexNow refuses (422) a request with an address of another host. func SiteURLs(site string, urls []string) []string { host := hostOf(site) if host == "" { return nil } seen := map[string]bool{} out := []string{} for _, u := range urls { if u == "" || seen[u] || hostOf(u) != host { continue } seen[u] = true out = append(out, u) } return out } // NotifyIndexNow queues the IndexNow job for what a rebuild changed: its // added, changed and removed addresses on the site's host, in requests of // at most MaxIndexNowURLs. Nothing is queued outside production, while // IndexNow is off or has no key, without a site address or when nothing // changed. func NotifyIndexNow(ctx context.Context, app *backpack.App, db *gorm.DB, res RebuildResult) error { if !Production(app) { return nil } settings, err := LoadSettings(db) if err != nil { return err } if ActiveKey(settings) == "" { return nil } all := make([]string, 0, len(res.Added)+len(res.Changed)+len(res.Removed)) all = append(append(append(all, res.Added...), res.Changed...), res.Removed...) urls := SiteURLs(SiteURL(app, settings), all) if len(urls) == 0 { return nil } m, err := conga.From(app) if err != nil { return err } for start := 0; start < len(urls); start += MaxIndexNowURLs { end := min(start+MaxIndexNowURLs, len(urls)) if err := m.Enqueue(ctx, db, IndexNowArgs{URLs: urls[start:end]}, conga.EnqueueOpts{}); err != nil { return err } } return nil } // indexNowBody is the JSON of an IndexNow request. keyLocation is not sent: // the key file is served at the site root, which covers every address of // the host (P33-14). type indexNowBody struct { Host string `json:"host"` Key string `json:"key"` URLList []string `json:"urlList"` } // errIndexNowRetry is a retryable IndexNow answer. type errIndexNowRetry struct{ status int } func (e errIndexNowRetry) Error() string { return fmt.Sprintf("golem15.sitemap: indexnow answered %d; retrying", e.status) } // SendIndexNow runs one IndexNow job: it posts {host, key, urlList} to // IndexNowEndpoint through a fetchguard client that reaches that host only // (HTTPS, no redirects) and records the answer (indexnow_last_at, _count, // _status). 200 and 202 are success; 400, 403 and 422 are recorded and not // retried; 429 and 5xx are recorded and returned as an error, so the queue // retries the job. Outside production, while IndexNow is off or without a // site address nothing is sent; addresses of another host never are. func SendIndexNow(ctx context.Context, app *backpack.App, args IndexNowArgs) error { if !Production(app) { pluginLog.Info("golem15.sitemap: indexnow skipped outside production", "urls", len(args.URLs)) return nil } db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return ErrNoDatabase } db = db.WithContext(ctx) settings, err := LoadSettings(db) if err != nil { return err } key := ActiveKey(settings) site := SiteURL(app, settings) urls := SiteURLs(site, args.URLs) if key == "" || len(urls) == 0 { return nil } if len(urls) > MaxIndexNowURLs { urls = urls[:MaxIndexNowURLs] } client, err := fetchguard.NewClient(fetchguard.Policy{ Mode: fetchguard.AllowHostsMode, AllowHosts: []string{IndexNowHost}, Timeout: indexNowTimeout, }, app.Config) if err != nil { return err } res, err := client.PostJSON(ctx, IndexNowEndpoint, nil, indexNowBody{Host: hostOf(site), Key: key, URLList: urls}) if err != nil { return err } status := res.StatusCode if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_last_at = ?, indexnow_last_count = ?, indexnow_last_status = ? WHERE id = 1`, time.Now().UTC(), len(urls), status).Error; err != nil { return err } switch { case status == http.StatusOK || status == http.StatusAccepted: pluginLog.Info("golem15.sitemap: indexnow notified", "urls", len(urls), "status", status) return nil case status == http.StatusTooManyRequests || status >= 500: pluginLog.Warn("golem15.sitemap: indexnow will retry", "urls", len(urls), "status", status) return errIndexNowRetry{status: status} default: pluginLog.Warn("golem15.sitemap: indexnow refused", "urls", len(urls), "status", status) return nil } } // EnsureKey gives settings row 1 an IndexNow key on save: while enabled, // a new key is generated when none is stored or rotate is set (the // indexnow_new_key checkbox), so the old key file answers 404 at once. It // returns the stored key ("" while disabled without one). func EnsureKey(db *gorm.DB, enabled, rotate bool) (string, error) { settings, err := LoadSettings(db) if err != nil { return "", err } current := strings.TrimSpace(models.Text(settings.IndexNowKey)) if !enabled || (ValidKey(current) && !rotate) { return current, nil } key, err := GenerateKey() if err != nil { return "", err } if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_key = ? WHERE id = 1`, key).Error; err != nil { return "", err } return key, nil } // IndexNowRetryAt is when the queue next tries a pending IndexNow job // (after a 429 or 5xx answer), nil when none is pending. func IndexNowRetryAt(db *gorm.DB) *time.Time { var at []time.Time if err := db.Raw(`SELECT scheduled_at FROM river_job WHERE kind = ? AND state IN ('retryable', 'scheduled', 'available') ORDER BY scheduled_at ASC LIMIT 1`, IndexNowKind).Scan(&at).Error; err != nil || len(at) == 0 { return nil } t := at[0].UTC() return &t } // IndexNowStatus is the state the S2 partial and the management API show: // the key and its public URL (empty before a key exists), the last // notification (nil before the first) and, after a 429, the next attempt. type IndexNowStatus struct { Enabled bool `json:"enabled"` Key string `json:"key"` KeyURL string `json:"key_url"` LastAt *time.Time `json:"last_at"` LastCount int `json:"last_count"` LastStatus int `json:"last_status"` RetryAt *time.Time `json:"retry_at"` } // LoadIndexNowStatus reads the IndexNow state of settings. func LoadIndexNowStatus(db *gorm.DB, app *backpack.App, s models.Settings) IndexNowStatus { out := IndexNowStatus{Enabled: s.IndexNowEnabled, LastAt: s.IndexNowLastAt} if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) { out.Key = k out.KeyURL = SiteURL(app, s) + "/" + KeyFileName(k) } if s.IndexNowLastCount != nil { out.LastCount = *s.IndexNowLastCount } if s.IndexNowLastStatus != nil { out.LastStatus = *s.IndexNowLastStatus } if out.LastStatus == http.StatusTooManyRequests { out.RetryAt = IndexNowRetryAt(db) } return out }