sm-sitemap-plugin/classes/indexnow.go

package classes

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/conga"
	"git.golem15.com/golem15/summercms/modules/fetchguard"
	"gorm.io/gorm"
)

// IndexNow (D-31, P33-14): after a rebuild the added, changed and removed
// addresses are posted to the shared IndexNow endpoint, which hands them to
// every participating engine. The endpoint is fixed, not configurable: the
// outbound client may reach that one host only (T-33-13).
const (
	IndexNowKind     = "golem15.sitemap.indexnow"
	IndexNowHost     = "api.indexnow.org"
	IndexNowEndpoint = "https://" + IndexNowHost + "/indexnow"
	// MaxIndexNowURLs is the most addresses one request carries.
	MaxIndexNowURLs = 10000
	// indexNowTimeout bounds one request.
	indexNowTimeout = 10 * time.Second
	// IndexNowAttempts is the attempt limit of the job: an answer of 429 or
	// 5xx, or a network error, is retried by the queue up to this many times.
	IndexNowAttempts = 5
)

// IndexNowArgs are the arguments of the IndexNow job: the absolute
// addresses of one request (at most MaxIndexNowURLs).
type IndexNowArgs struct {
	URLs []string `json:"urls"`
}

// Kind is IndexNowKind.
func (IndexNowArgs) Kind() string { return IndexNowKind }

// keyPattern is the form of an IndexNow key: 8 to 128 letters, digits and
// hyphens (indexnow.org documentation).
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9-]{8,128}$`)

// ValidKey reports whether k may be an IndexNow key.
func ValidKey(k string) bool { return keyPattern.MatchString(k) }

// GenerateKey returns a new IndexNow key: 32 lowercase hex characters from
// the system random source.
func GenerateKey() (string, error) {
	var b [16]byte
	if _, err := rand.Read(b[:]); err != nil {
		return "", fmt.Errorf("golem15.sitemap: indexnow key: %w", err)
	}
	return hex.EncodeToString(b[:]), nil
}

// KeyFileName is the name of the key file of key: <key>.txt, served at the
// site root by the SEO layer (from GET /_sitemap/indexnow/<key>.txt); ""
// when key is not a ValidKey.
func KeyFileName(key string) string {
	if !ValidKey(key) {
		return ""
	}
	return key + ".txt"
}

// ActiveKey is the key of settings while IndexNow is on, "" otherwise.
func ActiveKey(s models.Settings) string {
	if !s.IndexNowEnabled {
		return ""
	}
	if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
		return k
	}
	return ""
}

// Production reports whether app runs in the production environment; no
// IndexNow request is sent anywhere else (T-33-15).
func Production(app *backpack.App) bool {
	return app != nil && app.Config != nil && app.Config.Environment() == "production"
}

// hostOf is the lowercase host of an absolute http(s) URL, "" otherwise.
func hostOf(u string) string {
	parsed, err := url.Parse(u)
	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
		return ""
	}
	return strings.ToLower(parsed.Hostname())
}

// SiteURLs keeps the addresses on the host of site, each once, in order:
// IndexNow refuses (422) a request with an address of another host.
func SiteURLs(site string, urls []string) []string {
	host := hostOf(site)
	if host == "" {
		return nil
	}
	seen := map[string]bool{}
	out := []string{}
	for _, u := range urls {
		if u == "" || seen[u] || hostOf(u) != host {
			continue
		}
		seen[u] = true
		out = append(out, u)
	}
	return out
}

// NotifyIndexNow queues the IndexNow job for what a rebuild changed: its
// added, changed and removed addresses on the site's host, in requests of
// at most MaxIndexNowURLs. Nothing is queued outside production, while
// IndexNow is off or has no key, without a site address or when nothing
// changed.
func NotifyIndexNow(ctx context.Context, app *backpack.App, db *gorm.DB, res RebuildResult) error {
	if !Production(app) {
		return nil
	}
	settings, err := LoadSettings(db)
	if err != nil {
		return err
	}
	if ActiveKey(settings) == "" {
		return nil
	}
	all := make([]string, 0, len(res.Added)+len(res.Changed)+len(res.Removed))
	all = append(append(append(all, res.Added...), res.Changed...), res.Removed...)
	urls := SiteURLs(SiteURL(app, settings), all)
	if len(urls) == 0 {
		return nil
	}
	m, err := conga.From(app)
	if err != nil {
		return err
	}
	for start := 0; start < len(urls); start += MaxIndexNowURLs {
		end := min(start+MaxIndexNowURLs, len(urls))
		if err := m.Enqueue(ctx, db, IndexNowArgs{URLs: urls[start:end]}, conga.EnqueueOpts{}); err != nil {
			return err
		}
	}
	return nil
}

// indexNowBody is the JSON of an IndexNow request. keyLocation is not sent:
// the key file is served at the site root, which covers every address of
// the host (P33-14).
type indexNowBody struct {
	Host    string   `json:"host"`
	Key     string   `json:"key"`
	URLList []string `json:"urlList"`
}

// errIndexNowRetry is a retryable IndexNow answer.
type errIndexNowRetry struct{ status int }

func (e errIndexNowRetry) Error() string {
	return fmt.Sprintf("golem15.sitemap: indexnow answered %d; retrying", e.status)
}

// SendIndexNow runs one IndexNow job: it posts {host, key, urlList} to
// IndexNowEndpoint through a fetchguard client that reaches that host only
// (HTTPS, no redirects) and records the answer (indexnow_last_at, _count,
// _status). 200 and 202 are success; 400, 403 and 422 are recorded and not
// retried; 429 and 5xx are recorded and returned as an error, so the queue
// retries the job. Outside production, while IndexNow is off or without a
// site address nothing is sent; addresses of another host never are.
func SendIndexNow(ctx context.Context, app *backpack.App, args IndexNowArgs) error {
	if !Production(app) {
		pluginLog.Info("golem15.sitemap: indexnow skipped outside production", "urls", len(args.URLs))
		return nil
	}
	db, ok := app.Lookup[*gorm.DB]()
	if !ok || db == nil {
		return ErrNoDatabase
	}
	db = db.WithContext(ctx)
	settings, err := LoadSettings(db)
	if err != nil {
		return err
	}
	key := ActiveKey(settings)
	site := SiteURL(app, settings)
	urls := SiteURLs(site, args.URLs)
	if key == "" || len(urls) == 0 {
		return nil
	}
	if len(urls) > MaxIndexNowURLs {
		urls = urls[:MaxIndexNowURLs]
	}
	client, err := fetchguard.NewClient(fetchguard.Policy{
		Mode:       fetchguard.AllowHostsMode,
		AllowHosts: []string{IndexNowHost},
		Timeout:    indexNowTimeout,
	}, app.Config)
	if err != nil {
		return err
	}
	res, err := client.PostJSON(ctx, IndexNowEndpoint, nil, indexNowBody{Host: hostOf(site), Key: key, URLList: urls})
	if err != nil {
		return err
	}
	status := res.StatusCode
	if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_last_at = ?, indexnow_last_count = ?, indexnow_last_status = ? WHERE id = 1`,
		time.Now().UTC(), len(urls), status).Error; err != nil {
		return err
	}
	switch {
	case status == http.StatusOK || status == http.StatusAccepted:
		pluginLog.Info("golem15.sitemap: indexnow notified", "urls", len(urls), "status", status)
		return nil
	case status == http.StatusTooManyRequests || status >= 500:
		pluginLog.Warn("golem15.sitemap: indexnow will retry", "urls", len(urls), "status", status)
		return errIndexNowRetry{status: status}
	default:
		pluginLog.Warn("golem15.sitemap: indexnow refused", "urls", len(urls), "status", status)
		return nil
	}
}

// EnsureKey gives settings row 1 an IndexNow key on save: while enabled,
// a new key is generated when none is stored or rotate is set (the
// indexnow_new_key checkbox), so the old key file answers 404 at once. It
// returns the stored key ("" while disabled without one).
func EnsureKey(db *gorm.DB, enabled, rotate bool) (string, error) {
	settings, err := LoadSettings(db)
	if err != nil {
		return "", err
	}
	current := strings.TrimSpace(models.Text(settings.IndexNowKey))
	if !enabled || (ValidKey(current) && !rotate) {
		return current, nil
	}
	key, err := GenerateKey()
	if err != nil {
		return "", err
	}
	if err := db.Exec(`UPDATE golem15_sitemap_settings SET indexnow_key = ? WHERE id = 1`, key).Error; err != nil {
		return "", err
	}
	return key, nil
}

// IndexNowRetryAt is when the queue next tries a pending IndexNow job
// (after a 429 or 5xx answer), nil when none is pending.
func IndexNowRetryAt(db *gorm.DB) *time.Time {
	var at []time.Time
	if err := db.Raw(`SELECT scheduled_at FROM river_job WHERE kind = ? AND state IN ('retryable', 'scheduled', 'available') ORDER BY scheduled_at ASC LIMIT 1`, IndexNowKind).Scan(&at).Error; err != nil || len(at) == 0 {
		return nil
	}
	t := at[0].UTC()
	return &t
}

// IndexNowStatus is the state the S2 partial and the management API show:
// the key and its public URL (empty before a key exists), the last
// notification (nil before the first) and, after a 429, the next attempt.
type IndexNowStatus struct {
	Enabled    bool       `json:"enabled"`
	Key        string     `json:"key"`
	KeyURL     string     `json:"key_url"`
	LastAt     *time.Time `json:"last_at"`
	LastCount  int        `json:"last_count"`
	LastStatus int        `json:"last_status"`
	RetryAt    *time.Time `json:"retry_at"`
}

// LoadIndexNowStatus reads the IndexNow state of settings.
func LoadIndexNowStatus(db *gorm.DB, app *backpack.App, s models.Settings) IndexNowStatus {
	out := IndexNowStatus{Enabled: s.IndexNowEnabled, LastAt: s.IndexNowLastAt}
	if k := strings.TrimSpace(models.Text(s.IndexNowKey)); ValidKey(k) {
		out.Key = k
		out.KeyURL = SiteURL(app, s) + "/" + KeyFileName(k)
	}
	if s.IndexNowLastCount != nil {
		out.LastCount = *s.IndexNowLastCount
	}
	if s.IndexNowLastStatus != nil {
		out.LastStatus = *s.IndexNowLastStatus
	}
	if out.LastStatus == http.StatusTooManyRequests {
		out.RetryAt = IndexNowRetryAt(db)
	}
	return out
}