diff --git a/README.md b/README.md index aa595b1..b0f5e00 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ The News sources setting (`news_sources`) narrows the list. It takes the values [IndexNow](https://www.indexnow.org/documentation) tells Bing, Yandex and the other participating engines about changed addresses at once; Google does not use it. - **Key.** Saving the settings with IndexNow on generates a key of 32 hex characters from `crypto/rand` (`classes.GenerateKey`, `classes.EnsureKey`). The "Generate a new key on save" checkbox (`indexnow_new_key`, no column) replaces it, and the old key file answers 404 at once. The key file `.txt` is served at `/_sitemap/indexnow/.txt` and belongs at the site root, which covers every address of the host, so no `keyLocation` is sent. -- **Notifications.** After a rebuild commits, in the `production` environment only, with IndexNow on, a key and a site address, `classes.NotifyIndexNow` queues the conga job `golem15.sitemap.indexnow` (`classes.IndexNowArgs`) with the added, changed and removed addresses on the site's host, at most 10 000 per job. The job (`classes.SendIndexNow`) posts `{host, key, urlList}` to `https://api.indexnow.org/indexnow` through a `fetchguard` client that may reach only that host (HTTPS, no redirects, 10 s), never an address of another host. +- **Notifications.** After a rebuild commits, in the `production` environment only, with IndexNow on, a key and a site address, `classes.NotifyIndexNow` queues the conga job `golem15.sitemap.indexnow` (`classes.IndexNowArgs`) with the added, changed and removed addresses on the site's host, at most 10 000 per job. A record deleted in the admin (`pact.RecordDeleted`) is sent as removed and a content edit that keeps the address (`pact.RecordSaved` moving its lastmod) as changed: the listeners mark the rows (`indexnow_pending_at`) and the next rebuild sends and clears them; outside production the marks are cleared without a notification. The job (`classes.SendIndexNow`) posts `{host, key, urlList}` to `https://api.indexnow.org/indexnow` through a `fetchguard` client that may reach only that host (HTTPS, no redirects, 10 s), never an address of another host. - **Answers.** 200 and 202 are success. Every answer is recorded (`indexnow_last_at`, `indexnow_last_count`, `indexnow_last_status`). 400, 403 and 422 are not retried; 429 and 5xx answers and network errors fail the job, which retries up to five times. - **Status.** The IndexNow tab's status partial shows the key and its public address (or that a key is generated on save), the last notification (or that none was sent), and a callout after a 403 (the engines could not read the key file), a 422 (the addresses do not match the site address) or a 429 (with the next attempt's time while a retry is queued). diff --git a/classes/lastmod.go b/classes/lastmod.go index 30c9ac3..1f6ffcf 100644 --- a/classes/lastmod.go +++ b/classes/lastmod.go @@ -106,12 +106,16 @@ func RealChanges(changes []pact.RecordChange, ignore map[string]bool) Changed { // RecordSaved handles a saved record of a content type (D-30): when the // save changed an attribute outside golem15.sitemap.lastmod_ignore, the // lastmod of the record's rows moves to now (every locale's for a change -// on the record's own row, else only the changed locales'), and a -// debounced rebuild is queued. A model no content type declares is -// ignored. +// on the record's own row, else only the changed locales'), the same rows +// are marked for IndexNow (indexnow_pending_at, D-31: the next rebuild +// sends the listed ones as changed), and a debounced rebuild is queued. A +// model no content type declares, or a nil event, is ignored. func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) error { + if ev == nil { + return nil + } typ, db, ok := contentTypeOf(app, ev.Model) - if !ok || ev == nil { + if !ok { return nil } changed := RealChanges(ev.Changes, LastmodIgnore(app.Config)) @@ -128,14 +132,16 @@ func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) e } q = q.Where("locale IN ?", locales) } - if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now}).Error; err != nil { + if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now, "indexnow_pending_at": now}).Error; err != nil { return err } return RequestDebouncedRebuild(ctx, app, db) } // RecordDeleted handles a deleted record of a content type: its rows leave -// the files at once (reason deleted, their file numbers kept as holes) and +// the files at once (reason deleted, their file numbers kept as holes), +// those that were listed are marked for IndexNow in the same update +// (indexnow_pending_at, D-31: the next rebuild sends them as removed), and // a debounced rebuild is queued. func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDeleted) error { if ev == nil { @@ -149,7 +155,11 @@ func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDelete now := time.Now().UTC() if err := db.Model(&models.URL{}). Where("type_key = ? AND record_id = ? AND excluded_reason <> ?", typ, ev.ID, models.ReasonDeleted). - Updates(map[string]any{"excluded_reason": models.ReasonDeleted, "updated_at": now}).Error; err != nil { + Updates(map[string]any{ + "excluded_reason": models.ReasonDeleted, + "updated_at": now, + "indexnow_pending_at": gorm.Expr("CASE WHEN excluded_reason = '' THEN ?::timestamptz ELSE indexnow_pending_at END", now), + }).Error; err != nil { return err } return RequestDebouncedRebuild(ctx, app, db) diff --git a/classes/lastmod_test.go b/classes/lastmod_test.go index 5e47944..b686d4b 100644 --- a/classes/lastmod_test.go +++ b/classes/lastmod_test.go @@ -100,3 +100,14 @@ func testConfig(t *testing.T) *compass.Config { } return cfg } + +// TestRecordSavedNilEvent pins L2 IN-01: a nil event is ignored before it +// is read. +func TestRecordSavedNilEvent(t *testing.T) { + if err := RecordSaved(t.Context(), nil, nil); err != nil { + t.Fatal(err) + } + if err := RecordDeleted(t.Context(), nil, nil); err != nil { + t.Fatal(err) + } +} diff --git a/classes/rebuild.go b/classes/rebuild.go index 92aa3c9..47b21ca 100644 --- a/classes/rebuild.go +++ b/classes/rebuild.go @@ -119,7 +119,9 @@ type collected struct { // leaves its rows in the table with a reason, as a hole; only a changed // file size renumbers the files (D-30). In production, with IndexNow on, // the added, changed and removed addresses are queued for IndexNow after -// the commit (NotifyIndexNow). +// the commit (NotifyIndexNow), with the rows the record listeners marked +// (indexnow_pending_at): a deleted record's addresses as removed, an +// edited record's listed addresses as changed. func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) { var res RebuildResult if app == nil { @@ -424,6 +426,11 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) dirty := map[uint64]bool{} var created []*models.URL matched := map[slotKey]bool{} + // pending are the rows the record listeners marked for IndexNow + // (P33-62), with the mark this rebuild read; notified the addresses + // already in Added or Changed. + pending := map[uint64]time.Time{} + notified := map[string]bool{} for _, w := range found.rows { matched[w.key] = true r, ok := byKey[w.key] @@ -476,10 +483,21 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) switch { case listed && !wasListed: res.Added = append(res.Added, Absolute(site, r.URL)) + notified[r.URL] = true case !listed && wasListed: res.Removed = append(res.Removed, Absolute(site, before.URL)) case listed && before.URL != r.URL: res.Changed = append(res.Changed, Absolute(site, r.URL)) + notified[r.URL] = true + } + if r.IndexNowPendingAt != nil { + pending[r.ID] = *r.IndexNowPendingAt + if listed && !notified[r.URL] { + // A content edit that kept the address (RecordSaved + // moved its lastmod). + res.Changed = append(res.Changed, Absolute(site, r.URL)) + notified[r.URL] = true + } } if rowChanged(before, *r) { dirty[r.ID] = true @@ -495,6 +513,14 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] { reason = models.ReasonUnpublished } + if r.IndexNowPendingAt != nil { + // RecordDeleted already excluded the row; it was listed when + // it marked it, so IndexNow hears of it as removed. + pending[r.ID] = *r.IndexNowPendingAt + if r.ExcludedReason != "" { + res.Removed = append(res.Removed, Absolute(site, r.URL)) + } + } if r.ExcludedReason == reason { continue } @@ -574,6 +600,13 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string) return res, err } } + // The IndexNow marks this rebuild read are cleared; a mark set after + // the read (a later save) survives for the next rebuild. + for id, at := range pending { + if err := tx.Exec(`UPDATE golem15_sitemap_urls SET indexnow_pending_at = NULL WHERE id = ? AND indexnow_pending_at <= ?`, id, at).Error; err != nil { + return res, err + } + } if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil { return res, err } diff --git a/content_fixture_test.go b/content_fixture_test.go index 14f81b0..ecb5761 100644 --- a/content_fixture_test.go +++ b/content_fixture_test.go @@ -161,3 +161,24 @@ func init() { func page1(id int, slug string, mod time.Time) fakeRecord { return fakeRecord{ID: id, Published: true, LastMod: mod, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/" + slug, Title: "Page " + slug}}} } + +// deleteRecord removes a record of typ from the store and fires +// pact.RecordDeleted for its model, as the admin does after a delete. +func (e *sitemapEnv) deleteRecord(t *testing.T, typ, model string, id int) { + t.Helper() + contentStore.remove(typ, id) + if err := e.app.Events.Fire(context.Background(), &pact.RecordDeleted{Model: model, ID: strconv.Itoa(id)}); err != nil { + t.Fatal(err) + } +} + +// editRecord replaces a record of typ in the store and fires +// pact.RecordSaved for its model with the changed attributes, as the admin +// does after a save. +func (e *sitemapEnv) editRecord(t *testing.T, typ, model string, rec fakeRecord, changes ...pact.RecordChange) { + t.Helper() + contentStore.put(typ, rec) + if err := e.app.Events.Fire(context.Background(), &pact.RecordSaved{Model: model, ID: strconv.Itoa(rec.ID), Changes: changes}); err != nil { + t.Fatal(err) + } +} diff --git a/models/url.go b/models/url.go index 60d4c60..0a950be 100644 --- a/models/url.go +++ b/models/url.go @@ -51,7 +51,10 @@ type Ref struct { // its sitemap-images file (nil while it has none). Kind is the content // type's kind (pact.ContentKindArticle and the others), PublishedAt the // record's first publication and ParentRefs its parents, for the News -// sitemap and the HTML sitemap. +// sitemap and the HTML sitemap. IndexNowPendingAt is set by the record +// listeners on a row the next rebuild sends to IndexNow (a deleted +// record's listed row, an edited record's row whose lastmod moved) and +// cleared by that rebuild. type URL struct { ID uint64 `gorm:"column:id;primaryKey"` Grp string `gorm:"column:grp"` @@ -71,6 +74,8 @@ type URL struct { ParentRefs lagoon.Jsonable[[]Ref] `gorm:"column:parent_refs"` ExcludedReason string `gorm:"column:excluded_reason"` UpdatedAt time.Time `gorm:"column:updated_at"` + // IndexNowPendingAt (P33-62). + IndexNowPendingAt *time.Time `gorm:"column:indexnow_pending_at"` } func (URL) TableName() string { return "golem15_sitemap_urls" } diff --git a/sitemap_extras_test.go b/sitemap_extras_test.go index 88f9f7c..b67bab2 100644 --- a/sitemap_extras_test.go +++ b/sitemap_extras_test.go @@ -433,14 +433,33 @@ func TestIndexNow(t *testing.T) { t.Fatalf("S2 with a key:\n%s", got) } - // One new and one removed address: one job with both. + // One new address and one record deleted as the admin deletes it + // (pact.RecordDeleted): one job with both (L2 WR-01). contentStore.put(postType, post(2, "b", "en", at)) - contentStore.remove(postType, 1) + e.deleteRecord(t, postType, postType, 1) e.rebuild(t) q := e.queuedIndexNow(t) if len(q) != 1 || strings.Join(q[0], " ") != "https://api.example.test/blog/b https://api.example.test/blog/a" { t.Fatalf("queued = %v", q) } + // A content-only edit (pact.RecordSaved, the address kept) is sent as + // changed; a rebuild with nothing new queues nothing. + edited := post(2, "b", "en", at) + edited.Locales[0].Title = "Post b, edited" + e.editRecord(t, postType, postType, edited, pact.RecordChange{Attribute: "title"}) + e.rebuild(t) + if got := e.queuedIndexNow(t); len(got) != 2 || strings.Join(got[1], " ") != "https://api.example.test/blog/b" { + t.Fatalf("queued after the edit = %v", got) + } + e.rebuild(t) + if got := e.queuedIndexNow(t); len(got) != 2 { + t.Fatalf("a rebuild with nothing new queued %v", got[2:]) + } + var marks int64 + e.gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_urls WHERE indexnow_pending_at IS NOT NULL`).Scan(&marks) + if marks != 0 { + t.Fatalf("%d IndexNow marks left after the rebuild", marks) + } engine := &fakeIndexNow{status: http.StatusAccepted} ctx := fetchguard.WithTransport(t.Context(), engine) @@ -528,10 +547,19 @@ func TestIndexNowStaging(t *testing.T) { e := bootJournal(t) e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`) contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC))) + contentStore.put(postType, post(2, "b", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC))) + e.rebuild(t) + e.deleteRecord(t, postType, postType, 1) + e.editRecord(t, postType, postType, post(2, "b", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)), pact.RecordChange{Attribute: "title"}) e.rebuild(t) if q := e.queuedIndexNow(t); len(q) != 0 { t.Fatalf("queued outside production: %v", q) } + var marks int64 + e.gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_urls WHERE indexnow_pending_at IS NOT NULL`).Scan(&marks) + if marks != 0 { + t.Fatalf("%d IndexNow marks left outside production", marks) + } engine := &fakeIndexNow{status: http.StatusOK} if err := e.job(t, classes.IndexNowKind).Work(fetchguard.WithTransport(t.Context(), engine), classes.IndexNowArgs{URLs: []string{"https://api.example.test/blog/a"}}); err != nil { t.Fatal(err) diff --git a/updates/202611040001_add_golem15_sitemap_urls_indexnow_pending.go b/updates/202611040001_add_golem15_sitemap_urls_indexnow_pending.go new file mode 100644 index 0000000..19496aa --- /dev/null +++ b/updates/202611040001_add_golem15_sitemap_urls_indexnow_pending.go @@ -0,0 +1,27 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// addIndexNowPending adds indexnow_pending_at to golem15_sitemap_urls +// (P33-62): the record listeners mark the rows of a deleted record that +// were listed, and the rows of an edited record whose lastmod moved, so +// the next rebuild sends them to IndexNow as removed or changed and clears +// the mark. +var addIndexNowPending = []*gormigrate.Migration{ + { + ID: "202611040001_add_golem15_sitemap_urls_indexnow_pending", + Migrate: func(tx *gorm.DB) error { + return tx.Exec(`ALTER TABLE golem15_sitemap_urls ADD COLUMN indexnow_pending_at TIMESTAMPTZ NULL`).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`ALTER TABLE golem15_sitemap_urls DROP COLUMN IF EXISTS indexnow_pending_at`).Error + }, + }, +} + +func init() { + Register(addIndexNowPending...) +}