fix: IndexNow hears of deleted and edited records

RecordDeleted marks the listed rows it excludes and RecordSaved the rows
whose lastmod it moves (indexnow_pending_at); the next rebuild sends
them to IndexNow as removed or changed and clears the marks. RecordSaved
checks for a nil event before reading it. TestIndexNow now deletes and
edits through the record events, as the admin does.

README.md

@@ -129,7 +129,7 @@ The News sources setting (`news_sources`) narrows the list. It takes the values
 [IndexNow](https://www.indexnow.org/documentation) tells Bing, Yandex and the other participating engines about changed addresses at once; Google does not use it.
 
 - **Key.** Saving the settings with IndexNow on generates a key of 32 hex characters from `crypto/rand` (`classes.GenerateKey`, `classes.EnsureKey`). The "Generate a new key on save" checkbox (`indexnow_new_key`, no column) replaces it, and the old key file answers 404 at once. The key file `<key>.txt` is served at `/_sitemap/indexnow/<key>.txt` and belongs at the site root, which covers every address of the host, so no `keyLocation` is sent.
-- **Notifications.** After a rebuild commits, in the `production` environment only, with IndexNow on, a key and a site address, `classes.NotifyIndexNow` queues the conga job `golem15.sitemap.indexnow` (`classes.IndexNowArgs`) with the added, changed and removed addresses on the site's host, at most 10 000 per job. The job (`classes.SendIndexNow`) posts `{host, key, urlList}` to `https://api.indexnow.org/indexnow` through a `fetchguard` client that may reach only that host (HTTPS, no redirects, 10 s), never an address of another host.
+- **Notifications.** After a rebuild commits, in the `production` environment only, with IndexNow on, a key and a site address, `classes.NotifyIndexNow` queues the conga job `golem15.sitemap.indexnow` (`classes.IndexNowArgs`) with the added, changed and removed addresses on the site's host, at most 10 000 per job. A record deleted in the admin (`pact.RecordDeleted`) is sent as removed and a content edit that keeps the address (`pact.RecordSaved` moving its lastmod) as changed: the listeners mark the rows (`indexnow_pending_at`) and the next rebuild sends and clears them; outside production the marks are cleared without a notification. The job (`classes.SendIndexNow`) posts `{host, key, urlList}` to `https://api.indexnow.org/indexnow` through a `fetchguard` client that may reach only that host (HTTPS, no redirects, 10 s), never an address of another host.
 - **Answers.** 200 and 202 are success. Every answer is recorded (`indexnow_last_at`, `indexnow_last_count`, `indexnow_last_status`). 400, 403 and 422 are not retried; 429 and 5xx answers and network errors fail the job, which retries up to five times.
 - **Status.** The IndexNow tab's status partial shows the key and its public address (or that a key is generated on save), the last notification (or that none was sent), and a callout after a 403 (the engines could not read the key file), a 422 (the addresses do not match the site address) or a 429 (with the next attempt's time while a retry is queued).
 

classes/lastmod.go

@@ -106,12 +106,16 @@ func RealChanges(changes []pact.RecordChange, ignore map[string]bool) Changed {
 // RecordSaved handles a saved record of a content type (D-30): when the
 // save changed an attribute outside golem15.sitemap.lastmod_ignore, the
 // lastmod of the record's rows moves to now (every locale's for a change
-// on the record's own row, else only the changed locales'), and a
-// debounced rebuild is queued. A model no content type declares is
-// ignored.
+// on the record's own row, else only the changed locales'), the same rows
+// are marked for IndexNow (indexnow_pending_at, D-31: the next rebuild
+// sends the listed ones as changed), and a debounced rebuild is queued. A
+// model no content type declares, or a nil event, is ignored.
 func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) error {
+	if ev == nil {
+		return nil
+	}
 	typ, db, ok := contentTypeOf(app, ev.Model)
-	if !ok || ev == nil {
+	if !ok {
 		return nil
 	}
 	changed := RealChanges(ev.Changes, LastmodIgnore(app.Config))
@@ -128,14 +132,16 @@ func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) e
 		}
 		q = q.Where("locale IN ?", locales)
 	}
-	if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now}).Error; err != nil {
+	if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now, "indexnow_pending_at": now}).Error; err != nil {
 		return err
 	}
 	return RequestDebouncedRebuild(ctx, app, db)
 }
 
 // RecordDeleted handles a deleted record of a content type: its rows leave
-// the files at once (reason deleted, their file numbers kept as holes) and
+// the files at once (reason deleted, their file numbers kept as holes),
+// those that were listed are marked for IndexNow in the same update
+// (indexnow_pending_at, D-31: the next rebuild sends them as removed), and
 // a debounced rebuild is queued.
 func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDeleted) error {
 	if ev == nil {
@@ -149,7 +155,11 @@ func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDelete
 	now := time.Now().UTC()
 	if err := db.Model(&models.URL{}).
 		Where("type_key = ? AND record_id = ? AND excluded_reason <> ?", typ, ev.ID, models.ReasonDeleted).
-		Updates(map[string]any{"excluded_reason": models.ReasonDeleted, "updated_at": now}).Error; err != nil {
+		Updates(map[string]any{
+			"excluded_reason":     models.ReasonDeleted,
+			"updated_at":          now,
+			"indexnow_pending_at": gorm.Expr("CASE WHEN excluded_reason = '' THEN ?::timestamptz ELSE indexnow_pending_at END", now),
+		}).Error; err != nil {
 		return err
 	}
 	return RequestDebouncedRebuild(ctx, app, db)

classes/lastmod_test.go

@@ -100,3 +100,14 @@ func testConfig(t *testing.T) *compass.Config {
 	}
 	return cfg
 }
+
+// TestRecordSavedNilEvent pins L2 IN-01: a nil event is ignored before it
+// is read.
+func TestRecordSavedNilEvent(t *testing.T) {
+	if err := RecordSaved(t.Context(), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := RecordDeleted(t.Context(), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+}

classes/rebuild.go

@@ -119,7 +119,9 @@ type collected struct {
 // leaves its rows in the table with a reason, as a hole; only a changed
 // file size renumbers the files (D-30). In production, with IndexNow on,
 // the added, changed and removed addresses are queued for IndexNow after
-// the commit (NotifyIndexNow).
+// the commit (NotifyIndexNow), with the rows the record listeners marked
+// (indexnow_pending_at): a deleted record's addresses as removed, an
+// edited record's listed addresses as changed.
 func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
 	var res RebuildResult
 	if app == nil {
@@ -424,6 +426,11 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 	dirty := map[uint64]bool{}
 	var created []*models.URL
 	matched := map[slotKey]bool{}
+	// pending are the rows the record listeners marked for IndexNow
+	// (P33-62), with the mark this rebuild read; notified the addresses
+	// already in Added or Changed.
+	pending := map[uint64]time.Time{}
+	notified := map[string]bool{}
 	for _, w := range found.rows {
 		matched[w.key] = true
 		r, ok := byKey[w.key]
@@ -476,10 +483,21 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 		switch {
 		case listed && !wasListed:
 			res.Added = append(res.Added, Absolute(site, r.URL))
+			notified[r.URL] = true
 		case !listed && wasListed:
 			res.Removed = append(res.Removed, Absolute(site, before.URL))
 		case listed && before.URL != r.URL:
 			res.Changed = append(res.Changed, Absolute(site, r.URL))
+			notified[r.URL] = true
+		}
+		if r.IndexNowPendingAt != nil {
+			pending[r.ID] = *r.IndexNowPendingAt
+			if listed && !notified[r.URL] {
+				// A content edit that kept the address (RecordSaved
+				// moved its lastmod).
+				res.Changed = append(res.Changed, Absolute(site, r.URL))
+				notified[r.URL] = true
+			}
 		}
 		if rowChanged(before, *r) {
 			dirty[r.ID] = true
@@ -495,6 +513,14 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 		if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
 			reason = models.ReasonUnpublished
 		}
+		if r.IndexNowPendingAt != nil {
+			// RecordDeleted already excluded the row; it was listed when
+			// it marked it, so IndexNow hears of it as removed.
+			pending[r.ID] = *r.IndexNowPendingAt
+			if r.ExcludedReason != "" {
+				res.Removed = append(res.Removed, Absolute(site, r.URL))
+			}
+		}
 		if r.ExcludedReason == reason {
 			continue
 		}
@@ -574,6 +600,13 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 			return res, err
 		}
 	}
+	// The IndexNow marks this rebuild read are cleared; a mark set after
+	// the read (a later save) survives for the next rebuild.
+	for id, at := range pending {
+		if err := tx.Exec(`UPDATE golem15_sitemap_urls SET indexnow_pending_at = NULL WHERE id = ? AND indexnow_pending_at <= ?`, id, at).Error; err != nil {
+			return res, err
+		}
+	}
 	if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
 		return res, err
 	}

content_fixture_test.go

@@ -161,3 +161,24 @@ func init() {
 func page1(id int, slug string, mod time.Time) fakeRecord {
 	return fakeRecord{ID: id, Published: true, LastMod: mod, Locales: []pact.ContentLocale{{Locale: "en", URL: "/pages/" + slug, Title: "Page " + slug}}}
 }
+
+// deleteRecord removes a record of typ from the store and fires
+// pact.RecordDeleted for its model, as the admin does after a delete.
+func (e *sitemapEnv) deleteRecord(t *testing.T, typ, model string, id int) {
+	t.Helper()
+	contentStore.remove(typ, id)
+	if err := e.app.Events.Fire(context.Background(), &pact.RecordDeleted{Model: model, ID: strconv.Itoa(id)}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// editRecord replaces a record of typ in the store and fires
+// pact.RecordSaved for its model with the changed attributes, as the admin
+// does after a save.
+func (e *sitemapEnv) editRecord(t *testing.T, typ, model string, rec fakeRecord, changes ...pact.RecordChange) {
+	t.Helper()
+	contentStore.put(typ, rec)
+	if err := e.app.Events.Fire(context.Background(), &pact.RecordSaved{Model: model, ID: strconv.Itoa(rec.ID), Changes: changes}); err != nil {
+		t.Fatal(err)
+	}
+}

models/url.go

@@ -51,7 +51,10 @@ type Ref struct {
 // its sitemap-images file (nil while it has none). Kind is the content
 // type's kind (pact.ContentKindArticle and the others), PublishedAt the
 // record's first publication and ParentRefs its parents, for the News
-// sitemap and the HTML sitemap.
+// sitemap and the HTML sitemap. IndexNowPendingAt is set by the record
+// listeners on a row the next rebuild sends to IndexNow (a deleted
+// record's listed row, an edited record's row whose lastmod moved) and
+// cleared by that rebuild.
 type URL struct {
 	ID             uint64                       `gorm:"column:id;primaryKey"`
 	Grp            string                       `gorm:"column:grp"`
@@ -71,6 +74,8 @@ type URL struct {
 	ParentRefs     lagoon.Jsonable[[]Ref]       `gorm:"column:parent_refs"`
 	ExcludedReason string                       `gorm:"column:excluded_reason"`
 	UpdatedAt      time.Time                    `gorm:"column:updated_at"`
+	// IndexNowPendingAt (P33-62).
+	IndexNowPendingAt *time.Time `gorm:"column:indexnow_pending_at"`
 }
 
 func (URL) TableName() string { return "golem15_sitemap_urls" }

sitemap_extras_test.go

@@ -433,14 +433,33 @@ func TestIndexNow(t *testing.T) {
 		t.Fatalf("S2 with a key:\n%s", got)
 	}
 
-	// One new and one removed address: one job with both.
+	// One new address and one record deleted as the admin deletes it
+	// (pact.RecordDeleted): one job with both (L2 WR-01).
 	contentStore.put(postType, post(2, "b", "en", at))
-	contentStore.remove(postType, 1)
+	e.deleteRecord(t, postType, postType, 1)
 	e.rebuild(t)
 	q := e.queuedIndexNow(t)
 	if len(q) != 1 || strings.Join(q[0], " ") != "https://api.example.test/blog/b https://api.example.test/blog/a" {
 		t.Fatalf("queued = %v", q)
 	}
+	// A content-only edit (pact.RecordSaved, the address kept) is sent as
+	// changed; a rebuild with nothing new queues nothing.
+	edited := post(2, "b", "en", at)
+	edited.Locales[0].Title = "Post b, edited"
+	e.editRecord(t, postType, postType, edited, pact.RecordChange{Attribute: "title"})
+	e.rebuild(t)
+	if got := e.queuedIndexNow(t); len(got) != 2 || strings.Join(got[1], " ") != "https://api.example.test/blog/b" {
+		t.Fatalf("queued after the edit = %v", got)
+	}
+	e.rebuild(t)
+	if got := e.queuedIndexNow(t); len(got) != 2 {
+		t.Fatalf("a rebuild with nothing new queued %v", got[2:])
+	}
+	var marks int64
+	e.gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_urls WHERE indexnow_pending_at IS NOT NULL`).Scan(&marks)
+	if marks != 0 {
+		t.Fatalf("%d IndexNow marks left after the rebuild", marks)
+	}
 
 	engine := &fakeIndexNow{status: http.StatusAccepted}
 	ctx := fetchguard.WithTransport(t.Context(), engine)
@@ -528,10 +547,19 @@ func TestIndexNowStaging(t *testing.T) {
 	e := bootJournal(t)
 	e.exec(t, `UPDATE golem15_sitemap_settings SET indexnow_enabled = TRUE, indexnow_key = '0123456789abcdef0123456789abcdef' WHERE id = 1`)
 	contentStore.put(postType, post(1, "a", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)))
+	contentStore.put(postType, post(2, "b", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)))
+	e.rebuild(t)
+	e.deleteRecord(t, postType, postType, 1)
+	e.editRecord(t, postType, postType, post(2, "b", "en", time.Date(2026, 10, 1, 8, 0, 0, 0, time.UTC)), pact.RecordChange{Attribute: "title"})
 	e.rebuild(t)
 	if q := e.queuedIndexNow(t); len(q) != 0 {
 		t.Fatalf("queued outside production: %v", q)
 	}
+	var marks int64
+	e.gdb.Raw(`SELECT COUNT(*) FROM golem15_sitemap_urls WHERE indexnow_pending_at IS NOT NULL`).Scan(&marks)
+	if marks != 0 {
+		t.Fatalf("%d IndexNow marks left outside production", marks)
+	}
 	engine := &fakeIndexNow{status: http.StatusOK}
 	if err := e.job(t, classes.IndexNowKind).Work(fetchguard.WithTransport(t.Context(), engine), classes.IndexNowArgs{URLs: []string{"https://api.example.test/blog/a"}}); err != nil {
 		t.Fatal(err)

updates/202611040001_add_golem15_sitemap_urls_indexnow_pending.go

@@ -0,0 +1,27 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// addIndexNowPending adds indexnow_pending_at to golem15_sitemap_urls
+// (P33-62): the record listeners mark the rows of a deleted record that
+// were listed, and the rows of an edited record whose lastmod moved, so
+// the next rebuild sends them to IndexNow as removed or changed and clears
+// the mark.
+var addIndexNowPending = []*gormigrate.Migration{
+	{
+		ID: "202611040001_add_golem15_sitemap_urls_indexnow_pending",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.Exec(`ALTER TABLE golem15_sitemap_urls ADD COLUMN indexnow_pending_at TIMESTAMPTZ NULL`).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return tx.Exec(`ALTER TABLE golem15_sitemap_urls DROP COLUMN IF EXISTS indexnow_pending_at`).Error
+		},
+	},
+}
+
+func init() {
+	Register(addIndexNowPending...)
+}