fix: IndexNow hears of deleted and edited records

RecordDeleted marks the listed rows it excludes and RecordSaved the rows
whose lastmod it moves (indexnow_pending_at); the next rebuild sends
them to IndexNow as removed or changed and clears the marks. RecordSaved
checks for a nil event before reading it. TestIndexNow now deletes and
edits through the record events, as the admin does.

classes/lastmod.go

@@ -106,12 +106,16 @@ func RealChanges(changes []pact.RecordChange, ignore map[string]bool) Changed {
 // RecordSaved handles a saved record of a content type (D-30): when the
 // save changed an attribute outside golem15.sitemap.lastmod_ignore, the
 // lastmod of the record's rows moves to now (every locale's for a change
-// on the record's own row, else only the changed locales'), and a
-// debounced rebuild is queued. A model no content type declares is
-// ignored.
+// on the record's own row, else only the changed locales'), the same rows
+// are marked for IndexNow (indexnow_pending_at, D-31: the next rebuild
+// sends the listed ones as changed), and a debounced rebuild is queued. A
+// model no content type declares, or a nil event, is ignored.
 func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) error {
+	if ev == nil {
+		return nil
+	}
 	typ, db, ok := contentTypeOf(app, ev.Model)
-	if !ok || ev == nil {
+	if !ok {
 		return nil
 	}
 	changed := RealChanges(ev.Changes, LastmodIgnore(app.Config))
@@ -128,14 +132,16 @@ func RecordSaved(ctx context.Context, app *backpack.App, ev *pact.RecordSaved) e
 		}
 		q = q.Where("locale IN ?", locales)
 	}
-	if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now}).Error; err != nil {
+	if err := q.Updates(map[string]any{"lastmod": now, "updated_at": now, "indexnow_pending_at": now}).Error; err != nil {
 		return err
 	}
 	return RequestDebouncedRebuild(ctx, app, db)
 }
 
 // RecordDeleted handles a deleted record of a content type: its rows leave
-// the files at once (reason deleted, their file numbers kept as holes) and
+// the files at once (reason deleted, their file numbers kept as holes),
+// those that were listed are marked for IndexNow in the same update
+// (indexnow_pending_at, D-31: the next rebuild sends them as removed), and
 // a debounced rebuild is queued.
 func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDeleted) error {
 	if ev == nil {
@@ -149,7 +155,11 @@ func RecordDeleted(ctx context.Context, app *backpack.App, ev *pact.RecordDelete
 	now := time.Now().UTC()
 	if err := db.Model(&models.URL{}).
 		Where("type_key = ? AND record_id = ? AND excluded_reason <> ?", typ, ev.ID, models.ReasonDeleted).
-		Updates(map[string]any{"excluded_reason": models.ReasonDeleted, "updated_at": now}).Error; err != nil {
+		Updates(map[string]any{
+			"excluded_reason":     models.ReasonDeleted,
+			"updated_at":          now,
+			"indexnow_pending_at": gorm.Expr("CASE WHEN excluded_reason = '' THEN ?::timestamptz ELSE indexnow_pending_at END", now),
+		}).Error; err != nil {
 		return err
 	}
 	return RequestDebouncedRebuild(ctx, app, db)

classes/lastmod_test.go

@@ -100,3 +100,14 @@ func testConfig(t *testing.T) *compass.Config {
 	}
 	return cfg
 }
+
+// TestRecordSavedNilEvent pins L2 IN-01: a nil event is ignored before it
+// is read.
+func TestRecordSavedNilEvent(t *testing.T) {
+	if err := RecordSaved(t.Context(), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := RecordDeleted(t.Context(), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+}

classes/rebuild.go

@@ -119,7 +119,9 @@ type collected struct {
 // leaves its rows in the table with a reason, as a hole; only a changed
 // file size renumbers the files (D-30). In production, with IndexNow on,
 // the added, changed and removed addresses are queued for IndexNow after
-// the commit (NotifyIndexNow).
+// the commit (NotifyIndexNow), with the rows the record listeners marked
+// (indexnow_pending_at): a deleted record's addresses as removed, an
+// edited record's listed addresses as changed.
 func Rebuild(ctx context.Context, app *backpack.App) (RebuildResult, error) {
 	var res RebuildResult
 	if app == nil {
@@ -424,6 +426,11 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 	dirty := map[uint64]bool{}
 	var created []*models.URL
 	matched := map[slotKey]bool{}
+	// pending are the rows the record listeners marked for IndexNow
+	// (P33-62), with the mark this rebuild read; notified the addresses
+	// already in Added or Changed.
+	pending := map[uint64]time.Time{}
+	notified := map[string]bool{}
 	for _, w := range found.rows {
 		matched[w.key] = true
 		r, ok := byKey[w.key]
@@ -476,10 +483,21 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 		switch {
 		case listed && !wasListed:
 			res.Added = append(res.Added, Absolute(site, r.URL))
+			notified[r.URL] = true
 		case !listed && wasListed:
 			res.Removed = append(res.Removed, Absolute(site, before.URL))
 		case listed && before.URL != r.URL:
 			res.Changed = append(res.Changed, Absolute(site, r.URL))
+			notified[r.URL] = true
+		}
+		if r.IndexNowPendingAt != nil {
+			pending[r.ID] = *r.IndexNowPendingAt
+			if listed && !notified[r.URL] {
+				// A content edit that kept the address (RecordSaved
+				// moved its lastmod).
+				res.Changed = append(res.Changed, Absolute(site, r.URL))
+				notified[r.URL] = true
+			}
 		}
 		if rowChanged(before, *r) {
 			dirty[r.ID] = true
@@ -495,6 +513,14 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 		if found.unpublished[slotKey{r.TypeKey, r.RecordID, ""}] {
 			reason = models.ReasonUnpublished
 		}
+		if r.IndexNowPendingAt != nil {
+			// RecordDeleted already excluded the row; it was listed when
+			// it marked it, so IndexNow hears of it as removed.
+			pending[r.ID] = *r.IndexNowPendingAt
+			if r.ExcludedReason != "" {
+				res.Removed = append(res.Removed, Absolute(site, r.URL))
+			}
+		}
 		if r.ExcludedReason == reason {
 			continue
 		}
@@ -574,6 +600,13 @@ func apply(tx *gorm.DB, found collected, settings models.Settings, site string)
 			return res, err
 		}
 	}
+	// The IndexNow marks this rebuild read are cleared; a mark set after
+	// the read (a later save) survives for the next rebuild.
+	for id, at := range pending {
+		if err := tx.Exec(`UPDATE golem15_sitemap_urls SET indexnow_pending_at = NULL WHERE id = ? AND indexnow_pending_at <= ?`, id, at).Error; err != nil {
+			return res, err
+		}
+	}
 	if err := tx.Exec(`UPDATE golem15_sitemap_settings SET numbered_size = ?, built_at = ? WHERE id = 1`, size, now).Error; err != nil {
 		return res, err
 	}