feat: keep the materials index in sync and add bm:materials-reindex

- the materials admin syncs after create and update and removes after delete, once the save transaction commits (UseMaterialIndex, set at Boot) - POST and PUT /api/v1/bm/materials sync after the transaction, DELETE removes the document (its statement has no primary key, so no callback) - bm:materials-reindex [--fresh] [--if-configured] ensures the index and syncs every material; it refuses without an engine and runs nightly at 04:15 with --if-configured - TestMaterialSearchAgainstDatabase (Postgres) is written for the final verification phase
2026-11-08 19:50:50 +01:00
parent 4ffce6e6ac
commit ffd19342b2
12 changed files with 624 additions and 12 deletions
--- a/console/materials_reindex.go
+++ b/console/materials_reindex.go
@@ -0,0 +1,139 @@
+package console
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/bonfire"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+	"gorm.io/gorm"
+)
+
+// MaterialsReindexName is the name of the materials reindex command.
+const MaterialsReindexName = "bm:materials-reindex"
+
+// materialsReindexBatch is how many material ids one database read loads.
+const materialsReindexBatch = 200
+
+// errSearchNotConfigured is the refusal of a reindex without an engine.
+var errSearchNotConfigured = errors.New("jz.bm: the search engine is not configured (set search.driver and search.typesense.api_key)")
+
+// MaterialsReindexDeps are what the reindex command needs. DB returns the
+// application database, published on the app (Sync skips silently when no
+// database is published), and done, which closes a pool DB opened. Index
+// reaches the beachcomber service.
+type MaterialsReindexDeps struct {
+	DB    func(ctx context.Context) (*gorm.DB, func(), error)
+	Index *classes.MaterialIndexer
+}
+
+// MaterialsReindexCommand returns bm:materials-reindex: it ensures the
+// materials index (dropping it first with --fresh) and syncs every
+// material. It refuses with an error when the engine is not configured,
+// unless --if-configured, which the nightly schedule passes, makes that a
+// notice.
+func MaterialsReindexCommand(deps MaterialsReindexDeps) bonfire.Command {
+	return bonfire.Command{
+		Name:        MaterialsReindexName,
+		Description: "Sync every material into the search index",
+		Long: `Upserts the search document of every material, drafts and private ones
+included (visibility is enforced in SQL, never by the index), through the same
+path as the sync after a save.
+
+--fresh first drops the collection and recreates it from the material schema
+(models.Material.SearchIndexSchema). Use it on first deploy and after the
+schema changed; search finds nothing while the collection is being refilled.
+
+The command refuses, and sends nothing, when the search engine is not
+configured (search.driver null, or no search.typesense.api_key).
+--if-configured makes that a notice and a zero exit; the nightly schedule
+runs the command with it. It ends with "Reindexed: N synced, M failed" and
+exits non-zero when any material failed.`,
+		Flags: []bonfire.Flag{
+			{Name: "fresh", Description: "Drop and recreate the collection from the schema before syncing", Bare: true},
+			{Name: "if-configured", Description: "Exit 0 with a notice when the search engine is not configured", Bare: true},
+		},
+		Examples: []bonfire.Example{
+			{Command: MaterialsReindexName, Description: "Sync every material into the existing collection"},
+			{Command: MaterialsReindexName + " --fresh", Description: "Recreate the collection from the schema, then sync every material"},
+		},
+		Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
+			return reindexMaterials(ctx, deps, flagOn(in, "fresh"), flagOn(in, "if-configured"), out)
+		},
+	}
+}
+
+func reindexMaterials(ctx context.Context, deps MaterialsReindexDeps, fresh, ifConfigured bool, out bonfire.Output) error {
+	var svc *beachcomber.Service
+	if deps.Index != nil && deps.Index.Service != nil {
+		svc = deps.Index.Service()
+	}
+	engine := svc.Engine()
+	if engine == nil || !engine.Configured() {
+		if ifConfigured {
+			out.Info(MaterialsReindexName + ": search is not configured; nothing to do")
+			return nil
+		}
+		return errSearchNotConfigured
+	}
+	if deps.DB == nil {
+		return errors.New("jz.bm: no database")
+	}
+	gdb, done, err := deps.DB(ctx)
+	if err != nil {
+		return err
+	}
+	defer done()
+
+	index := svc.IndexName(&models.Material{})
+	if fresh {
+		existed, err := beachcomber.DropIndex(ctx, engine, index)
+		if err != nil {
+			return fmt.Errorf("jz.bm: drop %s: %w", index, err)
+		}
+		if existed {
+			out.Printf("Dropped %s\n", index)
+		}
+	}
+	if err := beachcomber.EnsureIndex(ctx, engine, index, models.Material{}.SearchIndexSchema()); err != nil {
+		return fmt.Errorf("jz.bm: create %s: %w", index, err)
+	}
+
+	marked := classes.MarkMaterialSearch(ctx)
+	synced, failed := 0, 0
+	var batch []models.Material
+	err = gdb.WithContext(ctx).Model(&models.Material{}).
+		Select("id").
+		Order("id").
+		FindInBatches(&batch, materialsReindexBatch, func(_ *gorm.DB, _ int) error {
+			for i := range batch {
+				id := batch[i].ID
+				if err := svc.Sync(marked, gdb, &models.Material{ID: id}); err != nil {
+					failed++
+					out.Warning(fmt.Sprintf("material %d: %v", id, err))
+					continue
+				}
+				synced++
+			}
+			return ctx.Err()
+		}).Error
+	if err != nil {
+		return err
+	}
+	msg := fmt.Sprintf("Reindexed: %d synced, %d failed", synced, failed)
+	if failed > 0 {
+		out.Error(msg)
+		return fmt.Errorf("jz.bm: %d materials failed to sync", failed)
+	}
+	out.Success(msg)
+	return nil
+}
+
+// flagOn reports whether a bare flag was given.
+func flagOn(in bonfire.Input, name string) bool {
+	v, _ := in.Flag(name)
+	return v == "true"
+}
--- a/console/materials_reindex_test.go
+++ b/console/materials_reindex_test.go
@@ -0,0 +1,89 @@
+package console_test
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"slices"
+	"strings"
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/bonfire"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/console"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func TestMaterialsReindexRefuses(t *testing.T) {
+	asked := false
+	deps := console.MaterialsReindexDeps{
+		DB: func(context.Context) (*gorm.DB, func(), error) {
+			asked = true
+			return nil, func() {}, nil
+		},
+		Index: classes.NewMaterialIndexer(searchtest.App(t)),
+	}
+	cmds := []bonfire.Command{console.MaterialsReindexCommand(deps)}
+	var out bytes.Buffer
+	err := bonfire.Call(context.Background(), cmds, console.MaterialsReindexName, nil, &out)
+	if err == nil || !strings.Contains(err.Error(), "search.driver") || !strings.Contains(err.Error(), "search.typesense.api_key") {
+		t.Errorf("unconfigured engine: err = %v, want the configuration hint", err)
+	}
+	out.Reset()
+	if err := bonfire.Call(context.Background(), cmds, console.MaterialsReindexName, []string{"--if-configured"}, &out); err != nil {
+		t.Errorf("--if-configured: %v", err)
+	}
+	if !strings.Contains(out.String(), "not configured") {
+		t.Errorf("--if-configured output %q, want the notice", out.String())
+	}
+	if asked {
+		t.Error("the command opened the database without an engine")
+	}
+}
+
+func TestMaterialsReindexFresh(t *testing.T) {
+	stub := &searchtest.Stub{}
+	app := searchtest.App(t, searchtest.Start(t, stub)...)
+	gdb, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sqlDB, err := gdb.DB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := lagoon.Publish(app, sqlDB, gdb); err != nil {
+		t.Fatal(err)
+	}
+	deps := console.MaterialsReindexDeps{
+		DB:    func(context.Context) (*gorm.DB, func(), error) { return gdb, func() {}, nil },
+		Index: classes.NewMaterialIndexer(app),
+	}
+	cmds := []bonfire.Command{console.MaterialsReindexCommand(deps)}
+	coll := "/collections/" + searchtest.Index
+	var out bytes.Buffer
+	for run := 1; run <= 2; run++ {
+		before := len(stub.Requests())
+		out.Reset()
+		if err := bonfire.Call(context.Background(), cmds, console.MaterialsReindexName, []string{"--fresh"}, &out); err != nil {
+			t.Fatalf("run %d: %v\n%s", run, err, out.String())
+		}
+		var sent []string
+		for _, r := range stub.Requests()[before:] {
+			sent = append(sent, r.Method+" "+r.Path)
+		}
+		if want := []string{http.MethodDelete + " " + coll, http.MethodGet + " " + coll, http.MethodPost + " /collections"}; !slices.Equal(sent, want) {
+			t.Errorf("run %d sent %v, want %v", run, sent, want)
+		}
+		if !strings.Contains(out.String(), "Reindexed: 0 synced, 0 failed") {
+			t.Errorf("run %d output %q", run, out.String())
+		}
+	}
+	// The second run dropped the collection the first one created.
+	if !strings.Contains(out.String(), "Dropped "+searchtest.Index) {
+		t.Errorf("second run output %q, want Dropped", out.String())
+	}
+}