feat: extract file text on every attach path and backfill on reindex

- inline and signed uploads queue the text job in their transaction
- the admin form queues it in the save transaction
- detach resyncs the material (its text row goes by cascade)
- bm:materials-reindex backfills missing PDF texts before each sync

console/export_test.go

@@ -0,0 +1,24 @@
+package console
+
+import (
+	"context"
+
+	"git.golem15.com/golem15/summercms/modules/bonfire"
+	"gorm.io/gorm"
+)
+
+// ReindexRun exposes one reindex walk to the console_test package.
+type ReindexRun struct{ r *reindexRun }
+
+// NewReindexRun is a walk with deps' Backfill and the given sync.
+func NewReindexRun(deps MaterialsReindexDeps, sync func(id uint) error, out bonfire.Output) ReindexRun {
+	return ReindexRun{r: &reindexRun{backfill: deps.Backfill, sync: sync, out: out}}
+}
+
+// Material runs the per-material step.
+func (w ReindexRun) Material(ctx context.Context, db *gorm.DB, id uint) { w.r.material(ctx, db, id) }
+
+// Counts are the run's counters and the tool-missing flag.
+func (w ReindexRun) Counts() (synced, failed int, noTool bool) {
+	return w.r.synced, w.r.failed, w.r.noTool
+}

console/materials_reindex.go

@@ -28,6 +28,10 @@ var errSearchNotConfigured = errors.New("jz.bm: the search engine is not configu
 type MaterialsReindexDeps struct {
 	DB    func(ctx context.Context) (*gorm.DB, func(), error)
 	Index *classes.MaterialIndexer
+	// Backfill extracts the text of a material's PDFs that have none yet
+	// (classes.EnsureFileTexts) before the material is synced. Nil skips
+	// it.
+	Backfill func(ctx context.Context, db *gorm.DB, materialID uint) error
 }
 
 // MaterialsReindexCommand returns bm:materials-reindex: it ensures the
@@ -41,7 +45,10 @@ func MaterialsReindexCommand(deps MaterialsReindexDeps) bonfire.Command {
 		Description: "Sync every material into the search index",
 		Long: `Upserts the search document of every material, drafts and private ones
 included (visibility is enforced in SQL, never by the index), through the same
-path as the sync after a save.
+path as the sync after a save. Before each material is synced, the text of
+its attached PDFs that have none yet is extracted with pdftotext, so PDFs
+attached before file search existed become searchable; a missing pdftotext
+(poppler-utils) is one warning and the run goes on without file text.
 
 --fresh first drops the collection and recreates it from the material schema
 (models.Material.SearchIndexSchema). Use it on first deploy and after the
@@ -103,35 +110,72 @@ func reindexMaterials(ctx context.Context, deps MaterialsReindexDeps, fresh, ifC
 	}
 
 	marked := classes.MarkMaterialSearch(ctx)
-	synced, failed := 0, 0
+	run := &reindexRun{
+		backfill: deps.Backfill,
+		sync: func(id uint) error {
+			return svc.Sync(marked, gdb, &models.Material{ID: id})
+		},
+		out: out,
+	}
 	var batch []models.Material
 	err = gdb.WithContext(ctx).Model(&models.Material{}).
 		Select("id").
 		Order("id").
 		FindInBatches(&batch, materialsReindexBatch, func(_ *gorm.DB, _ int) error {
 			for i := range batch {
-				id := batch[i].ID
-				if err := svc.Sync(marked, gdb, &models.Material{ID: id}); err != nil {
-					failed++
-					out.Warning(fmt.Sprintf("material %d: %v", id, err))
-					continue
-				}
-				synced++
+				run.material(ctx, gdb, batch[i].ID)
 			}
 			return ctx.Err()
 		}).Error
 	if err != nil {
 		return err
 	}
-	msg := fmt.Sprintf("Reindexed: %d synced, %d failed", synced, failed)
-	if failed > 0 {
+	msg := fmt.Sprintf("Reindexed: %d synced, %d failed", run.synced, run.failed)
+	if run.failed > 0 {
 		out.Error(msg)
-		return fmt.Errorf("jz.bm: %d materials failed to sync", failed)
+		return fmt.Errorf("jz.bm: %d materials failed to sync", run.failed)
 	}
 	out.Success(msg)
 	return nil
 }
 
+// pdfToolMissingWarning is printed once per run when pdftotext is missing.
+const pdfToolMissingWarning = "pdftotext not found: PDFs add no search text; install poppler-utils and rerun"
+
+// reindexRun is the state of one reindex walk: the run's counters and
+// whether pdftotext turned out to be missing.
+type reindexRun struct {
+	backfill func(ctx context.Context, db *gorm.DB, materialID uint) error
+	sync     func(id uint) error
+	out      bonfire.Output
+	synced   int
+	failed   int
+	noTool   bool
+}
+
+// material backfills the file texts of material id, then syncs it. A
+// missing pdftotext warns once and stops the backfill for the rest of the
+// run; another backfill error warns for this material, which is still
+// synced.
+func (r *reindexRun) material(ctx context.Context, db *gorm.DB, id uint) {
+	if r.backfill != nil && !r.noTool {
+		err := r.backfill(ctx, db, id)
+		switch {
+		case errors.Is(err, classes.ErrPDFToolMissing):
+			r.noTool = true
+			r.out.Warning(pdfToolMissingWarning)
+		case err != nil:
+			r.out.Warning(fmt.Sprintf("material %d: file text: %v", id, err))
+		}
+	}
+	if err := r.sync(id); err != nil {
+		r.failed++
+		r.out.Warning(fmt.Sprintf("material %d: %v", id, err))
+		return
+	}
+	r.synced++
+}
+
 // flagOn reports whether a bare flag was given.
 func flagOn(in bonfire.Input, name string) bool {
 	v, _ := in.Flag(name)

console/materials_reindex_test.go

@@ -3,8 +3,10 @@ package console_test
 import (
 	"bytes"
 	"context"
+	"errors"
 	"net/http"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -87,3 +89,83 @@ func TestMaterialsReindexFresh(t *testing.T) {
 		t.Errorf("second run output %q, want Dropped", out.String())
 	}
 }
+
+func TestMaterialsReindexBackfill(t *testing.T) {
+	ctx := context.Background()
+	var calls []string
+	var synced []uint
+	sync := func(id uint) error {
+		calls = append(calls, "sync "+strconv.FormatUint(uint64(id), 10))
+		synced = append(synced, id)
+		return nil
+	}
+
+	// pdftotext missing on the first material: one warning, no more
+	// backfill calls, and every material is still synced.
+	deps := console.MaterialsReindexDeps{Backfill: func(_ context.Context, _ *gorm.DB, id uint) error {
+		calls = append(calls, "backfill "+strconv.FormatUint(uint64(id), 10))
+		return classes.ErrPDFToolMissing
+	}}
+	var stdout, stderr bytes.Buffer
+	run := console.NewReindexRun(deps, sync, bonfire.NewOutput(strings.NewReader(""), &stdout, &stderr))
+	for _, id := range []uint{1, 2, 3} {
+		run.Material(ctx, nil, id)
+	}
+	if want := []string{"backfill 1", "sync 1", "sync 2", "sync 3"}; !slices.Equal(calls, want) {
+		t.Errorf("calls = %v, want %v", calls, want)
+	}
+	if !slices.Equal(synced, []uint{1, 2, 3}) {
+		t.Errorf("synced = %v, want all three", synced)
+	}
+	all := stdout.String() + stderr.String()
+	if n := strings.Count(all, "pdftotext not found"); n != 1 {
+		t.Errorf("the tool warning was printed %d times, want once:\n%s", n, all)
+	}
+	if !strings.Contains(all, "install poppler-utils") {
+		t.Errorf("output %q, want the poppler-utils hint", all)
+	}
+	if s, f, noTool := run.Counts(); s != 3 || f != 0 || !noTool {
+		t.Errorf("counts = %d synced %d failed noTool %v, want 3 0 true", s, f, noTool)
+	}
+
+	// Another backfill error warns for that material only; it is still
+	// synced and the next material is backfilled. A failed sync counts.
+	calls, synced = nil, nil
+	stdout.Reset()
+	stderr.Reset()
+	deps.Backfill = func(_ context.Context, _ *gorm.DB, id uint) error {
+		calls = append(calls, "backfill "+strconv.FormatUint(uint64(id), 10))
+		if id == 1 {
+			return errors.New("blob gone")
+		}
+		return nil
+	}
+	failing := func(id uint) error {
+		if err := sync(id); err != nil || id == 3 {
+			return errors.New("engine down")
+		}
+		return nil
+	}
+	run = console.NewReindexRun(deps, failing, bonfire.NewOutput(strings.NewReader(""), &stdout, &stderr))
+	for _, id := range []uint{1, 2, 3} {
+		run.Material(ctx, nil, id)
+	}
+	if want := []string{"backfill 1", "sync 1", "backfill 2", "sync 2", "backfill 3", "sync 3"}; !slices.Equal(calls, want) {
+		t.Errorf("calls = %v, want %v", calls, want)
+	}
+	all = stdout.String() + stderr.String()
+	if !strings.Contains(all, "material 1: file text: blob gone") || strings.Contains(all, "pdftotext not found") {
+		t.Errorf("output %q, want the material 1 file text warning only", all)
+	}
+	if s, f, noTool := run.Counts(); s != 2 || f != 1 || noTool {
+		t.Errorf("counts = %d synced %d failed noTool %v, want 2 1 false", s, f, noTool)
+	}
+
+	// No Backfill: only syncs.
+	calls = nil
+	run = console.NewReindexRun(console.MaterialsReindexDeps{}, sync, bonfire.NewOutput(strings.NewReader(""), &stdout, &stderr))
+	run.Material(ctx, nil, 9)
+	if !slices.Equal(calls, []string{"sync 9"}) {
+		t.Errorf("calls without Backfill = %v", calls)
+	}
+}