feat: extract file text on every attach path and backfill on reindex

- inline and signed uploads queue the text job in their transaction
- the admin form queues it in the save transaction
- detach resyncs the material (its text row goes by cascade)
- bm:materials-reindex backfills missing PDF texts before each sync

material_file_text_postgres_test.go

@@ -0,0 +1,178 @@
+package bm_test
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os/exec"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	userclasses "git.golem15.com/golem15/sm-user-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/surf"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/pdftest"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+	"git.golem15.com/jakub/sm-bm-plugin/jobs"
+	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/memblob"
+)
+
+// TestMaterialFileTextAgainstDatabase drives file search end to end against
+// a migrated database and a Typesense stand-in: an inline PDF attach queues
+// one extraction job in its transaction, the extraction stores the PDF's
+// text, the sync sends it as file_texts, and a detach drops the text row
+// (cascade) and resyncs the document without it.
+func TestMaterialFileTextAgainstDatabase(t *testing.T) {
+	stub := &searchtest.Stub{}
+	app, plugins, gdb := migratedWith(t, searchtest.Start(t, stub)...)
+	if _, err := exec.LookPath("pdftotext"); err != nil {
+		t.Skip("pdftotext is not installed (poppler-utils)")
+	}
+	bucket, err := blob.OpenBucket(t.Context(), "mem://")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { bucket.Close() })
+	if err := app.Publish(bucket); err != nil {
+		t.Fatal(err)
+	}
+	handler, err := surf.Assemble(app, plugins)
+	if err != nil {
+		t.Fatal(err)
+	}
+	managerID := createUser(t, gdb, "files-manager@example.test")
+	var groupID uint
+	if err := gdb.Raw(`
+INSERT INTO user_groups (name, code, permissions, created_at, updated_at)
+VALUES ('BM materials', 'bm-materials', '{"jz.bm.access_materials":1}', NOW(), NOW())
+RETURNING id`).Scan(&groupID).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := gdb.Exec(`INSERT INTO users_groups (user_id, user_group_id) VALUES (?, ?)`, managerID, groupID).Error; err != nil {
+		t.Fatal(err)
+	}
+	secret, token, err := userclasses.MintAPIToken("usr_", managerID, "BM file text", []string{"read", "write"}, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := gdb.Create(token).Error; err != nil {
+		t.Fatal(err)
+	}
+	send := func(method, path string, payload any) (int, map[string]any) {
+		t.Helper()
+		var buf bytes.Buffer
+		if payload != nil {
+			if err := json.NewEncoder(&buf).Encode(payload); err != nil {
+				t.Fatal(err)
+			}
+		}
+		req := httptest.NewRequest(method, path, &buf)
+		req.Header.Set("Authorization", "Bearer "+secret)
+		if payload != nil {
+			req.Header.Set("Content-Type", "application/json")
+		}
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+		var out map[string]any
+		if rec.Body.Len() > 0 {
+			_ = json.Unmarshal(rec.Body.Bytes(), &out)
+		}
+		return rec.Code, out
+	}
+
+	code, out := send(http.MethodPost, "/api/v1/bm/materials", map[string]any{"title": "Wykład z PDF", "type": "notatka"})
+	if code != http.StatusCreated {
+		t.Fatalf("POST material = %d %v", code, out)
+	}
+	materialID := id(obj(out["data"])["id"])
+	n, _ := strconv.ParseUint(materialID, 10, 64)
+	mid := uint(n)
+
+	const phrase = "Globalizacja i demokracja"
+	code, out = send(http.MethodPost, "/api/v1/bm/materials/"+materialID+"/files/inline", map[string]any{
+		"name":           "wyklad.pdf",
+		"content_base64": base64.StdEncoding.EncodeToString(pdftest.OnePage(phrase)),
+	})
+	if code != http.StatusCreated {
+		t.Fatalf("inline attach = %d %v", code, out)
+	}
+	var queued int64
+	if err := gdb.Raw(`SELECT COUNT(*) FROM river_job WHERE kind = ?`, jobs.ExtractMaterialTextKind).Scan(&queued).Error; err != nil {
+		t.Fatal(err)
+	}
+	if queued != 1 {
+		t.Errorf("river_job rows of kind %s = %d, want 1", jobs.ExtractMaterialTextKind, queued)
+	}
+
+	pdf := classes.DefaultPDFText()
+	pdf.TempDir = t.TempDir()
+	added, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid)
+	if err != nil || added != 1 {
+		t.Fatalf("EnsureFileTexts = %d, %v, want 1 row", added, err)
+	}
+	var texts []string
+	if err := gdb.Raw(`SELECT text FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&texts).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(texts) != 1 || !strings.Contains(texts[0], phrase) {
+		t.Fatalf("text rows = %q, want one holding %q", texts, phrase)
+	}
+	// A second run finds nothing new.
+	if again, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid); err != nil || again != 0 {
+		t.Errorf("second EnsureFileTexts = %d, %v, want 0", again, err)
+	}
+
+	lastDoc := func() map[string]any {
+		t.Helper()
+		var doc map[string]any
+		for _, d := range stub.Imported() {
+			if d["id"] == materialID {
+				doc = d
+			}
+		}
+		if doc == nil {
+			t.Fatalf("material %s was never upserted", materialID)
+		}
+		return doc
+	}
+	ix := classes.NewMaterialIndexer(app)
+	ix.Sync(t.Context(), gdb, mid)
+	fileTexts, _ := lastDoc()["file_texts"].([]any)
+	if len(fileTexts) != 1 || !strings.Contains(fileTexts[0].(string), phrase) {
+		t.Errorf("upserted file_texts = %v, want the PDF's text", lastDoc()["file_texts"])
+	}
+
+	var fileIDs []uint
+	if err := gdb.Raw(`SELECT id FROM system_files WHERE attachment_id = ? AND field = 'files'`, materialID).Scan(&fileIDs).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(fileIDs) != 1 {
+		t.Fatalf("files = %v, want one", fileIDs)
+	}
+	before := len(stub.Imported())
+	code, out = send(http.MethodDelete, "/api/v1/bm/materials/"+materialID+"/files/"+strconv.FormatUint(uint64(fileIDs[0]), 10), nil)
+	if code != http.StatusNoContent {
+		t.Fatalf("detach = %d %v", code, out)
+	}
+	var left int64
+	if err := gdb.Raw(`SELECT COUNT(*) FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&left).Error; err != nil {
+		t.Fatal(err)
+	}
+	if left != 0 {
+		t.Errorf("text rows after detach = %d, want 0 (cascade)", left)
+	}
+	if len(stub.Imported()) <= before {
+		t.Fatal("the detach did not resync the material")
+	}
+	if _, ok := lastDoc()["file_texts"]; ok {
+		t.Errorf("the document after detach still has file_texts: %v", lastDoc())
+	}
+	if slices.Contains(stub.Removed(), materialID) {
+		t.Error("the detach removed the material's document instead of resyncing it")
+	}
+}