feat: index the text of attached PDFs

- jz_bm_material_file_texts caches the pdftotext output per file
- the River job jz.bm.extract_material_text extracts and resyncs
- the management multipart attach queues it inside its transaction
- file_texts in the document and a file snippet with its name

controllers/api/export_test.go

@@ -9,4 +9,5 @@ var (
 	PickSnippet    = pickSnippet
 	FlattenSnippet = flattenSnippet
 	RankOrder      = rankOrder
+	HasFileSnippet = hasFileSnippet
 )

controllers/api/manage.go

@@ -251,13 +251,21 @@ func (h *Handlers) ManagedMaterials(w http.ResponseWriter, r *http.Request) {
 		writeOpaque500(w)
 		return
 	}
+	var files map[uint][]models.IndexedFile
+	if cands != nil {
+		files, err = snippetFiles(db, materialIDs(rows), cands.Hits)
+		if err != nil {
+			writeOpaque500(w)
+			return
+		}
+	}
 	mcp := viaMCP(r)
 	out := make([]any, 0, len(rows))
 	for _, m := range rows {
 		row := newManagedMaterialJSON(m, people)
 		if cands != nil {
 			if hit, ok := cands.Hits[m.ID]; ok {
-				row.Snippet = snippetValue(pickSnippet(hit, m), mcp)
+				row.Snippet = snippetValue(pickSnippet(hit, m, files[m.ID]), mcp)
 			}
 		}
 		out = append(out, managedMaterialView(row, mcp, includeBody(r)))

controllers/api/manage_files.go

@@ -115,7 +115,8 @@ func (h *Handlers) AttachMaterialFiles(w http.ResponseWriter, r *http.Request) {
 	h.writeMaterialDetail(w, r, db, m.ID, http.StatusCreated)
 }
 
-// storeParts stores and binds every "file" part in one transaction. A
+// storeParts stores and binds every "file" part in one transaction and
+// queues the file text job in it. A
 // refusal (no file, too many, a type or size attach.Store turns down) is
 // set in *refusal and rolls back; the returned keys are the blobs written.
 func (h *Handlers) storeParts(ctx context.Context, db *gorm.DB, bucket *blob.Bucket, m models.Material, mr *multipart.Reader, refusal *string) ([]string, error) {
@@ -153,6 +154,9 @@ func (h *Handlers) storeParts(ctx context.Context, db *gorm.DB, bucket *blob.Buc
 			*refusal = "No file: send at least one multipart part named \"file\"."
 			return errRefused
 		}
+		// The text job joins the transaction: it exists only when the
+		// files commit, and it syncs the material's document.
+		h.MaterialIndex.QueueExtract(ctx, tx, m.ID)
 		return nil
 	})
 	if errors.Is(err, errRefused) {

controllers/api/material_search.go

@@ -9,6 +9,7 @@ import (
 	"git.golem15.com/golem15/summercms/modules/beachcomber"
 	"git.golem15.com/golem15/summercms/modules/wire"
 	"git.golem15.com/jakub/sm-bm-plugin/models"
+	"gorm.io/gorm"
 )
 
 // Material search: the engine (beachcomber, Typesense) answers candidate ids
@@ -66,17 +67,20 @@ type snippetJSON struct {
 }
 
 // pickSnippet returns the snippet of m from hit: the first body snippet,
-// mapped one to one. It is nil when the hit has none. Call it only for a
-// row the SQL re-gate returned.
-func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON {
+// mapped one to one, or else the first file_texts snippet, with the name of
+// the file at its array index in files (the IndexedFiles of m, in document
+// order; nil when the index does not resolve). A file text is long and is
+// not loaded for the page, so a file snippet is always cut on both sides.
+// It is nil when the hit has neither. Call it only for a row the SQL
+// re-gate returned.
+func pickSnippet(hit beachcomber.Hit, m models.Material, files []models.IndexedFile) *snippetJSON {
 	for _, sn := range hit.Highlights {
 		if sn.Field != "body" || len(sn.Segments) == 0 {
 			continue
 		}
-		out := &snippetJSON{Source: "body", Segments: make([]segmentJSON, 0, len(sn.Segments))}
+		out := &snippetJSON{Source: "body", Segments: snippetSegments(sn)}
 		var joined strings.Builder
 		for _, seg := range sn.Segments {
-			out.Segments = append(out.Segments, segmentJSON{Text: seg.Text, Hit: seg.Match})
 			joined.WriteString(seg.Text)
 		}
 		body := ""
@@ -88,9 +92,66 @@ func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON {
 		out.Trail = !strings.HasSuffix(body, text)
 		return out
 	}
+	for _, sn := range hit.Highlights {
+		if sn.Field != fileTextsField || len(sn.Segments) == 0 {
+			continue
+		}
+		out := &snippetJSON{Source: "file", Lead: true, Trail: true, Segments: snippetSegments(sn)}
+		if sn.ArrayIndex >= 0 && sn.ArrayIndex < len(files) {
+			name := files[sn.ArrayIndex].FileName
+			out.FileName = &name
+		}
+		return out
+	}
 	return nil
 }
 
+// fileTextsField is the document field of the attached files' texts.
+const fileTextsField = "file_texts"
+
+// snippetSegments maps the engine's segments one to one.
+func snippetSegments(sn beachcomber.Snippet) []segmentJSON {
+	out := make([]segmentJSON, 0, len(sn.Segments))
+	for _, seg := range sn.Segments {
+		out = append(out, segmentJSON{Text: seg.Text, Hit: seg.Match})
+	}
+	return out
+}
+
+// hasFileSnippet reports whether hit carries a file_texts snippet and no
+// body one, so its snippet needs the material's file names.
+func hasFileSnippet(hit beachcomber.Hit) bool {
+	file := false
+	for _, sn := range hit.Highlights {
+		if len(sn.Segments) == 0 {
+			continue
+		}
+		switch sn.Field {
+		case "body":
+			return false
+		case fileTextsField:
+			file = true
+		}
+	}
+	return file
+}
+
+// snippetFiles loads, once per page, the indexed files (names only) of the
+// rows whose snippet comes from a file. It reads nothing when no row needs
+// it. ids are rows SQL returned.
+func snippetFiles(db *gorm.DB, ids []uint, hits map[uint]beachcomber.Hit) (map[uint][]models.IndexedFile, error) {
+	var need []uint
+	for _, id := range ids {
+		if hit, ok := hits[id]; ok && hasFileSnippet(hit) {
+			need = append(need, id)
+		}
+	}
+	if len(need) == 0 {
+		return nil, nil
+	}
+	return models.IndexedFiles(db, need, false)
+}
+
 // flattenSnippet is a snippet as plain text, for MCP: the segment texts
 // joined, with an ellipsis where text was cut.
 func flattenSnippet(s *snippetJSON) string {

controllers/api/material_search_test.go

@@ -33,7 +33,7 @@ func TestMaterialSnippet(t *testing.T) {
 		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
 		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia <b>państwa</b>.", false)},
 	}}
-	s := api.PickSnippet(hit, m)
+	s := api.PickSnippet(hit, m, nil)
 	if s == nil {
 		t.Fatal("no snippet")
 	}
@@ -59,24 +59,72 @@ func TestMaterialSnippet(t *testing.T) {
 
 	// The whole body: neither lead nor trail.
 	short := "Krótka **notatka**"
-	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short}, nil)
 	if whole == nil || whole.Lead || whole.Trail {
 		t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
 	}
 	// Start of the body only: trail but no lead.
-	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
+	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m, nil)
 	if start == nil || start.Lead || !start.Trail {
 		t.Errorf("start snippet = %+v, want trail only", start)
 	}
 
 	// A snippet on another field only, or none at all, gives no snippet.
-	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
+	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m, nil); got != nil {
 		t.Errorf("title-only hit gave %+v", got)
 	}
-	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
+	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m, nil); got != nil {
 		t.Errorf("hit without highlights gave %+v", got)
 	}
 
+	// A match only in an attached file: the file snippet, named by the
+	// snippet's array index into the indexed files, cut on both sides.
+	files := []models.IndexedFile{{FileID: 7, FileName: "wstep.pdf"}, {FileID: 9, FileName: "rozdzial-2.pdf"}}
+	fileHit := beachcomber.Hit{Highlights: []beachcomber.Snippet{
+		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
+		{Field: "file_texts", ArrayIndex: 1, Segments: segments("o ", false, "demokracji", true, " w Europie", false)},
+	}}
+	fs := api.PickSnippet(fileHit, m, files)
+	if fs == nil || fs.Source != "file" || fs.FileName == nil || *fs.FileName != "rozdzial-2.pdf" || !fs.Lead || !fs.Trail {
+		t.Fatalf("file snippet = %+v, want source file, rozdzial-2.pdf, lead and trail", fs)
+	}
+	if len(fs.Segments) != 3 || fs.Segments[1].Text != "demokracji" || !fs.Segments[1].Hit {
+		t.Errorf("file segments = %+v", fs.Segments)
+	}
+	fb, err := json.Marshal(fs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(fb); !strings.Contains(got, `"source":"file","file_name":"rozdzial-2.pdf","lead":true,"trail":true`) {
+		t.Errorf("file JSON = %s", got)
+	}
+	// The first file by index.
+	first := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "file_texts", ArrayIndex: 0, Segments: segments("x", true)}}}, m, files)
+	if first == nil || first.FileName == nil || *first.FileName != "wstep.pdf" {
+		t.Errorf("index 0 snippet = %+v, want wstep.pdf", first)
+	}
+	// An index the files do not resolve (out of range, -1, no files): no name.
+	for _, c := range []struct {
+		idx   int
+		files []models.IndexedFile
+	}{{5, files}, {-1, files}, {0, nil}} {
+		got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "file_texts", ArrayIndex: c.idx, Segments: segments("x", true)}}}, m, c.files)
+		if got == nil || got.Source != "file" || got.FileName != nil {
+			t.Errorf("index %d with %d files: snippet = %+v, want a file snippet with no name", c.idx, len(c.files), got)
+		}
+	}
+	// A body snippet is preferred over a file snippet, whatever the order.
+	both := beachcomber.Hit{Highlights: []beachcomber.Snippet{
+		{Field: "file_texts", ArrayIndex: 0, Segments: segments("plik", true)},
+		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true)},
+	}}
+	if got := api.PickSnippet(both, m, files); got == nil || got.Source != "body" || got.FileName != nil {
+		t.Errorf("body and file hit gave %+v, want the body snippet", got)
+	}
+	if !api.HasFileSnippet(fileHit) || api.HasFileSnippet(both) || api.HasFileSnippet(hit) {
+		t.Error("HasFileSnippet: want true for a file-only hit, false with a body snippet")
+	}
+
 	if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia <b>państwa</b>.…" {
 		t.Errorf("FlattenSnippet = %q", got)
 	}
@@ -221,19 +269,19 @@ func TestManagedMaterialsSearch(t *testing.T) {
 func TestMaterialSnippetMCP(t *testing.T) {
 	body := "Wstęp. Proces globalizacji zmienia państwa. Koniec."
 	m := models.Material{Body: &body}
-	mid := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia", false)}}}, m)
+	mid := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia", false)}}}, m, nil)
 	if got := api.FlattenSnippet(mid); got != "…Proces globalizacji zmienia…" {
 		t.Errorf("middle = %q, want ellipses on both sides", got)
 	}
-	head := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp", true, ". Proces", false)}}}, m)
+	head := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp", true, ". Proces", false)}}}, m, nil)
 	if got := api.FlattenSnippet(head); got != "Wstęp. Proces…" {
 		t.Errorf("head = %q, want a trailing ellipsis only", got)
 	}
-	tail := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("państwa. ", false, "Koniec", true, ".", false)}}}, m)
+	tail := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("państwa. ", false, "Koniec", true, ".", false)}}}, m, nil)
 	if got := api.FlattenSnippet(tail); got != "…państwa. Koniec." {
 		t.Errorf("tail = %q, want a leading ellipsis only", got)
 	}
-	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments(body, false)}}}, m)
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments(body, false)}}}, m, nil)
 	if got := api.FlattenSnippet(whole); got != body {
 		t.Errorf("whole = %q, want no ellipsis", got)
 	}

controllers/api/materials.go

@@ -157,15 +157,21 @@ func (h *Handlers) Materials(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if cands != nil {
-		attachSnippets(out, rows, cands.Hits, viaMCP(r))
+		files, err := snippetFiles(db, materialIDs(rows), cands.Hits)
+		if err != nil {
+			writeOpaque500(w)
+			return
+		}
+		attachSnippets(out, rows, cands.Hits, files, viaMCP(r))
 	}
 	writeJSONPage(w, lagoon.Paginate(out, page, perPage, total))
 }
 
-// attachSnippets sets the snippet of each list row from its Hit. rows are
+// attachSnippets sets the snippet of each list row from its Hit and, for a
+// file snippet, the row's indexed files. rows are
 // the rows SQL returned (they passed the re-gate), in the order of out; the
 // Hit of an id SQL did not return is never read.
-func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, mcp bool) {
+func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, files map[uint][]models.IndexedFile, mcp bool) {
 	for i := range out {
 		if i >= len(rows) || rows[i].ID != out[i].ID {
 			continue
@@ -174,10 +180,19 @@ func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uin
 		if !ok {
 			continue
 		}
-		out[i].Snippet = snippetValue(pickSnippet(hit, rows[i]), mcp)
+		out[i].Snippet = snippetValue(pickSnippet(hit, rows[i], files[rows[i].ID]), mcp)
 	}
 }
 
+// materialIDs are the ids of rows, in order.
+func materialIDs(rows []models.Material) []uint {
+	ids := make([]uint, 0, len(rows))
+	for _, m := range rows {
+		ids = append(ids, m.ID)
+	}
+	return ids
+}
+
 // Material answers GET /materials/{id}: one published material the caller
 // may see, with its body, tags and files. A draft, a future publication or
 // another user's private material is a 404. can_manage tells the author (a