feat: index the text of attached PDFs

- jz_bm_material_file_texts caches the pdftotext output per file
- the River job jz.bm.extract_material_text extracts and resyncs
- the management multipart attach queues it inside its transaction
- file_texts in the document and a file snippet with its name

controllers/api/material_search_test.go

@@ -33,7 +33,7 @@ func TestMaterialSnippet(t *testing.T) {
 		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
 		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia <b>państwa</b>.", false)},
 	}}
-	s := api.PickSnippet(hit, m)
+	s := api.PickSnippet(hit, m, nil)
 	if s == nil {
 		t.Fatal("no snippet")
 	}
@@ -59,24 +59,72 @@ func TestMaterialSnippet(t *testing.T) {
 
 	// The whole body: neither lead nor trail.
 	short := "Krótka **notatka**"
-	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short}, nil)
 	if whole == nil || whole.Lead || whole.Trail {
 		t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
 	}
 	// Start of the body only: trail but no lead.
-	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
+	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m, nil)
 	if start == nil || start.Lead || !start.Trail {
 		t.Errorf("start snippet = %+v, want trail only", start)
 	}
 
 	// A snippet on another field only, or none at all, gives no snippet.
-	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
+	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m, nil); got != nil {
 		t.Errorf("title-only hit gave %+v", got)
 	}
-	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
+	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m, nil); got != nil {
 		t.Errorf("hit without highlights gave %+v", got)
 	}
 
+	// A match only in an attached file: the file snippet, named by the
+	// snippet's array index into the indexed files, cut on both sides.
+	files := []models.IndexedFile{{FileID: 7, FileName: "wstep.pdf"}, {FileID: 9, FileName: "rozdzial-2.pdf"}}
+	fileHit := beachcomber.Hit{Highlights: []beachcomber.Snippet{
+		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
+		{Field: "file_texts", ArrayIndex: 1, Segments: segments("o ", false, "demokracji", true, " w Europie", false)},
+	}}
+	fs := api.PickSnippet(fileHit, m, files)
+	if fs == nil || fs.Source != "file" || fs.FileName == nil || *fs.FileName != "rozdzial-2.pdf" || !fs.Lead || !fs.Trail {
+		t.Fatalf("file snippet = %+v, want source file, rozdzial-2.pdf, lead and trail", fs)
+	}
+	if len(fs.Segments) != 3 || fs.Segments[1].Text != "demokracji" || !fs.Segments[1].Hit {
+		t.Errorf("file segments = %+v", fs.Segments)
+	}
+	fb, err := json.Marshal(fs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(fb); !strings.Contains(got, `"source":"file","file_name":"rozdzial-2.pdf","lead":true,"trail":true`) {
+		t.Errorf("file JSON = %s", got)
+	}
+	// The first file by index.
+	first := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "file_texts", ArrayIndex: 0, Segments: segments("x", true)}}}, m, files)
+	if first == nil || first.FileName == nil || *first.FileName != "wstep.pdf" {
+		t.Errorf("index 0 snippet = %+v, want wstep.pdf", first)
+	}
+	// An index the files do not resolve (out of range, -1, no files): no name.
+	for _, c := range []struct {
+		idx   int
+		files []models.IndexedFile
+	}{{5, files}, {-1, files}, {0, nil}} {
+		got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "file_texts", ArrayIndex: c.idx, Segments: segments("x", true)}}}, m, c.files)
+		if got == nil || got.Source != "file" || got.FileName != nil {
+			t.Errorf("index %d with %d files: snippet = %+v, want a file snippet with no name", c.idx, len(c.files), got)
+		}
+	}
+	// A body snippet is preferred over a file snippet, whatever the order.
+	both := beachcomber.Hit{Highlights: []beachcomber.Snippet{
+		{Field: "file_texts", ArrayIndex: 0, Segments: segments("plik", true)},
+		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true)},
+	}}
+	if got := api.PickSnippet(both, m, files); got == nil || got.Source != "body" || got.FileName != nil {
+		t.Errorf("body and file hit gave %+v, want the body snippet", got)
+	}
+	if !api.HasFileSnippet(fileHit) || api.HasFileSnippet(both) || api.HasFileSnippet(hit) {
+		t.Error("HasFileSnippet: want true for a file-only hit, false with a body snippet")
+	}
+
 	if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia <b>państwa</b>.…" {
 		t.Errorf("FlattenSnippet = %q", got)
 	}
@@ -221,19 +269,19 @@ func TestManagedMaterialsSearch(t *testing.T) {
 func TestMaterialSnippetMCP(t *testing.T) {
 	body := "Wstęp. Proces globalizacji zmienia państwa. Koniec."
 	m := models.Material{Body: &body}
-	mid := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia", false)}}}, m)
+	mid := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia", false)}}}, m, nil)
 	if got := api.FlattenSnippet(mid); got != "…Proces globalizacji zmienia…" {
 		t.Errorf("middle = %q, want ellipses on both sides", got)
 	}
-	head := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp", true, ". Proces", false)}}}, m)
+	head := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp", true, ". Proces", false)}}}, m, nil)
 	if got := api.FlattenSnippet(head); got != "Wstęp. Proces…" {
 		t.Errorf("head = %q, want a trailing ellipsis only", got)
 	}
-	tail := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("państwa. ", false, "Koniec", true, ".", false)}}}, m)
+	tail := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("państwa. ", false, "Koniec", true, ".", false)}}}, m, nil)
 	if got := api.FlattenSnippet(tail); got != "…państwa. Koniec." {
 		t.Errorf("tail = %q, want a leading ellipsis only", got)
 	}
-	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments(body, false)}}}, m)
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments(body, false)}}}, m, nil)
 	if got := api.FlattenSnippet(whole); got != body {
 		t.Errorf("whole = %q, want no ellipsis", got)
 	}