feat: index the text of attached PDFs

- jz_bm_material_file_texts caches the pdftotext output per file
- the River job jz.bm.extract_material_text extracts and resyncs
- the management multipart attach queues it inside its transaction
- file_texts in the document and a file snippet with its name

classes/pdftext_test.go

@@ -0,0 +1,89 @@
+package classes_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf8"
+
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/pdftest"
+)
+
+func TestPDFTextExtract(t *testing.T) {
+	if _, err := exec.LookPath("pdftotext"); err != nil {
+		t.Skip("pdftotext is not installed (poppler-utils)")
+	}
+	ctx := context.Background()
+	dir := t.TempDir()
+	pdf := classes.DefaultPDFText()
+	pdf.TempDir = dir
+	if pdf.Bin != "pdftotext" || pdf.Timeout != 60*time.Second || pdf.MaxBytes != 1<<20 {
+		t.Errorf("DefaultPDFText = %+v, want pdftotext, 60 s, 1 MiB", pdf)
+	}
+	doc := pdftest.OnePage("Globalizacja i demokracja")
+
+	text, truncated, err := pdf.Extract(ctx, bytes.NewReader(doc))
+	if err != nil {
+		t.Fatalf("Extract: %v", err)
+	}
+	if !strings.Contains(text, "Globalizacja i demokracja") || truncated {
+		t.Errorf("Extract = %q truncated %v, want the phrase, not truncated", text, truncated)
+	}
+
+	small := pdf
+	small.MaxBytes = 5
+	text, truncated, err = small.Extract(ctx, bytes.NewReader(doc))
+	if err != nil || !truncated || len(text) > 5 {
+		t.Errorf("MaxBytes 5: %q truncated %v err %v, want at most 5 bytes and truncated", text, truncated, err)
+	}
+
+	junk := make([]byte, 4096)
+	_, _ = rand.Read(junk)
+	text, _, err = pdf.Extract(ctx, bytes.NewReader(junk))
+	if err != nil || text != "" {
+		t.Errorf("random bytes: %q err %v, want empty text and no error", text, err)
+	}
+
+	missing := pdf
+	missing.Bin = "jz-bm-no-such-pdftotext"
+	if _, _, err := missing.Extract(ctx, bytes.NewReader(doc)); !errors.Is(err, classes.ErrPDFToolMissing) {
+		t.Errorf("missing binary: err %v, want ErrPDFToolMissing", err)
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(entries) != 0 {
+		t.Errorf("the temp dir holds %d files after extraction, want none", len(entries))
+	}
+	// A temp dir Extract creates itself is private to the app user.
+	sub := pdf
+	sub.TempDir = dir + "/sub"
+	if _, _, err := sub.Extract(ctx, bytes.NewReader(doc)); err != nil {
+		t.Fatal(err)
+	}
+	if info, err := os.Stat(sub.TempDir); err != nil || info.Mode().Perm() != 0o700 {
+		t.Errorf("created temp dir: %v %v, want mode 0700", info, err)
+	}
+
+	for _, c := range []struct{ in, want string }{
+		{"plain tekst", "plain tekst"},
+		{"a\x00b\x00", "ab"},
+		{"zażółć\xff\xfe gęślą", "zażółć gęślą"},
+		{"cut \xc5", "cut "},
+		{"", ""},
+	} {
+		got := classes.CleanExtractedText(c.in)
+		if got != c.want || !utf8.ValidString(got) {
+			t.Errorf("CleanExtractedText(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}