feat: index the text of attached PDFs

- jz_bm_material_file_texts caches the pdftotext output per file
- the River job jz.bm.extract_material_text extracts and resyncs
- the management multipart attach queues it inside its transaction
- file_texts in the document and a file snippet with its name

classes/pdftext.go

@@ -0,0 +1,124 @@
+package classes
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ErrPDFToolMissing is returned by Extract when the pdftotext binary is not
+// installed (poppler-utils). Nothing is stored then, so a later run
+// retries.
+var ErrPDFToolMissing = errors.New("jz.bm: pdftotext is not installed")
+
+// PDFText runs pdftotext (poppler-utils) on one PDF. The PDF comes from a
+// material manager, so it is untrusted input to a native tool: it runs
+// without a shell, from a 0600 temp file in a 0700 directory that is
+// removed afterwards, for at most Timeout and with at most MaxBytes of text
+// read. It runs only in the extraction job and the reindex command, never in
+// a request.
+type PDFText struct {
+	// Bin is the pdftotext binary, looked up in PATH.
+	Bin string
+	// Timeout bounds one run.
+	Timeout time.Duration
+	// MaxBytes is the most text kept per file.
+	MaxBytes int
+	// TempDir is where the PDF is copied for the tool.
+	TempDir string
+}
+
+// DefaultPDFText is pdftotext with a 60 s timeout and 1 MiB of text per
+// file, with its temp files under storage/app/tmp in the app's working
+// directory (beside the uploads bucket, not a tmpfs /tmp).
+func DefaultPDFText() PDFText {
+	return PDFText{
+		Bin:      "pdftotext",
+		Timeout:  60 * time.Second,
+		MaxBytes: 1 << 20,
+		TempDir:  filepath.Join("storage", "app", "tmp"),
+	}
+}
+
+// Extract returns the text of the PDF read from r. A PDF with no text layer
+// (a scan) or a broken one makes pdftotext fail; that is not an error: the
+// text read so far, usually "", comes back with a nil error. truncated is
+// set when the text went over MaxBytes or the run timed out. A missing
+// binary is ErrPDFToolMissing; a cancelled ctx is its error.
+func (p PDFText) Extract(ctx context.Context, r io.Reader) (string, bool, error) {
+	bin := p.Bin
+	if bin == "" {
+		bin = "pdftotext"
+	}
+	dir := p.TempDir
+	if dir == "" {
+		dir = os.TempDir()
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", false, err
+	}
+	tmp, err := os.CreateTemp(dir, "jz-bm-pdftext-*.pdf")
+	if err != nil {
+		return "", false, err
+	}
+	path := tmp.Name()
+	defer os.Remove(path)
+	if _, err := io.Copy(tmp, r); err != nil {
+		tmp.Close()
+		return "", false, err
+	}
+	if err := tmp.Close(); err != nil {
+		return "", false, err
+	}
+
+	runCtx := ctx
+	if p.Timeout > 0 {
+		var cancel context.CancelFunc
+		runCtx, cancel = context.WithTimeout(ctx, p.Timeout)
+		defer cancel()
+	}
+	cmd := exec.CommandContext(runCtx, bin, "-q", "-enc", "UTF-8", path, "-")
+	// No stdin (the PDF is a path) and stderr is discarded.
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", false, err
+	}
+	if err := cmd.Start(); err != nil {
+		if errors.Is(err, exec.ErrNotFound) {
+			return "", false, ErrPDFToolMissing
+		}
+		return "", false, err
+	}
+	limit := p.MaxBytes
+	if limit <= 0 {
+		limit = 1 << 20
+	}
+	out, _ := io.ReadAll(io.LimitReader(stdout, int64(limit)+1))
+	truncated := len(out) > limit
+	if truncated {
+		out = out[:limit]
+		// The rest is not wanted; stop the tool instead of draining it.
+		_ = cmd.Process.Kill()
+	}
+	_ = cmd.Wait()
+	if ctx.Err() != nil {
+		return "", false, ctx.Err()
+	}
+	if runCtx.Err() != nil {
+		truncated = true
+	}
+	return CleanExtractedText(string(out)), truncated, nil
+}
+
+// CleanExtractedText removes NUL bytes, which a Postgres TEXT column
+// refuses, and invalid UTF-8 (such as a sequence cut at the size cap) from
+// text a tool extracted.
+func CleanExtractedText(s string) string {
+	s = strings.ReplaceAll(s, "\x00", "")
+	return strings.ToValidUTF8(s, "")
+}