package classes import ( "context" "errors" "io" "os" "os/exec" "path/filepath" "strings" "time" ) // ErrPDFToolMissing is returned by Extract when the pdftotext binary is not // installed (poppler-utils). Nothing is stored then, so a later run // retries. var ErrPDFToolMissing = errors.New("jz.bm: pdftotext is not installed") // PDFText runs pdftotext (poppler-utils) on one PDF. The PDF comes from a // material manager, so it is untrusted input to a native tool: it runs // without a shell, from a 0600 temp file in a 0700 directory that is // removed afterwards, for at most Timeout and with at most MaxBytes of text // read. It runs only in the extraction job and the reindex command, never in // a request. type PDFText struct { // Bin is the pdftotext binary, looked up in PATH. Bin string // Timeout bounds one run. Timeout time.Duration // MaxBytes is the most text kept per file. MaxBytes int // TempDir is where the PDF is copied for the tool. TempDir string } // DefaultPDFText is pdftotext with a 60 s timeout and 1 MiB of text per // file, with its temp files under storage/app/tmp in the app's working // directory (beside the uploads bucket, not a tmpfs /tmp). func DefaultPDFText() PDFText { return PDFText{ Bin: "pdftotext", Timeout: 60 * time.Second, MaxBytes: 1 << 20, TempDir: filepath.Join("storage", "app", "tmp"), } } // Extract returns the text of the PDF read from r. A PDF with no text layer // (a scan) or a broken one makes pdftotext fail; that is not an error: the // text read so far, usually "", comes back with a nil error. truncated is // set when the text went over MaxBytes or the run timed out. A missing // binary is ErrPDFToolMissing; a cancelled ctx is its error. func (p PDFText) Extract(ctx context.Context, r io.Reader) (string, bool, error) { bin := p.Bin if bin == "" { bin = "pdftotext" } dir := p.TempDir if dir == "" { dir = os.TempDir() } if err := os.MkdirAll(dir, 0o700); err != nil { return "", false, err } tmp, err := os.CreateTemp(dir, "jz-bm-pdftext-*.pdf") if err != nil { return "", false, err } path := tmp.Name() defer os.Remove(path) if _, err := io.Copy(tmp, r); err != nil { tmp.Close() return "", false, err } if err := tmp.Close(); err != nil { return "", false, err } runCtx := ctx if p.Timeout > 0 { var cancel context.CancelFunc runCtx, cancel = context.WithTimeout(ctx, p.Timeout) defer cancel() } cmd := exec.CommandContext(runCtx, bin, "-q", "-enc", "UTF-8", path, "-") // No stdin (the PDF is a path) and stderr is discarded. stdout, err := cmd.StdoutPipe() if err != nil { return "", false, err } if err := cmd.Start(); err != nil { if errors.Is(err, exec.ErrNotFound) { return "", false, ErrPDFToolMissing } return "", false, err } limit := p.MaxBytes if limit <= 0 { limit = 1 << 20 } out, _ := io.ReadAll(io.LimitReader(stdout, int64(limit)+1)) truncated := len(out) > limit if truncated { out = out[:limit] // The rest is not wanted; stop the tool instead of draining it. _ = cmd.Process.Kill() } _ = cmd.Wait() if ctx.Err() != nil { return "", false, ctx.Err() } if runCtx.Err() != nil { truncated = true } return CleanExtractedText(string(out)), truncated, nil } // CleanExtractedText removes NUL bytes, which a Postgres TEXT column // refuses, and invalid UTF-8 (such as a sequence cut at the size cap) from // text a tool extracted. func CleanExtractedText(s string) string { s = strings.ReplaceAll(s, "\x00", "") return strings.ToValidUTF8(s, "") }