412821ec56
- jz_bm_material_file_texts caches the pdftotext output per file - the River job jz.bm.extract_material_text extracts and resyncs - the management multipart attach queues it inside its transaction - file_texts in the document and a file snippet with its name
125 lines
3.5 KiB
Go
125 lines
3.5 KiB
Go
package classes
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ErrPDFToolMissing is returned by Extract when the pdftotext binary is not
|
|
// installed (poppler-utils). Nothing is stored then, so a later run
|
|
// retries.
|
|
var ErrPDFToolMissing = errors.New("jz.bm: pdftotext is not installed")
|
|
|
|
// PDFText runs pdftotext (poppler-utils) on one PDF. The PDF comes from a
|
|
// material manager, so it is untrusted input to a native tool: it runs
|
|
// without a shell, from a 0600 temp file in a 0700 directory that is
|
|
// removed afterwards, for at most Timeout and with at most MaxBytes of text
|
|
// read. It runs only in the extraction job and the reindex command, never in
|
|
// a request.
|
|
type PDFText struct {
|
|
// Bin is the pdftotext binary, looked up in PATH.
|
|
Bin string
|
|
// Timeout bounds one run.
|
|
Timeout time.Duration
|
|
// MaxBytes is the most text kept per file.
|
|
MaxBytes int
|
|
// TempDir is where the PDF is copied for the tool.
|
|
TempDir string
|
|
}
|
|
|
|
// DefaultPDFText is pdftotext with a 60 s timeout and 1 MiB of text per
|
|
// file, with its temp files under storage/app/tmp in the app's working
|
|
// directory (beside the uploads bucket, not a tmpfs /tmp).
|
|
func DefaultPDFText() PDFText {
|
|
return PDFText{
|
|
Bin: "pdftotext",
|
|
Timeout: 60 * time.Second,
|
|
MaxBytes: 1 << 20,
|
|
TempDir: filepath.Join("storage", "app", "tmp"),
|
|
}
|
|
}
|
|
|
|
// Extract returns the text of the PDF read from r. A PDF with no text layer
|
|
// (a scan) or a broken one makes pdftotext fail; that is not an error: the
|
|
// text read so far, usually "", comes back with a nil error. truncated is
|
|
// set when the text went over MaxBytes or the run timed out. A missing
|
|
// binary is ErrPDFToolMissing; a cancelled ctx is its error.
|
|
func (p PDFText) Extract(ctx context.Context, r io.Reader) (string, bool, error) {
|
|
bin := p.Bin
|
|
if bin == "" {
|
|
bin = "pdftotext"
|
|
}
|
|
dir := p.TempDir
|
|
if dir == "" {
|
|
dir = os.TempDir()
|
|
}
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return "", false, err
|
|
}
|
|
tmp, err := os.CreateTemp(dir, "jz-bm-pdftext-*.pdf")
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
path := tmp.Name()
|
|
defer os.Remove(path)
|
|
if _, err := io.Copy(tmp, r); err != nil {
|
|
tmp.Close()
|
|
return "", false, err
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return "", false, err
|
|
}
|
|
|
|
runCtx := ctx
|
|
if p.Timeout > 0 {
|
|
var cancel context.CancelFunc
|
|
runCtx, cancel = context.WithTimeout(ctx, p.Timeout)
|
|
defer cancel()
|
|
}
|
|
cmd := exec.CommandContext(runCtx, bin, "-q", "-enc", "UTF-8", path, "-")
|
|
// No stdin (the PDF is a path) and stderr is discarded.
|
|
stdout, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if err := cmd.Start(); err != nil {
|
|
if errors.Is(err, exec.ErrNotFound) {
|
|
return "", false, ErrPDFToolMissing
|
|
}
|
|
return "", false, err
|
|
}
|
|
limit := p.MaxBytes
|
|
if limit <= 0 {
|
|
limit = 1 << 20
|
|
}
|
|
out, _ := io.ReadAll(io.LimitReader(stdout, int64(limit)+1))
|
|
truncated := len(out) > limit
|
|
if truncated {
|
|
out = out[:limit]
|
|
// The rest is not wanted; stop the tool instead of draining it.
|
|
_ = cmd.Process.Kill()
|
|
}
|
|
_ = cmd.Wait()
|
|
if ctx.Err() != nil {
|
|
return "", false, ctx.Err()
|
|
}
|
|
if runCtx.Err() != nil {
|
|
truncated = true
|
|
}
|
|
return CleanExtractedText(string(out)), truncated, nil
|
|
}
|
|
|
|
// CleanExtractedText removes NUL bytes, which a Postgres TEXT column
|
|
// refuses, and invalid UTF-8 (such as a sequence cut at the size cap) from
|
|
// text a tool extracted.
|
|
func CleanExtractedText(s string) string {
|
|
s = strings.ReplaceAll(s, "\x00", "")
|
|
return strings.ToValidUTF8(s, "")
|
|
}
|