412821ec56
- jz_bm_material_file_texts caches the pdftotext output per file - the River job jz.bm.extract_material_text extracts and resyncs - the management multipart attach queues it inside its transaction - file_texts in the document and a file snippet with its name
90 lines
2.5 KiB
Go
90 lines
2.5 KiB
Go
package classes_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"errors"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"git.golem15.com/jakub/sm-bm-plugin/classes"
|
|
"git.golem15.com/jakub/sm-bm-plugin/internal/pdftest"
|
|
)
|
|
|
|
func TestPDFTextExtract(t *testing.T) {
|
|
if _, err := exec.LookPath("pdftotext"); err != nil {
|
|
t.Skip("pdftotext is not installed (poppler-utils)")
|
|
}
|
|
ctx := context.Background()
|
|
dir := t.TempDir()
|
|
pdf := classes.DefaultPDFText()
|
|
pdf.TempDir = dir
|
|
if pdf.Bin != "pdftotext" || pdf.Timeout != 60*time.Second || pdf.MaxBytes != 1<<20 {
|
|
t.Errorf("DefaultPDFText = %+v, want pdftotext, 60 s, 1 MiB", pdf)
|
|
}
|
|
doc := pdftest.OnePage("Globalizacja i demokracja")
|
|
|
|
text, truncated, err := pdf.Extract(ctx, bytes.NewReader(doc))
|
|
if err != nil {
|
|
t.Fatalf("Extract: %v", err)
|
|
}
|
|
if !strings.Contains(text, "Globalizacja i demokracja") || truncated {
|
|
t.Errorf("Extract = %q truncated %v, want the phrase, not truncated", text, truncated)
|
|
}
|
|
|
|
small := pdf
|
|
small.MaxBytes = 5
|
|
text, truncated, err = small.Extract(ctx, bytes.NewReader(doc))
|
|
if err != nil || !truncated || len(text) > 5 {
|
|
t.Errorf("MaxBytes 5: %q truncated %v err %v, want at most 5 bytes and truncated", text, truncated, err)
|
|
}
|
|
|
|
junk := make([]byte, 4096)
|
|
_, _ = rand.Read(junk)
|
|
text, _, err = pdf.Extract(ctx, bytes.NewReader(junk))
|
|
if err != nil || text != "" {
|
|
t.Errorf("random bytes: %q err %v, want empty text and no error", text, err)
|
|
}
|
|
|
|
missing := pdf
|
|
missing.Bin = "jz-bm-no-such-pdftotext"
|
|
if _, _, err := missing.Extract(ctx, bytes.NewReader(doc)); !errors.Is(err, classes.ErrPDFToolMissing) {
|
|
t.Errorf("missing binary: err %v, want ErrPDFToolMissing", err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(entries) != 0 {
|
|
t.Errorf("the temp dir holds %d files after extraction, want none", len(entries))
|
|
}
|
|
// A temp dir Extract creates itself is private to the app user.
|
|
sub := pdf
|
|
sub.TempDir = dir + "/sub"
|
|
if _, _, err := sub.Extract(ctx, bytes.NewReader(doc)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if info, err := os.Stat(sub.TempDir); err != nil || info.Mode().Perm() != 0o700 {
|
|
t.Errorf("created temp dir: %v %v, want mode 0700", info, err)
|
|
}
|
|
|
|
for _, c := range []struct{ in, want string }{
|
|
{"plain tekst", "plain tekst"},
|
|
{"a\x00b\x00", "ab"},
|
|
{"zażółć\xff\xfe gęślą", "zażółć gęślą"},
|
|
{"cut \xc5", "cut "},
|
|
{"", ""},
|
|
} {
|
|
got := classes.CleanExtractedText(c.in)
|
|
if got != c.want || !utf8.ValidString(got) {
|
|
t.Errorf("CleanExtractedText(%q) = %q, want %q", c.in, got, c.want)
|
|
}
|
|
}
|
|
}
|