feat: extract file text on every attach path and backfill on reindex

- inline and signed uploads queue the text job in their transaction
- the admin form queues it in the save transaction
- detach resyncs the material (its text row goes by cascade)
- bm:materials-reindex backfills missing PDF texts before each sync
This commit is contained in:
Jakub Zych
2026-11-08 20:16:14 +01:00
parent 412821ec56
commit f841e8ed71
11 changed files with 480 additions and 22 deletions

View File

@@ -1,15 +1,20 @@
package classes_test
import (
"bytes"
"context"
"errors"
"log/slog"
"net/http"
"slices"
"strings"
"testing"
"git.golem15.com/golem15/summercms/modules/pact"
"git.golem15.com/jakub/sm-bm-plugin/classes"
"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
"git.golem15.com/jakub/sm-bm-plugin/jobs"
"gorm.io/gorm"
)
func TestMaterialSearchGate(t *testing.T) {
@@ -119,3 +124,60 @@ func materialSearchUnavailable(t *testing.T) {
none.Sync(context.Background(), nil, 1)
none.RemoveAfterCommit(context.Background(), nil, 1)
}
func TestMaterialQueueExtract(t *testing.T) {
ctx := context.Background()
// A nil indexer, or one without Enqueue, is a no-op.
var none *classes.MaterialIndexer
none.QueueExtract(ctx, nil, 3)
(&classes.MaterialIndexer{}).QueueExtract(ctx, nil, 3)
var logs bytes.Buffer
var got []pact.JobArgs
var txs []*gorm.DB
ix := &classes.MaterialIndexer{
Log: slog.New(slog.NewTextHandler(&logs, nil)),
Enqueue: func(_ context.Context, tx *gorm.DB, args pact.JobArgs) error {
got = append(got, args)
txs = append(txs, tx)
return nil
},
}
tx := &gorm.DB{}
ix.QueueExtract(ctx, tx, 12)
ix.QueueExtract(ctx, tx, 0)
if len(got) != 1 {
t.Fatalf("enqueued %d jobs, want 1 (id 0 queues nothing)", len(got))
}
args, ok := got[0].(jobs.ExtractMaterialTextArgs)
if !ok || args.MaterialID != 12 || args.Kind() != "jz.bm.extract_material_text" {
t.Errorf("args = %#v, want ExtractMaterialTextArgs{12} of kind jz.bm.extract_material_text", got[0])
}
if txs[0] != tx {
t.Error("the job was not enqueued in the given transaction")
}
if logs.Len() != 0 {
t.Errorf("a successful enqueue logged %q", logs.String())
}
// A failing Enqueue is logged with the material id and not returned.
ix.Enqueue = func(context.Context, *gorm.DB, pact.JobArgs) error { return errors.New("river is down") }
ix.QueueExtract(ctx, tx, 12)
if out := logs.String(); !strings.Contains(out, "could not queue the file text job") || !strings.Contains(out, "material_id=12") || !strings.Contains(out, "river is down") {
t.Errorf("log = %q, want the failure with the material id", out)
}
// The indexer of an app resolves conga at call time; without an app the
// enqueue fails and is only logged.
logs.Reset()
app := classes.NewMaterialIndexer(nil)
if app.Enqueue == nil {
t.Fatal("NewMaterialIndexer left Enqueue nil")
}
app.Log = slog.New(slog.NewTextHandler(&logs, nil))
app.QueueExtract(ctx, nil, 5)
if !strings.Contains(logs.String(), "material_id=5") {
t.Errorf("log = %q, want the failed enqueue logged", logs.String())
}
}