sm-bm-plugin/material_file_text_postgres_test.go

package bm_test

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os/exec"
	"slices"
	"strconv"
	"strings"
	"testing"

	userclasses "git.golem15.com/golem15/sm-user-plugin/classes"
	"git.golem15.com/golem15/summercms/modules/surf"
	"git.golem15.com/jakub/sm-bm-plugin/classes"
	"git.golem15.com/jakub/sm-bm-plugin/internal/pdftest"
	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
	"git.golem15.com/jakub/sm-bm-plugin/jobs"
	"gocloud.dev/blob"
	_ "gocloud.dev/blob/memblob"
)

// TestMaterialFileTextAgainstDatabase drives file search end to end against
// a migrated database and a Typesense stand-in: an inline PDF attach queues
// one extraction job in its transaction, the extraction stores the PDF's
// text, the sync sends it as file_texts, and a detach drops the text row
// (cascade) and resyncs the document without it.
func TestMaterialFileTextAgainstDatabase(t *testing.T) {
	stub := &searchtest.Stub{}
	app, plugins, gdb := migratedWith(t, searchtest.Start(t, stub)...)
	if _, err := exec.LookPath("pdftotext"); err != nil {
		t.Skip("pdftotext is not installed (poppler-utils)")
	}
	bucket, err := blob.OpenBucket(t.Context(), "mem://")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { bucket.Close() })
	if err := app.Publish(bucket); err != nil {
		t.Fatal(err)
	}
	handler, err := surf.Assemble(app, plugins)
	if err != nil {
		t.Fatal(err)
	}
	managerID := createUser(t, gdb, "files-manager@example.test")
	var groupID uint
	if err := gdb.Raw(`
INSERT INTO user_groups (name, code, permissions, created_at, updated_at)
VALUES ('BM materials', 'bm-materials', '{"jz.bm.access_materials":1}', NOW(), NOW())
RETURNING id`).Scan(&groupID).Error; err != nil {
		t.Fatal(err)
	}
	if err := gdb.Exec(`INSERT INTO users_groups (user_id, user_group_id) VALUES (?, ?)`, managerID, groupID).Error; err != nil {
		t.Fatal(err)
	}
	secret, token, err := userclasses.MintAPIToken("usr_", managerID, "BM file text", []string{"read", "write"}, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := gdb.Create(token).Error; err != nil {
		t.Fatal(err)
	}
	send := func(method, path string, payload any) (int, map[string]any) {
		t.Helper()
		var buf bytes.Buffer
		if payload != nil {
			if err := json.NewEncoder(&buf).Encode(payload); err != nil {
				t.Fatal(err)
			}
		}
		req := httptest.NewRequest(method, path, &buf)
		req.Header.Set("Authorization", "Bearer "+secret)
		if payload != nil {
			req.Header.Set("Content-Type", "application/json")
		}
		rec := httptest.NewRecorder()
		handler.ServeHTTP(rec, req)
		var out map[string]any
		if rec.Body.Len() > 0 {
			_ = json.Unmarshal(rec.Body.Bytes(), &out)
		}
		return rec.Code, out
	}

	code, out := send(http.MethodPost, "/api/v1/bm/materials", map[string]any{"title": "Wykład z PDF", "type": "notatka"})
	if code != http.StatusCreated {
		t.Fatalf("POST material = %d %v", code, out)
	}
	materialID := id(obj(out["data"])["id"])
	n, _ := strconv.ParseUint(materialID, 10, 64)
	mid := uint(n)

	const phrase = "Globalizacja i demokracja"
	code, out = send(http.MethodPost, "/api/v1/bm/materials/"+materialID+"/files/inline", map[string]any{
		"name":           "wyklad.pdf",
		"content_base64": base64.StdEncoding.EncodeToString(pdftest.OnePage(phrase)),
	})
	if code != http.StatusCreated {
		t.Fatalf("inline attach = %d %v", code, out)
	}
	var queued int64
	if err := gdb.Raw(`SELECT COUNT(*) FROM river_job WHERE kind = ?`, jobs.ExtractMaterialTextKind).Scan(&queued).Error; err != nil {
		t.Fatal(err)
	}
	if queued != 1 {
		t.Errorf("river_job rows of kind %s = %d, want 1", jobs.ExtractMaterialTextKind, queued)
	}

	pdf := classes.DefaultPDFText()
	pdf.TempDir = t.TempDir()
	added, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid)
	if err != nil || added != 1 {
		t.Fatalf("EnsureFileTexts = %d, %v, want 1 row", added, err)
	}
	var texts []string
	if err := gdb.Raw(`SELECT text FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&texts).Error; err != nil {
		t.Fatal(err)
	}
	if len(texts) != 1 || !strings.Contains(texts[0], phrase) {
		t.Fatalf("text rows = %q, want one holding %q", texts, phrase)
	}
	// A second run finds nothing new.
	if again, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid); err != nil || again != 0 {
		t.Errorf("second EnsureFileTexts = %d, %v, want 0", again, err)
	}

	lastDoc := func() map[string]any {
		t.Helper()
		var doc map[string]any
		for _, d := range stub.Imported() {
			if d["id"] == materialID {
				doc = d
			}
		}
		if doc == nil {
			t.Fatalf("material %s was never upserted", materialID)
		}
		return doc
	}
	ix := classes.NewMaterialIndexer(app)
	ix.Sync(t.Context(), gdb, mid)
	fileTexts, _ := lastDoc()["file_texts"].([]any)
	if len(fileTexts) != 1 || !strings.Contains(fileTexts[0].(string), phrase) {
		t.Errorf("upserted file_texts = %v, want the PDF's text", lastDoc()["file_texts"])
	}

	var fileIDs []uint
	if err := gdb.Raw(`SELECT id FROM system_files WHERE attachment_id = ? AND field = 'files'`, materialID).Scan(&fileIDs).Error; err != nil {
		t.Fatal(err)
	}
	if len(fileIDs) != 1 {
		t.Fatalf("files = %v, want one", fileIDs)
	}
	before := len(stub.Imported())
	code, out = send(http.MethodDelete, "/api/v1/bm/materials/"+materialID+"/files/"+strconv.FormatUint(uint64(fileIDs[0]), 10), nil)
	if code != http.StatusNoContent {
		t.Fatalf("detach = %d %v", code, out)
	}
	var left int64
	if err := gdb.Raw(`SELECT COUNT(*) FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&left).Error; err != nil {
		t.Fatal(err)
	}
	if left != 0 {
		t.Errorf("text rows after detach = %d, want 0 (cascade)", left)
	}
	if len(stub.Imported()) <= before {
		t.Fatal("the detach did not resync the material")
	}
	if _, ok := lastDoc()["file_texts"]; ok {
		t.Errorf("the document after detach still has file_texts: %v", lastDoc())
	}
	if slices.Contains(stub.Removed(), materialID) {
		t.Error("the detach removed the material's document instead of resyncing it")
	}
}