package bm_test import ( "bytes" "encoding/base64" "encoding/json" "net/http" "net/http/httptest" "os/exec" "slices" "strconv" "strings" "testing" userclasses "git.golem15.com/golem15/sm-user-plugin/classes" "git.golem15.com/golem15/summercms/modules/surf" "git.golem15.com/jakub/sm-bm-plugin/classes" "git.golem15.com/jakub/sm-bm-plugin/internal/pdftest" "git.golem15.com/jakub/sm-bm-plugin/internal/searchtest" "git.golem15.com/jakub/sm-bm-plugin/jobs" "gocloud.dev/blob" _ "gocloud.dev/blob/memblob" ) // TestMaterialFileTextAgainstDatabase drives file search end to end against // a migrated database and a Typesense stand-in: an inline PDF attach queues // one extraction job in its transaction, the extraction stores the PDF's // text, the sync sends it as file_texts, and a detach drops the text row // (cascade) and resyncs the document without it. func TestMaterialFileTextAgainstDatabase(t *testing.T) { stub := &searchtest.Stub{} app, plugins, gdb := migratedWith(t, searchtest.Start(t, stub)...) if _, err := exec.LookPath("pdftotext"); err != nil { t.Skip("pdftotext is not installed (poppler-utils)") } bucket, err := blob.OpenBucket(t.Context(), "mem://") if err != nil { t.Fatal(err) } t.Cleanup(func() { bucket.Close() }) if err := app.Publish(bucket); err != nil { t.Fatal(err) } handler, err := surf.Assemble(app, plugins) if err != nil { t.Fatal(err) } managerID := createUser(t, gdb, "files-manager@example.test") var groupID uint if err := gdb.Raw(` INSERT INTO user_groups (name, code, permissions, created_at, updated_at) VALUES ('BM materials', 'bm-materials', '{"jz.bm.access_materials":1}', NOW(), NOW()) RETURNING id`).Scan(&groupID).Error; err != nil { t.Fatal(err) } if err := gdb.Exec(`INSERT INTO users_groups (user_id, user_group_id) VALUES (?, ?)`, managerID, groupID).Error; err != nil { t.Fatal(err) } secret, token, err := userclasses.MintAPIToken("usr_", managerID, "BM file text", []string{"read", "write"}, nil, nil) if err != nil { t.Fatal(err) } if err := gdb.Create(token).Error; err != nil { t.Fatal(err) } send := func(method, path string, payload any) (int, map[string]any) { t.Helper() var buf bytes.Buffer if payload != nil { if err := json.NewEncoder(&buf).Encode(payload); err != nil { t.Fatal(err) } } req := httptest.NewRequest(method, path, &buf) req.Header.Set("Authorization", "Bearer "+secret) if payload != nil { req.Header.Set("Content-Type", "application/json") } rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) var out map[string]any if rec.Body.Len() > 0 { _ = json.Unmarshal(rec.Body.Bytes(), &out) } return rec.Code, out } code, out := send(http.MethodPost, "/api/v1/bm/materials", map[string]any{"title": "Wykład z PDF", "type": "notatka"}) if code != http.StatusCreated { t.Fatalf("POST material = %d %v", code, out) } materialID := id(obj(out["data"])["id"]) n, _ := strconv.ParseUint(materialID, 10, 64) mid := uint(n) const phrase = "Globalizacja i demokracja" code, out = send(http.MethodPost, "/api/v1/bm/materials/"+materialID+"/files/inline", map[string]any{ "name": "wyklad.pdf", "content_base64": base64.StdEncoding.EncodeToString(pdftest.OnePage(phrase)), }) if code != http.StatusCreated { t.Fatalf("inline attach = %d %v", code, out) } var queued int64 if err := gdb.Raw(`SELECT COUNT(*) FROM river_job WHERE kind = ?`, jobs.ExtractMaterialTextKind).Scan(&queued).Error; err != nil { t.Fatal(err) } if queued != 1 { t.Errorf("river_job rows of kind %s = %d, want 1", jobs.ExtractMaterialTextKind, queued) } pdf := classes.DefaultPDFText() pdf.TempDir = t.TempDir() added, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid) if err != nil || added != 1 { t.Fatalf("EnsureFileTexts = %d, %v, want 1 row", added, err) } var texts []string if err := gdb.Raw(`SELECT text FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&texts).Error; err != nil { t.Fatal(err) } if len(texts) != 1 || !strings.Contains(texts[0], phrase) { t.Fatalf("text rows = %q, want one holding %q", texts, phrase) } // A second run finds nothing new. if again, err := classes.EnsureFileTexts(t.Context(), gdb, bucket, pdf, mid); err != nil || again != 0 { t.Errorf("second EnsureFileTexts = %d, %v, want 0", again, err) } lastDoc := func() map[string]any { t.Helper() var doc map[string]any for _, d := range stub.Imported() { if d["id"] == materialID { doc = d } } if doc == nil { t.Fatalf("material %s was never upserted", materialID) } return doc } ix := classes.NewMaterialIndexer(app) ix.Sync(t.Context(), gdb, mid) fileTexts, _ := lastDoc()["file_texts"].([]any) if len(fileTexts) != 1 || !strings.Contains(fileTexts[0].(string), phrase) { t.Errorf("upserted file_texts = %v, want the PDF's text", lastDoc()["file_texts"]) } var fileIDs []uint if err := gdb.Raw(`SELECT id FROM system_files WHERE attachment_id = ? AND field = 'files'`, materialID).Scan(&fileIDs).Error; err != nil { t.Fatal(err) } if len(fileIDs) != 1 { t.Fatalf("files = %v, want one", fileIDs) } before := len(stub.Imported()) code, out = send(http.MethodDelete, "/api/v1/bm/materials/"+materialID+"/files/"+strconv.FormatUint(uint64(fileIDs[0]), 10), nil) if code != http.StatusNoContent { t.Fatalf("detach = %d %v", code, out) } var left int64 if err := gdb.Raw(`SELECT COUNT(*) FROM jz_bm_material_file_texts WHERE material_id = ?`, mid).Scan(&left).Error; err != nil { t.Fatal(err) } if left != 0 { t.Errorf("text rows after detach = %d, want 0 (cascade)", left) } if len(stub.Imported()) <= before { t.Fatal("the detach did not resync the material") } if _, ok := lastDoc()["file_texts"]; ok { t.Errorf("the document after detach still has file_texts: %v", lastDoc()) } if slices.Contains(stub.Removed(), materialID) { t.Error("the detach removed the material's document instead of resyncing it") } }