feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
Jakub Zych
200
controllers/api/material_search_test.go
Normal file
200
controllers/api/material_search_test.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/beachcomber"
|
||||
"git.golem15.com/golem15/summercms/modules/bouncer"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/classes"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/controllers/api"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/models"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func segments(parts ...any) []beachcomber.Segment {
|
||||
out := make([]beachcomber.Segment, 0, len(parts)/2)
|
||||
for i := 0; i+1 < len(parts); i += 2 {
|
||||
out = append(out, beachcomber.Segment{Text: parts[i].(string), Match: parts[i+1].(bool)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestMaterialSnippet(t *testing.T) {
|
||||
body := "# Wstęp\n\nNa początku. Proces **globalizacji** zmienia państwa. Koniec tekstu."
|
||||
m := models.Material{ID: 3, Body: &body}
|
||||
hit := beachcomber.Hit{ID: "3", Highlights: []beachcomber.Snippet{
|
||||
{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
|
||||
{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia <b>państwa</b>.", false)},
|
||||
}}
|
||||
s := api.PickSnippet(hit, m)
|
||||
if s == nil {
|
||||
t.Fatal("no snippet")
|
||||
}
|
||||
if s.Source != "body" || s.FileName != nil {
|
||||
t.Errorf("source %q file %v, want body and no file", s.Source, s.FileName)
|
||||
}
|
||||
if len(s.Segments) != 3 || s.Segments[1].Text != "globalizacji" || !s.Segments[1].Hit || s.Segments[0].Hit {
|
||||
t.Errorf("segments = %+v", s.Segments)
|
||||
}
|
||||
if s.Segments[2].Text != " zmienia <b>państwa</b>." {
|
||||
t.Errorf("segment text was changed: %q", s.Segments[2].Text)
|
||||
}
|
||||
if !s.Lead || !s.Trail {
|
||||
t.Errorf("lead %v trail %v, want both (cut inside the body)", s.Lead, s.Trail)
|
||||
}
|
||||
b, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := string(b); !strings.Contains(got, `"source":"body","file_name":null,"lead":true,"trail":true,"segments":[{"text":"Proces ","hit":false}`) {
|
||||
t.Errorf("JSON = %s", got)
|
||||
}
|
||||
|
||||
// The whole body: neither lead nor trail.
|
||||
short := "Krótka **notatka**"
|
||||
whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
|
||||
if whole == nil || whole.Lead || whole.Trail {
|
||||
t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
|
||||
}
|
||||
// Start of the body only: trail but no lead.
|
||||
start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
|
||||
if start == nil || start.Lead || !start.Trail {
|
||||
t.Errorf("start snippet = %+v, want trail only", start)
|
||||
}
|
||||
|
||||
// A snippet on another field only, or none at all, gives no snippet.
|
||||
if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
|
||||
t.Errorf("title-only hit gave %+v", got)
|
||||
}
|
||||
if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
|
||||
t.Errorf("hit without highlights gave %+v", got)
|
||||
}
|
||||
|
||||
if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia <b>państwa</b>.…" {
|
||||
t.Errorf("FlattenSnippet = %q", got)
|
||||
}
|
||||
if got := api.FlattenSnippet(nil); got != "" {
|
||||
t.Errorf("FlattenSnippet(nil) = %q", got)
|
||||
}
|
||||
if got := api.RankOrder([]uint{9, 3}); got != "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" {
|
||||
t.Errorf("RankOrder = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterialsSearchUnavailable(t *testing.T) {
|
||||
h := noDB()
|
||||
user := &bouncer.Principal{ID: 7}
|
||||
code, body := serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
|
||||
if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." {
|
||||
t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body)
|
||||
}
|
||||
// One rune is no search: the list answers as without q (no database).
|
||||
code, body = serve(t, h.Materials, http.MethodGet, "/?q=%20g%20", user, nil)
|
||||
if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
|
||||
t.Errorf("one-rune q = %d %v, want the no-database answer", code, body)
|
||||
}
|
||||
code, body = serve(t, h.Materials, http.MethodGet, "/", user, nil)
|
||||
if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
|
||||
t.Errorf("no q = %d %v, want the no-database answer", code, body)
|
||||
}
|
||||
// An unconfigured engine is the same 503.
|
||||
h.MaterialIndex = classes.NewMaterialIndexer(searchtest.App(t))
|
||||
code, body = serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
|
||||
if code != http.StatusServiceUnavailable || body["message"] != "Search is unavailable." {
|
||||
t.Errorf("q with a null engine = %d %v, want 503 Search is unavailable.", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// sqlRecorder is a dry-run postgres handle that records every query it
|
||||
// builds, with its arguments inlined.
|
||||
type sqlRecorder struct {
|
||||
mu sync.Mutex
|
||||
sql []string
|
||||
}
|
||||
|
||||
func (r *sqlRecorder) queries() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.sql...)
|
||||
}
|
||||
|
||||
func dryDB(t *testing.T) (*gorm.DB, *sqlRecorder) {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := &sqlRecorder{}
|
||||
err = db.Callback().Query().After("gorm:query").Register("test:record_sql", func(tx *gorm.DB) {
|
||||
rec.mu.Lock()
|
||||
defer rec.mu.Unlock()
|
||||
rec.sql = append(rec.sql, tx.Dialector.Explain(tx.Statement.SQL.String(), tx.Statement.Vars...))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, rec
|
||||
}
|
||||
|
||||
func TestMaterialsSearchTrace(t *testing.T) {
|
||||
stub := &searchtest.Stub{Hits: []searchtest.Hit{
|
||||
{ID: "9"},
|
||||
{ID: "3", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd},
|
||||
}}
|
||||
db, rec := dryDB(t)
|
||||
h := &api.Handlers{
|
||||
DB: func() *gorm.DB { return db },
|
||||
MaterialIndex: classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)),
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/materials?q=globalizacja&course=7", nil)
|
||||
req = req.WithContext(bouncer.WithUser(req.Context(), &bouncer.Principal{ID: 5}))
|
||||
w := httptest.NewRecorder()
|
||||
h.Materials(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if n := len(stub.Searches()); n != 1 {
|
||||
t.Errorf("searches = %d, want 1", n)
|
||||
}
|
||||
var count, page string
|
||||
for _, q := range rec.queries() {
|
||||
switch {
|
||||
case strings.HasPrefix(q, "SELECT count(*)"):
|
||||
count = q
|
||||
case strings.HasPrefix(q, "SELECT") && strings.Contains(q, `FROM "jz_bm_materials"`):
|
||||
page = q
|
||||
}
|
||||
}
|
||||
if count == "" || page == "" {
|
||||
t.Fatalf("queries = %q, want a count and a page", rec.queries())
|
||||
}
|
||||
for name, q := range map[string]string{"count": count, "page": page} {
|
||||
for _, want := range []string{"jz_bm_materials.is_private = FALSE", "jz_bm_materials.published_at IS NOT NULL", "jz_bm_materials.course_id = 7", "jz_bm_materials.id IN (9,3)"} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Errorf("%s query lacks %q:\n%s", name, want, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(page, "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id)") {
|
||||
t.Errorf("page query is not in rank order:\n%s", page)
|
||||
}
|
||||
var out struct {
|
||||
Data []map[string]any `json:"data"`
|
||||
Meta struct {
|
||||
Total int `json:"total"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Meta.Total != 0 || len(out.Data) != 0 {
|
||||
t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user