From 4ffce6e6ac98cfaeae32fbaa0e8397bd1d5b35ee Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Sun, 8 Nov 2026 19:45:46 +0100 Subject: [PATCH] feat: search materials through beachcomber - Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged --- classes/material_index.go | 256 ++++++++++++++++++++++++ classes/material_index_test.go | 121 +++++++++++ controllers/api/api.go | 5 + controllers/api/export_test.go | 8 + controllers/api/handlers_test.go | 1 + controllers/api/json.go | 4 + controllers/api/material_search.go | 123 ++++++++++++ controllers/api/material_search_test.go | 200 ++++++++++++++++++ controllers/api/materials.go | 82 ++++++-- controllers/api/params.go | 1 + go.mod | 10 +- go.sum | 12 -- internal/searchtest/typesense.go | 198 ++++++++++++++++++ models/material_search.go | 173 ++++++++++++++++ models/material_search_test.go | 124 ++++++++++++ plugin.go | 9 +- routes.go | 1 + search.go | 23 +++ 18 files changed, 1315 insertions(+), 36 deletions(-) create mode 100644 classes/material_index.go create mode 100644 classes/material_index_test.go create mode 100644 controllers/api/material_search.go create mode 100644 controllers/api/material_search_test.go create mode 100644 internal/searchtest/typesense.go create mode 100644 models/material_search.go create mode 100644 models/material_search_test.go create mode 100644 search.go diff --git a/classes/material_index.go b/classes/material_index.go new file mode 100644 index 0000000..5b50502 --- /dev/null +++ b/classes/material_index.go @@ -0,0 +1,256 @@ +package classes + +import ( + "context" + "errors" + "fmt" + "log/slog" + "regexp" + "slices" + "strconv" + "strings" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/beachcomber" + "git.golem15.com/golem15/summercms/modules/beachcomber/typesense" + "git.golem15.com/golem15/summercms/modules/diary" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/jakub/sm-bm-plugin/models" + "gorm.io/gorm" +) + +// materialSearchKey marks a context whose syncs belong to jz.bm. +type materialSearchKey struct{} + +// MarkMaterialSearch returns ctx marked as a jz.bm material sync. The +// app-wide beachcomber Gate (MaterialSearchGate) is on only for a marked +// context, so every jz.bm Sync or Remove passes one. +func MarkMaterialSearch(ctx context.Context) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, materialSearchKey{}, true) +} + +// MaterialSearchGate is the app-wide beachcomber Gate jz.bm installs. The +// Gate sees only (ctx, db), not the model, so it is on only for a context +// marked by MarkMaterialSearch: golem15.chatchannels syncs its messages +// with unmarked contexts, and chat messages and DMs never reach the index. +// The automatic callbacks of material writes are off as well; every jz.bm +// write path syncs explicitly. +func MaterialSearchGate() beachcomber.Gate { + return beachcomber.GateFunc(func(ctx context.Context, _ *gorm.DB) bool { + return ctx != nil && ctx.Value(materialSearchKey{}) != nil + }) +} + +// MaterialSearchWindow is how many engine candidates one search re-gates in +// SQL. The list counts and pages over the gated rows, so a page never +// depends on how the engine paged. +const MaterialSearchWindow = typesense.MaxPerPage + +// MaterialQueryBy are the document fields a query is matched against, and +// MaterialQueryWeights their weights in the same order: the title over the +// tags, the source and the course, then the body, the link and file text. +var ( + MaterialQueryBy = []string{"title", "tags", "source", "course", "url", "body", "file_texts"} + MaterialQueryWeights = []int{6, 4, 3, 3, 1, 2, 1} +) + +// materialHighlightFields are the fields snippets are cut from. +var materialHighlightFields = []string{"body", "file_texts"} + +// ErrSearchUnavailable is returned by Search when the engine is not +// configured or the search failed. The API answers it with a 503. +var ErrSearchUnavailable = errors.New("jz.bm: search is unavailable") + +// MaterialSearch is one material search: the query text and the validated +// filters of the list. +type MaterialSearch struct { + Q string + CourseID uint + MeetingID uint + Type string + Tag string +} + +// MaterialCandidates are the engine's answer: the candidate ids in rank +// order and the Hit of each. Both are candidates only: the caller re-gates +// the ids in SQL and reads the Hit of a row only once SQL returned it. +// Highlights reports whether the engine was asked for snippets. +type MaterialCandidates struct { + IDs []uint + Hits map[uint]beachcomber.Hit + Highlights bool +} + +// MaterialIndexer searches the materials index and keeps it current. A nil +// *MaterialIndexer is valid: Search answers ErrSearchUnavailable and the +// syncs do nothing. +type MaterialIndexer struct { + // Service returns the app's beachcomber service, or nil. + Service func() *beachcomber.Service + // Log is where failures are logged; nil means slog.Default. + Log *slog.Logger +} + +// NewMaterialIndexer returns the indexer of app. The service is resolved on +// each call, so it may be built after the indexer. +func NewMaterialIndexer(app *backpack.App) *MaterialIndexer { + return &MaterialIndexer{ + Service: func() *beachcomber.Service { + if app == nil { + return nil + } + svc, err := beachcomber.From(app) + if err != nil { + return nil + } + return svc + }, + Log: diary.For("jz.bm"), + } +} + +func (ix *MaterialIndexer) service() *beachcomber.Service { + if ix == nil || ix.Service == nil { + return nil + } + return ix.Service() +} + +func (ix *MaterialIndexer) log() *slog.Logger { + if ix == nil || ix.Log == nil { + return slog.Default() + } + return ix.Log +} + +// Search asks the engine for up to MaterialSearchWindow candidates of req. +// It answers ErrSearchUnavailable (wrapping the cause) when the service or +// its engine is missing, the engine is not configured or the call fails. +// The query text is never logged. +func (ix *MaterialIndexer) Search(ctx context.Context, req MaterialSearch) (MaterialCandidates, error) { + svc := ix.service() + if svc == nil { + return MaterialCandidates{}, fmt.Errorf("%w: no search service", ErrSearchUnavailable) + } + engine := svc.Engine() + if engine == nil || !engine.Configured() { + return MaterialCandidates{}, fmt.Errorf("%w: the search engine is not configured", ErrSearchUnavailable) + } + q := beachcomber.Query{ + Q: req.Q, + QueryBy: MaterialQueryBy, + QueryByWeights: MaterialQueryWeights, + FilterBy: materialFilterBy(req), + SortBy: "_text_match:desc,published_at:desc", + Page: 1, + PerPage: MaterialSearchWindow, + ExcludeFields: []string{"body", "file_texts"}, + } + highlights := beachcomber.SupportsHighlights(engine) + if highlights { + q.Highlight = &beachcomber.Highlight{Fields: materialHighlightFields, AffixTokens: 12, MaxSnippetsPerArray: 1} + } + res, err := beachcomber.SearchPage(ctx, engine, svc.IndexName(&models.Material{}), q) + if err != nil { + ix.log().Warn("jz.bm: material search failed", "err", err) + return MaterialCandidates{}, fmt.Errorf("%w: %w", ErrSearchUnavailable, err) + } + out := MaterialCandidates{IDs: make([]uint, 0, len(res.IDs)), Hits: make(map[uint]beachcomber.Hit, len(res.IDs)), Highlights: highlights} + for i, raw := range res.IDs { + n, err := strconv.ParseUint(raw, 10, 64) + if err != nil || n == 0 { + continue + } + id := uint(n) + if _, seen := out.Hits[id]; seen { + continue + } + out.IDs = append(out.IDs, id) + hit := beachcomber.Hit{ID: raw} + if i < len(res.Hits) { + hit = res.Hits[i] + } + out.Hits[id] = hit + } + return out, nil +} + +// materialTagSlug is the shape of a tag slug the engine filter may carry. +var materialTagSlug = regexp.MustCompile(`^[a-z0-9-]{1,100}$`) + +// materialFilterBy narrows the engine's candidates to the list filters. It +// is built only from parsed integers, an enumerated type and a tag that is +// a plain slug; any other value is left out and applied by SQL alone, which +// applies every filter anyway. +func materialFilterBy(req MaterialSearch) string { + var parts []string + if req.CourseID != 0 { + parts = append(parts, "course_id:="+strconv.FormatUint(uint64(req.CourseID), 10)) + } + if req.MeetingID != 0 { + parts = append(parts, "meeting_id:="+strconv.FormatUint(uint64(req.MeetingID), 10)) + } + if slices.Contains(models.MaterialTypes, req.Type) { + parts = append(parts, "type:=`"+req.Type+"`") + } + if materialTagSlug.MatchString(req.Tag) { + parts = append(parts, "tag_slugs:=`"+req.Tag+"`") + } + return strings.Join(parts, " && ") +} + +// Sync upserts the document of material id now. A failure is logged with +// the id and never returned: a write never fails because of the index. +func (ix *MaterialIndexer) Sync(ctx context.Context, db *gorm.DB, id uint) { + ix.run(ctx, db, id, false) +} + +// Remove deletes the document of material id now, logging a failure like +// Sync. +func (ix *MaterialIndexer) Remove(ctx context.Context, db *gorm.DB, id uint) { + ix.run(ctx, db, id, true) +} + +// SyncAfterCommit runs Sync once tx commits; with a nil tx, or outside a +// transaction, it runs at once. +func (ix *MaterialIndexer) SyncAfterCommit(ctx context.Context, tx *gorm.DB, id uint) { + ix.afterCommit(ctx, tx, id, false) +} + +// RemoveAfterCommit runs Remove once tx commits, like SyncAfterCommit. +func (ix *MaterialIndexer) RemoveAfterCommit(ctx context.Context, tx *gorm.DB, id uint) { + ix.afterCommit(ctx, tx, id, true) +} + +func (ix *MaterialIndexer) afterCommit(ctx context.Context, tx *gorm.DB, id uint, remove bool) { + if ix == nil || id == 0 { + return + } + if tx == nil { + ix.run(ctx, nil, id, remove) + return + } + lagoon.AfterCommit(ctx, tx, func(ctx context.Context, db *gorm.DB) { + ix.run(ctx, db, id, remove) + }) +} + +func (ix *MaterialIndexer) run(ctx context.Context, db *gorm.DB, id uint, remove bool) { + svc := ix.service() + if svc == nil || id == 0 { + return + } + ctx = MarkMaterialSearch(ctx) + var err error + if remove { + err = svc.Remove(ctx, db, &models.Material{ID: id}) + } else { + err = svc.Sync(ctx, db, &models.Material{ID: id}) + } + if err != nil { + ix.log().Warn("jz.bm: material index sync failed", "material_id", id, "remove", remove, "err", err) + } +} diff --git a/classes/material_index_test.go b/classes/material_index_test.go new file mode 100644 index 0000000..ee6df2b --- /dev/null +++ b/classes/material_index_test.go @@ -0,0 +1,121 @@ +package classes_test + +import ( + "context" + "errors" + "net/http" + "slices" + "strings" + "testing" + + "git.golem15.com/jakub/sm-bm-plugin/classes" + "git.golem15.com/jakub/sm-bm-plugin/internal/searchtest" +) + +func TestMaterialSearchGate(t *testing.T) { + gate := classes.MaterialSearchGate() + if !gate.Enabled(classes.MarkMaterialSearch(context.Background()), nil) { + t.Error("a marked context is off") + } + if gate.Enabled(context.Background(), nil) { + t.Error("an unmarked context (a chat message sync) is on") + } +} + +func TestMaterialSearchRequest(t *testing.T) { + t.Run("unavailable", materialSearchUnavailable) + stub := &searchtest.Stub{Hits: []searchtest.Hit{ + {ID: "12", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd + " dziś"}, + {ID: "abc"}, + {ID: "0"}, + {ID: "5"}, + }} + ix := classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)) + got, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "globalizacja", CourseID: 7, MeetingID: 3, Type: "pdf", Tag: "globalizacja"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + searches := stub.Searches() + if len(searches) != 1 { + t.Fatalf("searches = %d, want 1", len(searches)) + } + q := searches[0].Query + want := map[string]string{ + "q": "globalizacja", + "query_by": "title,tags,source,course,url,body,file_texts", + "query_by_weights": "6,4,3,3,1,2,1", + "filter_by": "course_id:=7 && meeting_id:=3 && type:=`pdf` && tag_slugs:=`globalizacja`", + "sort_by": "_text_match:desc,published_at:desc", + "page": "1", + "per_page": "250", + "exclude_fields": "body,file_texts", + "highlight_fields": "body,file_texts", + } + for k, v := range want { + if q.Get(k) != v { + t.Errorf("%s = %q, want %q", k, q.Get(k), v) + } + } + if !slices.Equal(got.IDs, []uint{12, 5}) { + t.Errorf("ids = %v, want [12 5]", got.IDs) + } + if !got.Highlights { + t.Error("Highlights is false against the typesense engine") + } + if h := got.Hits[12]; len(h.Highlights) != 1 || h.Highlights[0].Field != "body" { + t.Errorf("hit 12 = %+v", h) + } else if segs := h.Highlights[0].Segments; len(segs) != 3 || segs[1].Text != "globalizacji" || !segs[1].Match { + t.Errorf("hit 12 segments = %+v", segs) + } + if _, ok := got.Hits[5]; !ok { + t.Error("hit 5 is missing") + } + if len(got.Hits) != 2 { + t.Errorf("hits = %v, want only 12 and 5", got.Hits) + } + + // A tag that is not a plain slug and an unknown type stay out of the + // engine filter; SQL applies them. + if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "x`y", Type: "exe"}); err != nil { + t.Fatal(err) + } + searches = stub.Searches() + if f := searches[len(searches)-1].Query.Get("filter_by"); f != "" { + t.Errorf("filter_by = %q, want empty", f) + } + if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "zła-Wielka"}); err != nil { + t.Fatal(err) + } + searches = stub.Searches() + if f := searches[len(searches)-1].Query.Get("filter_by"); strings.Contains(f, "tag_slugs") { + t.Errorf("filter_by = %q carries a tag that is not a slug", f) + } +} + +// materialSearchUnavailable runs under TestMaterialSearchRequest. +func materialSearchUnavailable(t *testing.T) { + // No key: the engine is not configured and nothing is sent. + stub := &searchtest.Stub{} + env := searchtest.Start(t, stub) + noKey := slices.DeleteFunc(slices.Clone(env), func(s string) bool { return strings.Contains(s, "API_KEY") }) + ix := classes.NewMaterialIndexer(searchtest.App(t, noKey...)) + if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) { + t.Errorf("unconfigured engine: err = %v, want ErrSearchUnavailable", err) + } + if n := len(stub.Requests()); n != 0 { + t.Errorf("unconfigured engine sent %d requests", n) + } + + failing := &searchtest.Stub{Status: http.StatusInternalServerError} + ix = classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, failing)...)) + if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) { + t.Errorf("failing engine: err = %v, want ErrSearchUnavailable", err) + } + + var none *classes.MaterialIndexer + if _, err := none.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) { + t.Errorf("nil indexer: err = %v, want ErrSearchUnavailable", err) + } + none.Sync(context.Background(), nil, 1) + none.RemoveAfterCommit(context.Background(), nil, 1) +} diff --git a/controllers/api/api.go b/controllers/api/api.go index 48351e6..66ebf3d 100644 --- a/controllers/api/api.go +++ b/controllers/api/api.go @@ -17,6 +17,7 @@ import ( "git.golem15.com/golem15/summercms/modules/cooler" "git.golem15.com/golem15/summercms/modules/phrasebook" "git.golem15.com/golem15/summercms/modules/wire" + "git.golem15.com/jakub/sm-bm-plugin/classes" "gocloud.dev/blob" "gorm.io/gorm" ) @@ -73,6 +74,10 @@ type Handlers struct { // Cache returns the application cache that holds the issued upload // tickets until they are used, or nil while none is available. Cache func() cooler.Cache + // MaterialIndex searches the materials index and keeps it current. Nil + // means a search (q) answers 503 and the syncs after a write are + // skipped. + MaterialIndex *classes.MaterialIndexer } func (h *Handlers) now() time.Time { diff --git a/controllers/api/export_test.go b/controllers/api/export_test.go index a71314e..b50a62e 100644 --- a/controllers/api/export_test.go +++ b/controllers/api/export_test.go @@ -2,3 +2,11 @@ package api // DecodeInline exposes decodeInline to the api_test package. var DecodeInline = decodeInline + +// PickSnippet, FlattenSnippet and RankOrder expose the search helpers to the +// api_test package. +var ( + PickSnippet = pickSnippet + FlattenSnippet = flattenSnippet + RankOrder = rankOrder +) diff --git a/controllers/api/handlers_test.go b/controllers/api/handlers_test.go index fa1f442..25eec08 100644 --- a/controllers/api/handlers_test.go +++ b/controllers/api/handlers_test.go @@ -152,6 +152,7 @@ func TestQueryValidation(t *testing.T) { {"materials course negative", h.Materials, "/?course=-1", "course"}, {"materials meeting zero", h.Materials, "/?meeting=0", "meeting"}, {"materials tag too long", h.Materials, "/?tag=" + longTag(), "tag"}, + {"materials q too long", h.Materials, "/?q=" + longTag(), "q"}, } for _, tc := range cases { code, body := serve(t, tc.fn, http.MethodGet, tc.target, user, nil) diff --git a/controllers/api/json.go b/controllers/api/json.go index 4131bb4..84e41d4 100644 --- a/controllers/api/json.go +++ b/controllers/api/json.go @@ -213,6 +213,10 @@ type materialListJSON struct { Tags []tagJSON `json:"tags"` FilesCount int64 `json:"files_count"` IsPrivate bool `json:"is_private"` + // Snippet is the matched passage of a search result (*snippetJSON, or + // a plain string through MCP). It is nil, and the key absent, without + // q or without a highlight. + Snippet any `json:"snippet,omitempty"` } func newMaterialListJSON(m models.Material, files int64) materialListJSON { diff --git a/controllers/api/material_search.go b/controllers/api/material_search.go new file mode 100644 index 0000000..7e7d28c --- /dev/null +++ b/controllers/api/material_search.go @@ -0,0 +1,123 @@ +package api + +import ( + "net/http" + "strconv" + "strings" + "unicode/utf8" + + "git.golem15.com/golem15/summercms/modules/beachcomber" + "git.golem15.com/golem15/summercms/modules/wire" + "git.golem15.com/jakub/sm-bm-plugin/models" +) + +// Material search: the engine (beachcomber, Typesense) answers candidate ids +// only. Visibility, the filters, the total and the paging stay in SQL, and a +// row's snippet is read only once SQL returned that row, so a material the +// caller may not see never reaches the JSON, its count or a snippet. + +// minSearchRunes is the shortest query that searches; a shorter one is +// ignored, as the student app does. +const minSearchRunes = 2 + +// searchQuery returns the trimmed q of in, or "" when it is shorter than +// minSearchRunes. +func searchQuery(in map[string]any) string { + q := strings.TrimSpace(str(in, "q")) + if utf8.RuneCountInString(q) < minSearchRunes { + return "" + } + return q +} + +// writeSearchUnavailable answers a search the engine cannot serve: it is not +// configured or the call failed. The list without q keeps working. +func writeSearchUnavailable(w http.ResponseWriter) { + wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]any{"error": true, "message": "Search is unavailable."}) +} + +// rankOrder orders rows by their position in ids, the engine's rank. The ids +// are parsed integers, so the array literal carries no caller text. +func rankOrder(ids []uint) string { + parts := make([]string, len(ids)) + for i, id := range ids { + parts[i] = strconv.FormatUint(uint64(id), 10) + } + return "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" +} + +// segmentJSON is one run of a snippet: the engine's plain text, verbatim, +// and whether it matched the query. It is never markup; the client renders +// it as text. +type segmentJSON struct { + Text string `json:"text"` + Hit bool `json:"hit"` +} + +// snippetJSON is the one excerpt a search result shows. Source is "body" or +// "file" (FileName set); Lead and Trail say whether text was cut before or +// after the excerpt. +type snippetJSON struct { + Source string `json:"source"` + FileName *string `json:"file_name"` + Lead bool `json:"lead"` + Trail bool `json:"trail"` + Segments []segmentJSON `json:"segments"` +} + +// pickSnippet returns the snippet of m from hit: the first body snippet, +// mapped one to one. It is nil when the hit has none. Call it only for a +// row the SQL re-gate returned. +func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON { + for _, sn := range hit.Highlights { + if sn.Field != "body" || len(sn.Segments) == 0 { + continue + } + out := &snippetJSON{Source: "body", Segments: make([]segmentJSON, 0, len(sn.Segments))} + var joined strings.Builder + for _, seg := range sn.Segments { + out.Segments = append(out.Segments, segmentJSON{Text: seg.Text, Hit: seg.Match}) + joined.WriteString(seg.Text) + } + body := "" + if m.Body != nil { + body = models.MaterialPlainBody(*m.Body) + } + text := strings.TrimSpace(joined.String()) + out.Lead = !strings.HasPrefix(body, text) + out.Trail = !strings.HasSuffix(body, text) + return out + } + return nil +} + +// flattenSnippet is a snippet as plain text, for MCP: the segment texts +// joined, with an ellipsis where text was cut. +func flattenSnippet(s *snippetJSON) string { + if s == nil { + return "" + } + var b strings.Builder + if s.Lead { + b.WriteString("…") + } + for _, seg := range s.Segments { + b.WriteString(seg.Text) + } + if s.Trail { + b.WriteString("…") + } + return b.String() +} + +// snippetValue is the snippet key of a row: the struct, a plain string +// through MCP, or nil when there is none (the key is then left out). +func snippetValue(s *snippetJSON, mcp bool) any { + if s == nil { + return nil + } + if mcp { + return flattenSnippet(s) + } + return s +} diff --git a/controllers/api/material_search_test.go b/controllers/api/material_search_test.go new file mode 100644 index 0000000..2681351 --- /dev/null +++ b/controllers/api/material_search_test.go @@ -0,0 +1,200 @@ +package api_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "git.golem15.com/golem15/summercms/modules/beachcomber" + "git.golem15.com/golem15/summercms/modules/bouncer" + "git.golem15.com/jakub/sm-bm-plugin/classes" + "git.golem15.com/jakub/sm-bm-plugin/controllers/api" + "git.golem15.com/jakub/sm-bm-plugin/internal/searchtest" + "git.golem15.com/jakub/sm-bm-plugin/models" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +func segments(parts ...any) []beachcomber.Segment { + out := make([]beachcomber.Segment, 0, len(parts)/2) + for i := 0; i+1 < len(parts); i += 2 { + out = append(out, beachcomber.Segment{Text: parts[i].(string), Match: parts[i+1].(bool)}) + } + return out +} + +func TestMaterialSnippet(t *testing.T) { + body := "# Wstęp\n\nNa początku. Proces **globalizacji** zmienia państwa. Koniec tekstu." + m := models.Material{ID: 3, Body: &body} + hit := beachcomber.Hit{ID: "3", Highlights: []beachcomber.Snippet{ + {Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)}, + {Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia państwa.", false)}, + }} + s := api.PickSnippet(hit, m) + if s == nil { + t.Fatal("no snippet") + } + if s.Source != "body" || s.FileName != nil { + t.Errorf("source %q file %v, want body and no file", s.Source, s.FileName) + } + if len(s.Segments) != 3 || s.Segments[1].Text != "globalizacji" || !s.Segments[1].Hit || s.Segments[0].Hit { + t.Errorf("segments = %+v", s.Segments) + } + if s.Segments[2].Text != " zmienia państwa." { + t.Errorf("segment text was changed: %q", s.Segments[2].Text) + } + if !s.Lead || !s.Trail { + t.Errorf("lead %v trail %v, want both (cut inside the body)", s.Lead, s.Trail) + } + b, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + if got := string(b); !strings.Contains(got, `"source":"body","file_name":null,"lead":true,"trail":true,"segments":[{"text":"Proces ","hit":false}`) { + t.Errorf("JSON = %s", got) + } + + // The whole body: neither lead nor trail. + short := "Krótka **notatka**" + whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short}) + if whole == nil || whole.Lead || whole.Trail { + t.Errorf("whole body snippet = %+v, want no lead or trail", whole) + } + // Start of the body only: trail but no lead. + start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m) + if start == nil || start.Lead || !start.Trail { + t.Errorf("start snippet = %+v, want trail only", start) + } + + // A snippet on another field only, or none at all, gives no snippet. + if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil { + t.Errorf("title-only hit gave %+v", got) + } + if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil { + t.Errorf("hit without highlights gave %+v", got) + } + + if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia państwa.…" { + t.Errorf("FlattenSnippet = %q", got) + } + if got := api.FlattenSnippet(nil); got != "" { + t.Errorf("FlattenSnippet(nil) = %q", got) + } + if got := api.RankOrder([]uint{9, 3}); got != "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" { + t.Errorf("RankOrder = %q", got) + } +} + +func TestMaterialsSearchUnavailable(t *testing.T) { + h := noDB() + user := &bouncer.Principal{ID: 7} + code, body := serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil) + if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." { + t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body) + } + // One rune is no search: the list answers as without q (no database). + code, body = serve(t, h.Materials, http.MethodGet, "/?q=%20g%20", user, nil) + if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." { + t.Errorf("one-rune q = %d %v, want the no-database answer", code, body) + } + code, body = serve(t, h.Materials, http.MethodGet, "/", user, nil) + if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." { + t.Errorf("no q = %d %v, want the no-database answer", code, body) + } + // An unconfigured engine is the same 503. + h.MaterialIndex = classes.NewMaterialIndexer(searchtest.App(t)) + code, body = serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil) + if code != http.StatusServiceUnavailable || body["message"] != "Search is unavailable." { + t.Errorf("q with a null engine = %d %v, want 503 Search is unavailable.", code, body) + } +} + +// sqlRecorder is a dry-run postgres handle that records every query it +// builds, with its arguments inlined. +type sqlRecorder struct { + mu sync.Mutex + sql []string +} + +func (r *sqlRecorder) queries() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.sql...) +} + +func dryDB(t *testing.T) (*gorm.DB, *sqlRecorder) { + t.Helper() + db, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true}) + if err != nil { + t.Fatal(err) + } + rec := &sqlRecorder{} + err = db.Callback().Query().After("gorm:query").Register("test:record_sql", func(tx *gorm.DB) { + rec.mu.Lock() + defer rec.mu.Unlock() + rec.sql = append(rec.sql, tx.Dialector.Explain(tx.Statement.SQL.String(), tx.Statement.Vars...)) + }) + if err != nil { + t.Fatal(err) + } + return db, rec +} + +func TestMaterialsSearchTrace(t *testing.T) { + stub := &searchtest.Stub{Hits: []searchtest.Hit{ + {ID: "9"}, + {ID: "3", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd}, + }} + db, rec := dryDB(t) + h := &api.Handlers{ + DB: func() *gorm.DB { return db }, + MaterialIndex: classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)), + } + req := httptest.NewRequest(http.MethodGet, "/materials?q=globalizacja&course=7", nil) + req = req.WithContext(bouncer.WithUser(req.Context(), &bouncer.Principal{ID: 5})) + w := httptest.NewRecorder() + h.Materials(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + if n := len(stub.Searches()); n != 1 { + t.Errorf("searches = %d, want 1", n) + } + var count, page string + for _, q := range rec.queries() { + switch { + case strings.HasPrefix(q, "SELECT count(*)"): + count = q + case strings.HasPrefix(q, "SELECT") && strings.Contains(q, `FROM "jz_bm_materials"`): + page = q + } + } + if count == "" || page == "" { + t.Fatalf("queries = %q, want a count and a page", rec.queries()) + } + for name, q := range map[string]string{"count": count, "page": page} { + for _, want := range []string{"jz_bm_materials.is_private = FALSE", "jz_bm_materials.published_at IS NOT NULL", "jz_bm_materials.course_id = 7", "jz_bm_materials.id IN (9,3)"} { + if !strings.Contains(q, want) { + t.Errorf("%s query lacks %q:\n%s", name, want, q) + } + } + } + if !strings.Contains(page, "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id)") { + t.Errorf("page query is not in rank order:\n%s", page) + } + var out struct { + Data []map[string]any `json:"data"` + Meta struct { + Total int `json:"total"` + } `json:"meta"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if out.Meta.Total != 0 || len(out.Data) != 0 { + t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data) + } +} diff --git a/controllers/api/materials.go b/controllers/api/materials.go index 81c62bd..8311304 100644 --- a/controllers/api/materials.go +++ b/controllers/api/materials.go @@ -6,9 +6,11 @@ import ( "strings" "time" + "git.golem15.com/golem15/summercms/modules/beachcomber" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/lagoon/attach" "git.golem15.com/golem15/summercms/modules/surf" + "git.golem15.com/jakub/sm-bm-plugin/classes" "git.golem15.com/jakub/sm-bm-plugin/models" "gorm.io/gorm" ) @@ -49,23 +51,35 @@ func preloadMaterial(q *gorm.DB) *gorm.DB { } // listMaterials returns one page of the published materials userID may -// see, newest first. -func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, page, perPage int) ([]materialListJSON, int64, error) { +// see. With ids nil the page is newest first. With ids non-nil (search +// candidates in the engine's rank order) both the count and the page are +// also limited to ids and the page follows their order; the ids never widen +// what visibleMaterials and the filter allow. An empty ids is an empty page. +func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, ids []uint, page, perPage int) ([]materialListJSON, []models.Material, int64, error) { + if ids != nil && len(ids) == 0 { + return []materialListJSON{}, nil, 0, nil + } q := f.apply(visibleMaterials(db.Model(&models.Material{}), now, userID)) + if ids != nil { + q = q.Where("jz_bm_materials.id IN ?", ids) + } var total int64 if err := q.Count(&total).Error; err != nil { - return nil, 0, err + return nil, nil, 0, err + } + rowsQ := preloadMaterial(f.apply(visibleMaterials(db, now, userID))) + if ids != nil { + rowsQ = rowsQ.Where("jz_bm_materials.id IN ?", ids).Order(rankOrder(ids)) + } else { + rowsQ = rowsQ.Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC") } var rows []models.Material - err := preloadMaterial(f.apply(visibleMaterials(db, now, userID))). - Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC"). - Offset((page - 1) * perPage).Limit(perPage). - Find(&rows).Error + err := rowsQ.Offset((page - 1) * perPage).Limit(perPage).Find(&rows).Error if err != nil { - return nil, 0, err + return nil, nil, 0, err } out, err := materialsJSON(db, rows) - return out, total, err + return out, rows, total, err } // materialsJSON builds list rows with their file counts. @@ -99,31 +113,69 @@ func materialsJSON(db *gorm.DB, rows []models.Material) ([]materialListJSON, err return out, nil } -// Materials answers GET /materials?course=&meeting=&tag=&type=&page=&per_page=: +// Materials answers GET /materials?q=&course=&meeting=&tag=&type=&page=&per_page=: // a page of the published materials the caller may see, in the {data, meta} -// shape of lagoon.Paginate. +// shape of lagoon.Paginate. With q (2 to 100 characters) the engine answers +// candidates, which SQL re-gates, counts and pages in rank order, each row +// with at most one snippet; when the engine cannot answer the request is a +// 503. Without q the list is newest first and never calls the engine. func (h *Handlers) Materials(w http.ResponseWriter, r *http.Request) { userID, ok := h.caller(w, r) if !ok { return } - in := query(r, "course", "meeting", "tag", "type", "page", "per_page") + in := query(r, "q", "course", "meeting", "tag", "type", "page", "per_page") if !h.validate(w, r, in, materialsRules) { return } + filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")} + var cands *classes.MaterialCandidates + if q := searchQuery(in); q != "" { + c, err := h.MaterialIndex.Search(r.Context(), classes.MaterialSearch{Q: q, CourseID: filter.CourseID, MeetingID: filter.MeetingID, Type: filter.Type, Tag: filter.Tag}) + if err != nil { + writeSearchUnavailable(w) + return + } + cands = &c + } db, ok := h.db(w, r) if !ok { return } - filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")} page := min(intParam(in, "page", 1), 100000) perPage := intParam(in, "per_page", h.perPage()) - rows, total, err := listMaterials(db, h.now(), userID, filter, page, perPage) + var ids []uint + if cands != nil { + ids = cands.IDs + if ids == nil { + ids = []uint{} + } + } + out, rows, total, err := listMaterials(db, h.now(), userID, filter, ids, page, perPage) if err != nil { writeOpaque500(w) return } - writeJSONPage(w, lagoon.Paginate(rows, page, perPage, total)) + if cands != nil { + attachSnippets(out, rows, cands.Hits, viaMCP(r)) + } + writeJSONPage(w, lagoon.Paginate(out, page, perPage, total)) +} + +// attachSnippets sets the snippet of each list row from its Hit. rows are +// the rows SQL returned (they passed the re-gate), in the order of out; the +// Hit of an id SQL did not return is never read. +func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, mcp bool) { + for i := range out { + if i >= len(rows) || rows[i].ID != out[i].ID { + continue + } + hit, ok := hits[rows[i].ID] + if !ok { + continue + } + out[i].Snippet = snippetValue(pickSnippet(hit, rows[i]), mcp) + } } // Material answers GET /materials/{id}: one published material the caller diff --git a/controllers/api/params.go b/controllers/api/params.go index 34c95a2..515d259 100644 --- a/controllers/api/params.go +++ b/controllers/api/params.go @@ -37,6 +37,7 @@ var ( {Field: "type", Rules: append(lagoon.ParseRules("nullable"), lagoon.In(models.MaterialTypes...))}, {Field: "page", Rules: lagoon.ParseRules("nullable|integer|min:1")}, {Field: "per_page", Rules: lagoon.ParseRules("nullable|integer|between:1,100")}, + {Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")}, } ) diff --git a/go.mod b/go.mod index dd5259d..ef91cfd 100644 --- a/go.mod +++ b/go.mod @@ -6,19 +6,19 @@ require ( git.golem15.com/golem15/sm-chat-plugin v0.1.3 git.golem15.com/golem15/sm-chatchannels-plugin v0.0.0 git.golem15.com/golem15/sm-user-plugin v0.1.2 - git.golem15.com/golem15/summercms v0.5.0 + git.golem15.com/golem15/summercms v0.6.4 github.com/go-gormigrate/gormigrate/v2 v2.1.7 github.com/jackc/pgx/v5 v5.10.0 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 gocloud.dev v0.46.0 + gorm.io/driver/postgres v1.6.3 gorm.io/gorm v1.31.2 ) require ( charm.land/bubbles/v2 v2.2.1 // indirect charm.land/bubbletea/v2 v2.1.0 // indirect - charm.land/fang/v2 v2.0.1 // indirect charm.land/huh/v2 v2.0.3 // indirect charm.land/lipgloss/v2 v2.0.6 // indirect dario.cat/mergo v1.0.2 // indirect @@ -52,7 +52,6 @@ require ( github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20261008173134-6b8d4baf91b4 // indirect github.com/charmbracelet/x/ansi v0.11.9 // indirect - github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 // indirect github.com/charmbracelet/x/exp/ordered v0.1.0 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect @@ -123,10 +122,6 @@ require ( github.com/moby/term v0.5.2 // indirect github.com/modelcontextprotocol/go-sdk v1.8.0 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/muesli/mango v0.1.0 // indirect - github.com/muesli/mango-cobra v1.2.0 // indirect - github.com/muesli/mango-pflag v0.1.0 // indirect - github.com/muesli/roff v0.1.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.6.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect @@ -180,7 +175,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.11 // indirect - gorm.io/driver/postgres v1.6.3 // indirect ) replace git.golem15.com/golem15/summercms => ../../../../summercms.go diff --git a/go.sum b/go.sum index abcd5a0..d61206e 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ charm.land/bubbles/v2 v2.2.1 h1:Fq1+qm5hV6GkvzLQDhCBpXXE5tLgvh1PRriCLwSvIQU= charm.land/bubbles/v2 v2.2.1/go.mod h1:wdMgn+sje1KNXdwFizIWjbf328fIUBxqEmJ/vYPo8yc= charm.land/bubbletea/v2 v2.1.0 h1:A9fj9h5LFrvQhQHFNggp8K8rvY6CUfgn9dMbb6p2HIw= charm.land/bubbletea/v2 v2.1.0/go.mod h1:xWHeyxIX9/wiFU2kgPW04ojOZyTzrWf+8lGjUCaftiU= -charm.land/fang/v2 v2.0.1 h1:zQCM8JQJ1JnQX/66B5jlCYBUxL2as5JXQZ2KJ6EL0mY= -charm.land/fang/v2 v2.0.1/go.mod h1:S1GmkpcvK+OB5w9caywUnJcsMew45Ot8FXqoz8ALrII= charm.land/huh/v2 v2.0.3 h1:2cJsMqEPwSywGHvdlKsJyQKPtSJLVnFKyFbsYZTlLkU= charm.land/huh/v2 v2.0.3/go.mod h1:93eEveeeqn47MwiC3tf+2atZ2l7Is88rAtmZNZ8x9Wc= charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= @@ -110,8 +108,6 @@ github.com/charmbracelet/x/conpty v0.1.1 h1:s1bUxjoi7EpqiXysVtC+a8RrvPPNcNvAjfi4 github.com/charmbracelet/x/conpty v0.1.1/go.mod h1:OmtR77VODEFbiTzGE9G1XiRJAga6011PIm4u5fTNZpk= github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= -github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 h1:IJDiTgVE56gkAGfq0lBEloWgkXMk4hl/bmuPoicI4R0= -github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444/go.mod h1:T9jr8CzFpjhFVHjNjKwbAD7KwBNyFnj2pntAO7F2zw0= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I= github.com/charmbracelet/x/exp/ordered v0.1.0 h1:55/qLwjIh0gL0Vni+QAWk7T/qRVP6sBf+2agPBgnOFE= @@ -299,14 +295,6 @@ github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtw github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= -github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI= -github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4= -github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg= -github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA= -github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg= -github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= -github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= -github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ= github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/internal/searchtest/typesense.go b/internal/searchtest/typesense.go new file mode 100644 index 0000000..f559a26 --- /dev/null +++ b/internal/searchtest/typesense.go @@ -0,0 +1,198 @@ +// Package searchtest is a Typesense stand-in for the jz.bm tests: an +// httptest server that answers the collection, import, delete and search +// endpoints the beachcomber typesense engine calls, and an app configured to +// talk to it. Only tests import it. +package searchtest + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "git.golem15.com/golem15/summercms/modules/backpack" + _ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense" + "git.golem15.com/golem15/summercms/modules/compass" +) + +// Index is the collection of the materials index. +const Index = "jz_bm_materials" + +// HighlightStart and HighlightEnd are the highlight tags the typesense +// engine asks for (two private-use characters). +const ( + HighlightStart = "" + HighlightEnd = "" +) + +// Hit is one search answer: a document id and, optionally, the snippet of +// one field with the matched words between HighlightStart and HighlightEnd. +type Hit struct { + ID string + Field string + Snippet string +} + +// Request is one request the stub received. +type Request struct { + Method string + Path string + Query url.Values +} + +// Stub stands in for a Typesense server with one collection. +type Stub struct { + mu sync.Mutex + // Hits answer every search, in rank order. + Hits []Hit + // Status, when set, answers every request with that code. + Status int + + exists bool + requests []Request + imported []map[string]any + removed []string +} + +// ServeHTTP implements http.Handler. +func (s *Stub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query()}) + if s.Status != 0 { + w.WriteHeader(s.Status) + _, _ = io.WriteString(w, `{"message":"stub failure"}`) + return + } + coll := "/collections/" + Index + switch { + case r.Method == http.MethodGet && r.URL.Path == coll: + if !s.exists { + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"Not Found"}`) + return + } + _, _ = io.WriteString(w, `{}`) + case r.Method == http.MethodPost && r.URL.Path == "/collections": + s.exists = true + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{}`) + case r.Method == http.MethodDelete && r.URL.Path == coll: + if !s.exists { + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"Not Found"}`) + return + } + s.exists = false + _, _ = io.WriteString(w, `{}`) + case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import": + s.exists = true + sc := bufio.NewScanner(r.Body) + sc.Buffer(make([]byte, 0, 64*1024), 16<<20) + var answer []string + for sc.Scan() { + var doc map[string]any + if err := json.Unmarshal(sc.Bytes(), &doc); err == nil { + s.imported = append(s.imported, doc) + } + answer = append(answer, `{"success":true}`) + } + _, _ = io.WriteString(w, strings.Join(answer, "\n")) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"): + s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/")) + _, _ = io.WriteString(w, `{}`) + case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search": + hits := make([]string, 0, len(s.Hits)) + for _, h := range s.Hits { + id, _ := json.Marshal(h.ID) + highlights := "[]" + if h.Field != "" { + field, _ := json.Marshal(h.Field) + snippet, _ := json.Marshal(h.Snippet) + highlights = fmt.Sprintf(`[{"field":%s,"snippet":%s}]`, field, snippet) + } + hits = append(hits, fmt.Sprintf(`{"document":{"id":%s},"highlights":%s}`, id, highlights)) + } + // found is deliberately larger than the hits, so a caller that + // exposed it would show. + _, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.Hits)+100, strings.Join(hits, ",")) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"Not Found"}`) + } +} + +// Requests returns the requests received so far. +func (s *Stub) Requests() []Request { + s.mu.Lock() + defer s.mu.Unlock() + return append([]Request(nil), s.requests...) +} + +// Searches returns the search requests received so far. +func (s *Stub) Searches() []Request { + var out []Request + for _, r := range s.Requests() { + if r.Method == http.MethodGet && strings.HasSuffix(r.Path, "/documents/search") { + out = append(out, r) + } + } + return out +} + +// Imported returns the documents upserted so far. +func (s *Stub) Imported() []map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + return append([]map[string]any(nil), s.imported...) +} + +// Removed returns the document ids deleted so far. +func (s *Stub) Removed() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.removed...) +} + +// Start serves stub on an httptest server closed with the test, and returns +// the SUMMER_SEARCH__* environment that points the typesense engine at it +// with a test-only key. +func Start(t testing.TB, stub *Stub) []string { + t.Helper() + srv := httptest.NewServer(stub) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + return []string{ + "SUMMER_SEARCH__DRIVER=typesense", + "SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key", + "SUMMER_SEARCH__TYPESENSE__HOST=" + u.Hostname(), + "SUMMER_SEARCH__TYPESENSE__PORT=" + u.Port(), + "SUMMER_SEARCH__TYPESENSE__PROTOCOL=" + u.Scheme, + } +} + +// App returns a bare application whose config holds env (and SUMMER_ENV +// testing) and an empty config directory. +func App(t testing.TB, env ...string) *backpack.App { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: jz-bm-test\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: append([]string{"SUMMER_ENV=testing"}, env...)}) + if err != nil { + t.Fatal(err) + } + return backpack.New(cfg) +} diff --git a/models/material_search.go b/models/material_search.go new file mode 100644 index 0000000..5c145b1 --- /dev/null +++ b/models/material_search.go @@ -0,0 +1,173 @@ +package models + +import ( + "context" + "regexp" + "strconv" + "strings" + + "gorm.io/gorm" +) + +// A material is kept in the search index jz_bm_materials (beachcomber). The +// index holds every material, drafts and private ones included: visibility +// is enforced only in SQL, where the engine's ids are candidates that the +// student queries re-gate. This file is hand-written; material.go is +// generated. + +// SearchableAs is the index name without the search.prefix. +func (Material) SearchableAs() string { return "jz_bm_materials" } + +// ShouldBeSearchable is true for every material: the index carries no +// visibility, so drafts and private materials are indexed too. +func (m *Material) ShouldBeSearchable() bool { return m != nil } + +// ToSearchableArray builds the document of the material. beachcomber reloads +// the row by primary key without preloads, so the course, the meeting number +// and the tags are loaded here. +func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) { + db = db.WithContext(ctx) + var course *Course + if m.CourseID != nil { + var c Course + res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c) + if res.Error != nil { + return nil, res.Error + } + if res.RowsAffected > 0 { + course = &c + } + } + meetingNumber := 0 + if m.MeetingID != nil { + var mt Meeting + res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt) + if res.Error != nil { + return nil, res.Error + } + if res.RowsAffected > 0 { + meetingNumber = mt.Number + } + } + var tags []Tag + err := db.Model(&Tag{}). + Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id"). + Where("mt.material_id = ?", m.ID). + Order("jz_bm_tags.name"). + Find(&tags).Error + if err != nil { + return nil, err + } + return MaterialDocument(*m, course, meetingNumber, tags, nil), nil +} + +// MaterialDocument is the search document of m with its course, meeting +// number (0 when none), tags in order and the texts of its attached files. +// Absent numbers are 0, so filters and the sort tiebreak never see a +// missing value. +func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any { + doc := map[string]any{ + "id": strconv.FormatUint(uint64(m.ID), 10), + "title": m.Title, + "type": m.Type, + "course_id": int64(0), + "meeting_id": int64(0), + "published_at": int64(0), + } + if m.Source != nil && strings.TrimSpace(*m.Source) != "" { + doc["source"] = *m.Source + } + if m.URL != nil && strings.TrimSpace(*m.URL) != "" { + doc["url"] = *m.URL + } + if m.CourseID != nil { + doc["course_id"] = int64(*m.CourseID) + } + if m.MeetingID != nil { + doc["meeting_id"] = int64(*m.MeetingID) + } + if m.PublishedAt != nil { + doc["published_at"] = m.PublishedAt.Unix() + } + if course != nil { + label := course.Name + if course.Code != nil && strings.TrimSpace(*course.Code) != "" { + label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code)) + } + doc["course"] = label + } + if meetingNumber > 0 { + doc["meeting_number"] = meetingNumber + } + names := make([]string, 0, len(tags)) + slugs := make([]string, 0, len(tags)) + for _, t := range tags { + names = append(names, t.Name) + slugs = append(slugs, t.Slug) + } + doc["tags"] = names + doc["tag_slugs"] = slugs + if m.Body != nil { + if body := MaterialPlainBody(*m.Body); body != "" { + doc["body"] = body + } + } + if len(fileTexts) > 0 { + doc["file_texts"] = fileTexts + } + return doc +} + +// SearchIndexSchema is the Typesense collection schema. It applies only when +// the collection is created; bm:materials-reindex --fresh recreates it. +// Every field is stored (a search leaves body and file_texts out of the +// answer instead), and no field sets a locale: the default one folds +// diacritics, so lodz finds Łódź. +func (Material) SearchIndexSchema() map[string]any { + return map[string]any{ + "fields": []map[string]any{ + {"name": "id", "type": "string"}, + {"name": "title", "type": "string"}, + {"name": "source", "type": "string", "optional": true}, + {"name": "url", "type": "string", "optional": true}, + {"name": "tags", "type": "string[]", "optional": true}, + {"name": "tag_slugs", "type": "string[]", "optional": true}, + {"name": "course", "type": "string", "optional": true}, + {"name": "course_id", "type": "int64"}, + {"name": "meeting_id", "type": "int64"}, + {"name": "meeting_number", "type": "int32", "optional": true}, + {"name": "type", "type": "string"}, + {"name": "published_at", "type": "int64"}, + {"name": "body", "type": "string", "optional": true}, + {"name": "file_texts", "type": "string[]", "optional": true}, + }, + } +} + +var ( + mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`) + mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`) + mdHeading = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`) + mdQuote = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`) + mdList = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`) + mdEmphasis = regexp.MustCompile(`\*+|~~`) + mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`) + mdUnderEnd = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`) +) + +// MaterialPlainBody drops the Markdown syntax of a material body (heading +// hashes, blockquote and list markers, emphasis, inline code ticks, link +// and image syntax down to their text) and collapses white space, so the +// index holds and highlights only words. +func MaterialPlainBody(body string) string { + s := strings.ReplaceAll(body, "`", "") + s = mdImage.ReplaceAllString(s, "$1") + s = mdLink.ReplaceAllString(s, "$1") + s = mdHeading.ReplaceAllString(s, "") + s = mdQuote.ReplaceAllString(s, "") + s = mdList.ReplaceAllString(s, "") + s = mdEmphasis.ReplaceAllString(s, "") + s = mdUnderStart.ReplaceAllString(s, "$1") + s = mdUnderEnd.ReplaceAllString(s, "$1") + return strings.Join(strings.Fields(s), " ") +} diff --git a/models/material_search_test.go b/models/material_search_test.go new file mode 100644 index 0000000..9b32ae6 --- /dev/null +++ b/models/material_search_test.go @@ -0,0 +1,124 @@ +package models_test + +import ( + "slices" + "testing" + "time" + + "git.golem15.com/jakub/sm-bm-plugin/models" +) + +func TestMaterialSearchSchema(t *testing.T) { + m := models.Material{} + if got := m.SearchableAs(); got != "jz_bm_materials" { + t.Errorf("SearchableAs = %q, want jz_bm_materials", got) + } + fields, ok := m.SearchIndexSchema()["fields"].([]map[string]any) + if !ok || len(fields) == 0 { + t.Fatalf("schema fields = %#v", m.SearchIndexSchema()["fields"]) + } + byName := map[string]map[string]any{} + for _, f := range fields { + name, _ := f["name"].(string) + typ, _ := f["type"].(string) + if name == "" || typ == "" { + t.Errorf("field without a name or type: %v", f) + } + if _, ok := f["locale"]; ok { + t.Errorf("field %s sets a locale; the default one folds Polish diacritics", name) + } + if store, ok := f["store"]; ok && store == false { + t.Errorf("field %s turns storage off", name) + } + byName[name] = f + } + for _, name := range []string{"body", "file_texts"} { + f, ok := byName[name] + if !ok { + t.Errorf("schema has no %s field", name) + continue + } + if f["optional"] != true { + t.Errorf("%s is not optional", name) + } + } + if byName["body"]["type"] != "string" || byName["file_texts"]["type"] != "string[]" { + t.Errorf("body %v, file_texts %v: want string and string[]", byName["body"]["type"], byName["file_texts"]["type"]) + } + for _, name := range []string{"id", "title", "source", "url", "tags", "tag_slugs", "course", "course_id", "meeting_id", "meeting_number", "type", "published_at"} { + if _, ok := byName[name]; !ok { + t.Errorf("schema has no %s field", name) + } + } + if !(&m).ShouldBeSearchable() { + t.Error("a material is not searchable") + } + var nilMaterial *models.Material + if nilMaterial.ShouldBeSearchable() { + t.Error("a nil material is searchable") + } +} + +func TestMaterialDocument(t *testing.T) { + plain := models.MaterialPlainBody("# Nagłówek\n\nTekst **pogrubienie** i _kursywa_ oraz [tekst](https://x) i ![obraz](a.png).\n\n> cytat\n> dalej\n\n- punkt `kod` snake_case") + want := "Nagłówek Tekst pogrubienie i kursywa oraz tekst i obraz. cytat dalej punkt kod snake_case" + if plain != want { + t.Errorf("MaterialPlainBody =\n%q\nwant\n%q", plain, want) + } + + code := "BM-12" + course := &models.Course{ID: 4, Name: "Bezpieczeństwo", Code: &code} + courseID, meetingID := uint(4), uint(9) + at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) + body := "## Wstęp\n\n**Globalizacja** w praktyce" + source := "Giddens" + m := models.Material{ID: 12, Title: "Globalizacja", Type: "notatka", CourseID: &courseID, MeetingID: &meetingID, PublishedAt: &at, Body: &body, Source: &source} + tags := []models.Tag{{Name: "Globalizacja", Slug: "globalizacja"}, {Name: "Polityka", Slug: "polityka"}} + doc := models.MaterialDocument(m, course, 3, tags, []string{"tekst pliku"}) + checks := map[string]any{ + "id": "12", + "title": "Globalizacja", + "type": "notatka", + "source": "Giddens", + "course": "Bezpieczeństwo BM-12", + "course_id": int64(4), + "meeting_id": int64(9), + "meeting_number": 3, + "published_at": at.Unix(), + "body": "Wstęp Globalizacja w praktyce", + } + for k, v := range checks { + if doc[k] != v { + t.Errorf("doc[%s] = %#v, want %#v", k, doc[k], v) + } + } + if got, _ := doc["tags"].([]string); !slices.Equal(got, []string{"Globalizacja", "Polityka"}) { + t.Errorf("tags = %v", doc["tags"]) + } + if got, _ := doc["tag_slugs"].([]string); !slices.Equal(got, []string{"globalizacja", "polityka"}) { + t.Errorf("tag_slugs = %v", doc["tag_slugs"]) + } + if got, _ := doc["file_texts"].([]string); !slices.Equal(got, []string{"tekst pliku"}) { + t.Errorf("file_texts = %v", doc["file_texts"]) + } + + // No code, no course id, no meeting, a draft, no files. + bare := models.MaterialDocument(models.Material{ID: 5, Title: "Szkic", Type: "inne"}, &models.Course{Name: "Teoria"}, 0, nil, nil) + if bare["course"] != "Teoria" { + t.Errorf("course without code = %v", bare["course"]) + } + for _, k := range []string{"course_id", "meeting_id", "published_at"} { + if bare[k] != int64(0) { + t.Errorf("bare %s = %#v, want 0", k, bare[k]) + } + } + for _, k := range []string{"meeting_number", "file_texts", "body", "source", "url"} { + if _, ok := bare[k]; ok { + t.Errorf("bare document carries %s", k) + } + } + none := models.MaterialDocument(models.Material{ID: 6, Title: "X", Type: "link"}, nil, 0, nil, nil) + if _, ok := none["course"]; ok { + t.Error("a material without a course carries course") + } +} diff --git a/plugin.go b/plugin.go index 586fadd..5d22da3 100644 --- a/plugin.go +++ b/plugin.go @@ -59,6 +59,9 @@ type Plugin struct { // chatMember is the chat-group check of the realtime access guard; nil // means classes.IsChatMember. Tests inject their own. chatMember classes.MemberFunc + // materialIndex searches the materials index and keeps it current. Nil + // when Boot did not run: then search answers 503 and syncs are skipped. + materialIndex *classes.MaterialIndexer } func (p *Plugin) ID() string { return "jz.bm" } @@ -74,7 +77,8 @@ func (p *Plugin) Register(app *backpack.App) error { // Boot keeps the application, so route handlers and commands can reach its // services, such as the database, when they run, and wires the realtime -// service and, when golem15.chat is active, the BM chat context. The +// service, the materials search and, when golem15.chat is active, the BM +// chat context. The // management API uses the shared user.api_token middleware owned by // golem15.user. func (p *Plugin) Boot(app *backpack.App) error { @@ -83,6 +87,9 @@ func (p *Plugin) Boot(app *backpack.App) error { if err := p.wireRealtime(app); err != nil { return err } + if err := p.wireSearch(app); err != nil { + return err + } if err := p.registerChat(app); err != nil { return err } diff --git a/routes.go b/routes.go index f7bccd5..acf5200 100644 --- a/routes.go +++ b/routes.go @@ -304,6 +304,7 @@ func (p *Plugin) mountRealtime(r pact.Router) error { // publishes it. func (p *Plugin) handlers() *api.Handlers { h := &api.Handlers{ + MaterialIndex: p.materialIndex, DB: func() *gorm.DB { if p.app == nil { return nil diff --git a/search.go b/search.go new file mode 100644 index 0000000..8685585 --- /dev/null +++ b/search.go @@ -0,0 +1,23 @@ +package bm + +import ( + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/beachcomber" + _ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense" + "git.golem15.com/jakub/sm-bm-plugin/classes" +) + +// wireSearch builds the app's beachcomber service (search.driver; the +// typesense engine registers itself through the blank import), installs the +// jz.bm-only Gate and the material indexer. The Gate is app-wide, so it is +// on only for contexts jz.bm marks: golem15.chatchannels' message syncs stay +// off and chat messages never reach the index. +func (p *Plugin) wireSearch(app *backpack.App) error { + svc, err := beachcomber.From(app) + if err != nil { + return err + } + svc.SetGate(classes.MaterialSearchGate()) + p.materialIndex = classes.NewMaterialIndexer(app) + return nil +}