diff --git a/classes/material_index.go b/classes/material_index.go
new file mode 100644
index 0000000..5b50502
--- /dev/null
+++ b/classes/material_index.go
@@ -0,0 +1,256 @@
+package classes
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "regexp"
+ "slices"
+ "strconv"
+ "strings"
+
+ "git.golem15.com/golem15/summercms/modules/backpack"
+ "git.golem15.com/golem15/summercms/modules/beachcomber"
+ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+ "git.golem15.com/golem15/summercms/modules/diary"
+ "git.golem15.com/golem15/summercms/modules/lagoon"
+ "git.golem15.com/jakub/sm-bm-plugin/models"
+ "gorm.io/gorm"
+)
+
+// materialSearchKey marks a context whose syncs belong to jz.bm.
+type materialSearchKey struct{}
+
+// MarkMaterialSearch returns ctx marked as a jz.bm material sync. The
+// app-wide beachcomber Gate (MaterialSearchGate) is on only for a marked
+// context, so every jz.bm Sync or Remove passes one.
+func MarkMaterialSearch(ctx context.Context) context.Context {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ return context.WithValue(ctx, materialSearchKey{}, true)
+}
+
+// MaterialSearchGate is the app-wide beachcomber Gate jz.bm installs. The
+// Gate sees only (ctx, db), not the model, so it is on only for a context
+// marked by MarkMaterialSearch: golem15.chatchannels syncs its messages
+// with unmarked contexts, and chat messages and DMs never reach the index.
+// The automatic callbacks of material writes are off as well; every jz.bm
+// write path syncs explicitly.
+func MaterialSearchGate() beachcomber.Gate {
+ return beachcomber.GateFunc(func(ctx context.Context, _ *gorm.DB) bool {
+ return ctx != nil && ctx.Value(materialSearchKey{}) != nil
+ })
+}
+
+// MaterialSearchWindow is how many engine candidates one search re-gates in
+// SQL. The list counts and pages over the gated rows, so a page never
+// depends on how the engine paged.
+const MaterialSearchWindow = typesense.MaxPerPage
+
+// MaterialQueryBy are the document fields a query is matched against, and
+// MaterialQueryWeights their weights in the same order: the title over the
+// tags, the source and the course, then the body, the link and file text.
+var (
+ MaterialQueryBy = []string{"title", "tags", "source", "course", "url", "body", "file_texts"}
+ MaterialQueryWeights = []int{6, 4, 3, 3, 1, 2, 1}
+)
+
+// materialHighlightFields are the fields snippets are cut from.
+var materialHighlightFields = []string{"body", "file_texts"}
+
+// ErrSearchUnavailable is returned by Search when the engine is not
+// configured or the search failed. The API answers it with a 503.
+var ErrSearchUnavailable = errors.New("jz.bm: search is unavailable")
+
+// MaterialSearch is one material search: the query text and the validated
+// filters of the list.
+type MaterialSearch struct {
+ Q string
+ CourseID uint
+ MeetingID uint
+ Type string
+ Tag string
+}
+
+// MaterialCandidates are the engine's answer: the candidate ids in rank
+// order and the Hit of each. Both are candidates only: the caller re-gates
+// the ids in SQL and reads the Hit of a row only once SQL returned it.
+// Highlights reports whether the engine was asked for snippets.
+type MaterialCandidates struct {
+ IDs []uint
+ Hits map[uint]beachcomber.Hit
+ Highlights bool
+}
+
+// MaterialIndexer searches the materials index and keeps it current. A nil
+// *MaterialIndexer is valid: Search answers ErrSearchUnavailable and the
+// syncs do nothing.
+type MaterialIndexer struct {
+ // Service returns the app's beachcomber service, or nil.
+ Service func() *beachcomber.Service
+ // Log is where failures are logged; nil means slog.Default.
+ Log *slog.Logger
+}
+
+// NewMaterialIndexer returns the indexer of app. The service is resolved on
+// each call, so it may be built after the indexer.
+func NewMaterialIndexer(app *backpack.App) *MaterialIndexer {
+ return &MaterialIndexer{
+ Service: func() *beachcomber.Service {
+ if app == nil {
+ return nil
+ }
+ svc, err := beachcomber.From(app)
+ if err != nil {
+ return nil
+ }
+ return svc
+ },
+ Log: diary.For("jz.bm"),
+ }
+}
+
+func (ix *MaterialIndexer) service() *beachcomber.Service {
+ if ix == nil || ix.Service == nil {
+ return nil
+ }
+ return ix.Service()
+}
+
+func (ix *MaterialIndexer) log() *slog.Logger {
+ if ix == nil || ix.Log == nil {
+ return slog.Default()
+ }
+ return ix.Log
+}
+
+// Search asks the engine for up to MaterialSearchWindow candidates of req.
+// It answers ErrSearchUnavailable (wrapping the cause) when the service or
+// its engine is missing, the engine is not configured or the call fails.
+// The query text is never logged.
+func (ix *MaterialIndexer) Search(ctx context.Context, req MaterialSearch) (MaterialCandidates, error) {
+ svc := ix.service()
+ if svc == nil {
+ return MaterialCandidates{}, fmt.Errorf("%w: no search service", ErrSearchUnavailable)
+ }
+ engine := svc.Engine()
+ if engine == nil || !engine.Configured() {
+ return MaterialCandidates{}, fmt.Errorf("%w: the search engine is not configured", ErrSearchUnavailable)
+ }
+ q := beachcomber.Query{
+ Q: req.Q,
+ QueryBy: MaterialQueryBy,
+ QueryByWeights: MaterialQueryWeights,
+ FilterBy: materialFilterBy(req),
+ SortBy: "_text_match:desc,published_at:desc",
+ Page: 1,
+ PerPage: MaterialSearchWindow,
+ ExcludeFields: []string{"body", "file_texts"},
+ }
+ highlights := beachcomber.SupportsHighlights(engine)
+ if highlights {
+ q.Highlight = &beachcomber.Highlight{Fields: materialHighlightFields, AffixTokens: 12, MaxSnippetsPerArray: 1}
+ }
+ res, err := beachcomber.SearchPage(ctx, engine, svc.IndexName(&models.Material{}), q)
+ if err != nil {
+ ix.log().Warn("jz.bm: material search failed", "err", err)
+ return MaterialCandidates{}, fmt.Errorf("%w: %w", ErrSearchUnavailable, err)
+ }
+ out := MaterialCandidates{IDs: make([]uint, 0, len(res.IDs)), Hits: make(map[uint]beachcomber.Hit, len(res.IDs)), Highlights: highlights}
+ for i, raw := range res.IDs {
+ n, err := strconv.ParseUint(raw, 10, 64)
+ if err != nil || n == 0 {
+ continue
+ }
+ id := uint(n)
+ if _, seen := out.Hits[id]; seen {
+ continue
+ }
+ out.IDs = append(out.IDs, id)
+ hit := beachcomber.Hit{ID: raw}
+ if i < len(res.Hits) {
+ hit = res.Hits[i]
+ }
+ out.Hits[id] = hit
+ }
+ return out, nil
+}
+
+// materialTagSlug is the shape of a tag slug the engine filter may carry.
+var materialTagSlug = regexp.MustCompile(`^[a-z0-9-]{1,100}$`)
+
+// materialFilterBy narrows the engine's candidates to the list filters. It
+// is built only from parsed integers, an enumerated type and a tag that is
+// a plain slug; any other value is left out and applied by SQL alone, which
+// applies every filter anyway.
+func materialFilterBy(req MaterialSearch) string {
+ var parts []string
+ if req.CourseID != 0 {
+ parts = append(parts, "course_id:="+strconv.FormatUint(uint64(req.CourseID), 10))
+ }
+ if req.MeetingID != 0 {
+ parts = append(parts, "meeting_id:="+strconv.FormatUint(uint64(req.MeetingID), 10))
+ }
+ if slices.Contains(models.MaterialTypes, req.Type) {
+ parts = append(parts, "type:=`"+req.Type+"`")
+ }
+ if materialTagSlug.MatchString(req.Tag) {
+ parts = append(parts, "tag_slugs:=`"+req.Tag+"`")
+ }
+ return strings.Join(parts, " && ")
+}
+
+// Sync upserts the document of material id now. A failure is logged with
+// the id and never returned: a write never fails because of the index.
+func (ix *MaterialIndexer) Sync(ctx context.Context, db *gorm.DB, id uint) {
+ ix.run(ctx, db, id, false)
+}
+
+// Remove deletes the document of material id now, logging a failure like
+// Sync.
+func (ix *MaterialIndexer) Remove(ctx context.Context, db *gorm.DB, id uint) {
+ ix.run(ctx, db, id, true)
+}
+
+// SyncAfterCommit runs Sync once tx commits; with a nil tx, or outside a
+// transaction, it runs at once.
+func (ix *MaterialIndexer) SyncAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
+ ix.afterCommit(ctx, tx, id, false)
+}
+
+// RemoveAfterCommit runs Remove once tx commits, like SyncAfterCommit.
+func (ix *MaterialIndexer) RemoveAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
+ ix.afterCommit(ctx, tx, id, true)
+}
+
+func (ix *MaterialIndexer) afterCommit(ctx context.Context, tx *gorm.DB, id uint, remove bool) {
+ if ix == nil || id == 0 {
+ return
+ }
+ if tx == nil {
+ ix.run(ctx, nil, id, remove)
+ return
+ }
+ lagoon.AfterCommit(ctx, tx, func(ctx context.Context, db *gorm.DB) {
+ ix.run(ctx, db, id, remove)
+ })
+}
+
+func (ix *MaterialIndexer) run(ctx context.Context, db *gorm.DB, id uint, remove bool) {
+ svc := ix.service()
+ if svc == nil || id == 0 {
+ return
+ }
+ ctx = MarkMaterialSearch(ctx)
+ var err error
+ if remove {
+ err = svc.Remove(ctx, db, &models.Material{ID: id})
+ } else {
+ err = svc.Sync(ctx, db, &models.Material{ID: id})
+ }
+ if err != nil {
+ ix.log().Warn("jz.bm: material index sync failed", "material_id", id, "remove", remove, "err", err)
+ }
+}
diff --git a/classes/material_index_test.go b/classes/material_index_test.go
new file mode 100644
index 0000000..ee6df2b
--- /dev/null
+++ b/classes/material_index_test.go
@@ -0,0 +1,121 @@
+package classes_test
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "slices"
+ "strings"
+ "testing"
+
+ "git.golem15.com/jakub/sm-bm-plugin/classes"
+ "git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+)
+
+func TestMaterialSearchGate(t *testing.T) {
+ gate := classes.MaterialSearchGate()
+ if !gate.Enabled(classes.MarkMaterialSearch(context.Background()), nil) {
+ t.Error("a marked context is off")
+ }
+ if gate.Enabled(context.Background(), nil) {
+ t.Error("an unmarked context (a chat message sync) is on")
+ }
+}
+
+func TestMaterialSearchRequest(t *testing.T) {
+ t.Run("unavailable", materialSearchUnavailable)
+ stub := &searchtest.Stub{Hits: []searchtest.Hit{
+ {ID: "12", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd + " dziś"},
+ {ID: "abc"},
+ {ID: "0"},
+ {ID: "5"},
+ }}
+ ix := classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...))
+ got, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "globalizacja", CourseID: 7, MeetingID: 3, Type: "pdf", Tag: "globalizacja"})
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ searches := stub.Searches()
+ if len(searches) != 1 {
+ t.Fatalf("searches = %d, want 1", len(searches))
+ }
+ q := searches[0].Query
+ want := map[string]string{
+ "q": "globalizacja",
+ "query_by": "title,tags,source,course,url,body,file_texts",
+ "query_by_weights": "6,4,3,3,1,2,1",
+ "filter_by": "course_id:=7 && meeting_id:=3 && type:=`pdf` && tag_slugs:=`globalizacja`",
+ "sort_by": "_text_match:desc,published_at:desc",
+ "page": "1",
+ "per_page": "250",
+ "exclude_fields": "body,file_texts",
+ "highlight_fields": "body,file_texts",
+ }
+ for k, v := range want {
+ if q.Get(k) != v {
+ t.Errorf("%s = %q, want %q", k, q.Get(k), v)
+ }
+ }
+ if !slices.Equal(got.IDs, []uint{12, 5}) {
+ t.Errorf("ids = %v, want [12 5]", got.IDs)
+ }
+ if !got.Highlights {
+ t.Error("Highlights is false against the typesense engine")
+ }
+ if h := got.Hits[12]; len(h.Highlights) != 1 || h.Highlights[0].Field != "body" {
+ t.Errorf("hit 12 = %+v", h)
+ } else if segs := h.Highlights[0].Segments; len(segs) != 3 || segs[1].Text != "globalizacji" || !segs[1].Match {
+ t.Errorf("hit 12 segments = %+v", segs)
+ }
+ if _, ok := got.Hits[5]; !ok {
+ t.Error("hit 5 is missing")
+ }
+ if len(got.Hits) != 2 {
+ t.Errorf("hits = %v, want only 12 and 5", got.Hits)
+ }
+
+ // A tag that is not a plain slug and an unknown type stay out of the
+ // engine filter; SQL applies them.
+ if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "x`y", Type: "exe"}); err != nil {
+ t.Fatal(err)
+ }
+ searches = stub.Searches()
+ if f := searches[len(searches)-1].Query.Get("filter_by"); f != "" {
+ t.Errorf("filter_by = %q, want empty", f)
+ }
+ if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "zła-Wielka"}); err != nil {
+ t.Fatal(err)
+ }
+ searches = stub.Searches()
+ if f := searches[len(searches)-1].Query.Get("filter_by"); strings.Contains(f, "tag_slugs") {
+ t.Errorf("filter_by = %q carries a tag that is not a slug", f)
+ }
+}
+
+// materialSearchUnavailable runs under TestMaterialSearchRequest.
+func materialSearchUnavailable(t *testing.T) {
+ // No key: the engine is not configured and nothing is sent.
+ stub := &searchtest.Stub{}
+ env := searchtest.Start(t, stub)
+ noKey := slices.DeleteFunc(slices.Clone(env), func(s string) bool { return strings.Contains(s, "API_KEY") })
+ ix := classes.NewMaterialIndexer(searchtest.App(t, noKey...))
+ if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+ t.Errorf("unconfigured engine: err = %v, want ErrSearchUnavailable", err)
+ }
+ if n := len(stub.Requests()); n != 0 {
+ t.Errorf("unconfigured engine sent %d requests", n)
+ }
+
+ failing := &searchtest.Stub{Status: http.StatusInternalServerError}
+ ix = classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, failing)...))
+ if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+ t.Errorf("failing engine: err = %v, want ErrSearchUnavailable", err)
+ }
+
+ var none *classes.MaterialIndexer
+ if _, err := none.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+ t.Errorf("nil indexer: err = %v, want ErrSearchUnavailable", err)
+ }
+ none.Sync(context.Background(), nil, 1)
+ none.RemoveAfterCommit(context.Background(), nil, 1)
+}
diff --git a/controllers/api/api.go b/controllers/api/api.go
index 48351e6..66ebf3d 100644
--- a/controllers/api/api.go
+++ b/controllers/api/api.go
@@ -17,6 +17,7 @@ import (
"git.golem15.com/golem15/summercms/modules/cooler"
"git.golem15.com/golem15/summercms/modules/phrasebook"
"git.golem15.com/golem15/summercms/modules/wire"
+ "git.golem15.com/jakub/sm-bm-plugin/classes"
"gocloud.dev/blob"
"gorm.io/gorm"
)
@@ -73,6 +74,10 @@ type Handlers struct {
// Cache returns the application cache that holds the issued upload
// tickets until they are used, or nil while none is available.
Cache func() cooler.Cache
+ // MaterialIndex searches the materials index and keeps it current. Nil
+ // means a search (q) answers 503 and the syncs after a write are
+ // skipped.
+ MaterialIndex *classes.MaterialIndexer
}
func (h *Handlers) now() time.Time {
diff --git a/controllers/api/export_test.go b/controllers/api/export_test.go
index a71314e..b50a62e 100644
--- a/controllers/api/export_test.go
+++ b/controllers/api/export_test.go
@@ -2,3 +2,11 @@ package api
// DecodeInline exposes decodeInline to the api_test package.
var DecodeInline = decodeInline
+
+// PickSnippet, FlattenSnippet and RankOrder expose the search helpers to the
+// api_test package.
+var (
+ PickSnippet = pickSnippet
+ FlattenSnippet = flattenSnippet
+ RankOrder = rankOrder
+)
diff --git a/controllers/api/handlers_test.go b/controllers/api/handlers_test.go
index fa1f442..25eec08 100644
--- a/controllers/api/handlers_test.go
+++ b/controllers/api/handlers_test.go
@@ -152,6 +152,7 @@ func TestQueryValidation(t *testing.T) {
{"materials course negative", h.Materials, "/?course=-1", "course"},
{"materials meeting zero", h.Materials, "/?meeting=0", "meeting"},
{"materials tag too long", h.Materials, "/?tag=" + longTag(), "tag"},
+ {"materials q too long", h.Materials, "/?q=" + longTag(), "q"},
}
for _, tc := range cases {
code, body := serve(t, tc.fn, http.MethodGet, tc.target, user, nil)
diff --git a/controllers/api/json.go b/controllers/api/json.go
index 4131bb4..84e41d4 100644
--- a/controllers/api/json.go
+++ b/controllers/api/json.go
@@ -213,6 +213,10 @@ type materialListJSON struct {
Tags []tagJSON `json:"tags"`
FilesCount int64 `json:"files_count"`
IsPrivate bool `json:"is_private"`
+ // Snippet is the matched passage of a search result (*snippetJSON, or
+ // a plain string through MCP). It is nil, and the key absent, without
+ // q or without a highlight.
+ Snippet any `json:"snippet,omitempty"`
}
func newMaterialListJSON(m models.Material, files int64) materialListJSON {
diff --git a/controllers/api/material_search.go b/controllers/api/material_search.go
new file mode 100644
index 0000000..7e7d28c
--- /dev/null
+++ b/controllers/api/material_search.go
@@ -0,0 +1,123 @@
+package api
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+
+ "git.golem15.com/golem15/summercms/modules/beachcomber"
+ "git.golem15.com/golem15/summercms/modules/wire"
+ "git.golem15.com/jakub/sm-bm-plugin/models"
+)
+
+// Material search: the engine (beachcomber, Typesense) answers candidate ids
+// only. Visibility, the filters, the total and the paging stay in SQL, and a
+// row's snippet is read only once SQL returned that row, so a material the
+// caller may not see never reaches the JSON, its count or a snippet.
+
+// minSearchRunes is the shortest query that searches; a shorter one is
+// ignored, as the student app does.
+const minSearchRunes = 2
+
+// searchQuery returns the trimmed q of in, or "" when it is shorter than
+// minSearchRunes.
+func searchQuery(in map[string]any) string {
+ q := strings.TrimSpace(str(in, "q"))
+ if utf8.RuneCountInString(q) < minSearchRunes {
+ return ""
+ }
+ return q
+}
+
+// writeSearchUnavailable answers a search the engine cannot serve: it is not
+// configured or the call failed. The list without q keeps working.
+func writeSearchUnavailable(w http.ResponseWriter) {
+ wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]any{"error": true, "message": "Search is unavailable."})
+}
+
+// rankOrder orders rows by their position in ids, the engine's rank. The ids
+// are parsed integers, so the array literal carries no caller text.
+func rankOrder(ids []uint) string {
+ parts := make([]string, len(ids))
+ for i, id := range ids {
+ parts[i] = strconv.FormatUint(uint64(id), 10)
+ }
+ return "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC"
+}
+
+// segmentJSON is one run of a snippet: the engine's plain text, verbatim,
+// and whether it matched the query. It is never markup; the client renders
+// it as text.
+type segmentJSON struct {
+ Text string `json:"text"`
+ Hit bool `json:"hit"`
+}
+
+// snippetJSON is the one excerpt a search result shows. Source is "body" or
+// "file" (FileName set); Lead and Trail say whether text was cut before or
+// after the excerpt.
+type snippetJSON struct {
+ Source string `json:"source"`
+ FileName *string `json:"file_name"`
+ Lead bool `json:"lead"`
+ Trail bool `json:"trail"`
+ Segments []segmentJSON `json:"segments"`
+}
+
+// pickSnippet returns the snippet of m from hit: the first body snippet,
+// mapped one to one. It is nil when the hit has none. Call it only for a
+// row the SQL re-gate returned.
+func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON {
+ for _, sn := range hit.Highlights {
+ if sn.Field != "body" || len(sn.Segments) == 0 {
+ continue
+ }
+ out := &snippetJSON{Source: "body", Segments: make([]segmentJSON, 0, len(sn.Segments))}
+ var joined strings.Builder
+ for _, seg := range sn.Segments {
+ out.Segments = append(out.Segments, segmentJSON{Text: seg.Text, Hit: seg.Match})
+ joined.WriteString(seg.Text)
+ }
+ body := ""
+ if m.Body != nil {
+ body = models.MaterialPlainBody(*m.Body)
+ }
+ text := strings.TrimSpace(joined.String())
+ out.Lead = !strings.HasPrefix(body, text)
+ out.Trail = !strings.HasSuffix(body, text)
+ return out
+ }
+ return nil
+}
+
+// flattenSnippet is a snippet as plain text, for MCP: the segment texts
+// joined, with an ellipsis where text was cut.
+func flattenSnippet(s *snippetJSON) string {
+ if s == nil {
+ return ""
+ }
+ var b strings.Builder
+ if s.Lead {
+ b.WriteString("…")
+ }
+ for _, seg := range s.Segments {
+ b.WriteString(seg.Text)
+ }
+ if s.Trail {
+ b.WriteString("…")
+ }
+ return b.String()
+}
+
+// snippetValue is the snippet key of a row: the struct, a plain string
+// through MCP, or nil when there is none (the key is then left out).
+func snippetValue(s *snippetJSON, mcp bool) any {
+ if s == nil {
+ return nil
+ }
+ if mcp {
+ return flattenSnippet(s)
+ }
+ return s
+}
diff --git a/controllers/api/material_search_test.go b/controllers/api/material_search_test.go
new file mode 100644
index 0000000..2681351
--- /dev/null
+++ b/controllers/api/material_search_test.go
@@ -0,0 +1,200 @@
+package api_test
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "git.golem15.com/golem15/summercms/modules/beachcomber"
+ "git.golem15.com/golem15/summercms/modules/bouncer"
+ "git.golem15.com/jakub/sm-bm-plugin/classes"
+ "git.golem15.com/jakub/sm-bm-plugin/controllers/api"
+ "git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+ "git.golem15.com/jakub/sm-bm-plugin/models"
+ "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+)
+
+func segments(parts ...any) []beachcomber.Segment {
+ out := make([]beachcomber.Segment, 0, len(parts)/2)
+ for i := 0; i+1 < len(parts); i += 2 {
+ out = append(out, beachcomber.Segment{Text: parts[i].(string), Match: parts[i+1].(bool)})
+ }
+ return out
+}
+
+func TestMaterialSnippet(t *testing.T) {
+ body := "# Wstęp\n\nNa początku. Proces **globalizacji** zmienia państwa. Koniec tekstu."
+ m := models.Material{ID: 3, Body: &body}
+ hit := beachcomber.Hit{ID: "3", Highlights: []beachcomber.Snippet{
+ {Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
+ {Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia państwa.", false)},
+ }}
+ s := api.PickSnippet(hit, m)
+ if s == nil {
+ t.Fatal("no snippet")
+ }
+ if s.Source != "body" || s.FileName != nil {
+ t.Errorf("source %q file %v, want body and no file", s.Source, s.FileName)
+ }
+ if len(s.Segments) != 3 || s.Segments[1].Text != "globalizacji" || !s.Segments[1].Hit || s.Segments[0].Hit {
+ t.Errorf("segments = %+v", s.Segments)
+ }
+ if s.Segments[2].Text != " zmienia państwa." {
+ t.Errorf("segment text was changed: %q", s.Segments[2].Text)
+ }
+ if !s.Lead || !s.Trail {
+ t.Errorf("lead %v trail %v, want both (cut inside the body)", s.Lead, s.Trail)
+ }
+ b, err := json.Marshal(s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := string(b); !strings.Contains(got, `"source":"body","file_name":null,"lead":true,"trail":true,"segments":[{"text":"Proces ","hit":false}`) {
+ t.Errorf("JSON = %s", got)
+ }
+
+ // The whole body: neither lead nor trail.
+ short := "Krótka **notatka**"
+ whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
+ if whole == nil || whole.Lead || whole.Trail {
+ t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
+ }
+ // Start of the body only: trail but no lead.
+ start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
+ if start == nil || start.Lead || !start.Trail {
+ t.Errorf("start snippet = %+v, want trail only", start)
+ }
+
+ // A snippet on another field only, or none at all, gives no snippet.
+ if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
+ t.Errorf("title-only hit gave %+v", got)
+ }
+ if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
+ t.Errorf("hit without highlights gave %+v", got)
+ }
+
+ if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia państwa.…" {
+ t.Errorf("FlattenSnippet = %q", got)
+ }
+ if got := api.FlattenSnippet(nil); got != "" {
+ t.Errorf("FlattenSnippet(nil) = %q", got)
+ }
+ if got := api.RankOrder([]uint{9, 3}); got != "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" {
+ t.Errorf("RankOrder = %q", got)
+ }
+}
+
+func TestMaterialsSearchUnavailable(t *testing.T) {
+ h := noDB()
+ user := &bouncer.Principal{ID: 7}
+ code, body := serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+ if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." {
+ t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body)
+ }
+ // One rune is no search: the list answers as without q (no database).
+ code, body = serve(t, h.Materials, http.MethodGet, "/?q=%20g%20", user, nil)
+ if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+ t.Errorf("one-rune q = %d %v, want the no-database answer", code, body)
+ }
+ code, body = serve(t, h.Materials, http.MethodGet, "/", user, nil)
+ if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+ t.Errorf("no q = %d %v, want the no-database answer", code, body)
+ }
+ // An unconfigured engine is the same 503.
+ h.MaterialIndex = classes.NewMaterialIndexer(searchtest.App(t))
+ code, body = serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+ if code != http.StatusServiceUnavailable || body["message"] != "Search is unavailable." {
+ t.Errorf("q with a null engine = %d %v, want 503 Search is unavailable.", code, body)
+ }
+}
+
+// sqlRecorder is a dry-run postgres handle that records every query it
+// builds, with its arguments inlined.
+type sqlRecorder struct {
+ mu sync.Mutex
+ sql []string
+}
+
+func (r *sqlRecorder) queries() []string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return append([]string(nil), r.sql...)
+}
+
+func dryDB(t *testing.T) (*gorm.DB, *sqlRecorder) {
+ t.Helper()
+ db, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec := &sqlRecorder{}
+ err = db.Callback().Query().After("gorm:query").Register("test:record_sql", func(tx *gorm.DB) {
+ rec.mu.Lock()
+ defer rec.mu.Unlock()
+ rec.sql = append(rec.sql, tx.Dialector.Explain(tx.Statement.SQL.String(), tx.Statement.Vars...))
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return db, rec
+}
+
+func TestMaterialsSearchTrace(t *testing.T) {
+ stub := &searchtest.Stub{Hits: []searchtest.Hit{
+ {ID: "9"},
+ {ID: "3", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd},
+ }}
+ db, rec := dryDB(t)
+ h := &api.Handlers{
+ DB: func() *gorm.DB { return db },
+ MaterialIndex: classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)),
+ }
+ req := httptest.NewRequest(http.MethodGet, "/materials?q=globalizacja&course=7", nil)
+ req = req.WithContext(bouncer.WithUser(req.Context(), &bouncer.Principal{ID: 5}))
+ w := httptest.NewRecorder()
+ h.Materials(w, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", w.Code, w.Body.String())
+ }
+ if n := len(stub.Searches()); n != 1 {
+ t.Errorf("searches = %d, want 1", n)
+ }
+ var count, page string
+ for _, q := range rec.queries() {
+ switch {
+ case strings.HasPrefix(q, "SELECT count(*)"):
+ count = q
+ case strings.HasPrefix(q, "SELECT") && strings.Contains(q, `FROM "jz_bm_materials"`):
+ page = q
+ }
+ }
+ if count == "" || page == "" {
+ t.Fatalf("queries = %q, want a count and a page", rec.queries())
+ }
+ for name, q := range map[string]string{"count": count, "page": page} {
+ for _, want := range []string{"jz_bm_materials.is_private = FALSE", "jz_bm_materials.published_at IS NOT NULL", "jz_bm_materials.course_id = 7", "jz_bm_materials.id IN (9,3)"} {
+ if !strings.Contains(q, want) {
+ t.Errorf("%s query lacks %q:\n%s", name, want, q)
+ }
+ }
+ }
+ if !strings.Contains(page, "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id)") {
+ t.Errorf("page query is not in rank order:\n%s", page)
+ }
+ var out struct {
+ Data []map[string]any `json:"data"`
+ Meta struct {
+ Total int `json:"total"`
+ } `json:"meta"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
+ t.Fatal(err)
+ }
+ if out.Meta.Total != 0 || len(out.Data) != 0 {
+ t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data)
+ }
+}
diff --git a/controllers/api/materials.go b/controllers/api/materials.go
index 81c62bd..8311304 100644
--- a/controllers/api/materials.go
+++ b/controllers/api/materials.go
@@ -6,9 +6,11 @@ import (
"strings"
"time"
+ "git.golem15.com/golem15/summercms/modules/beachcomber"
"git.golem15.com/golem15/summercms/modules/lagoon"
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
"git.golem15.com/golem15/summercms/modules/surf"
+ "git.golem15.com/jakub/sm-bm-plugin/classes"
"git.golem15.com/jakub/sm-bm-plugin/models"
"gorm.io/gorm"
)
@@ -49,23 +51,35 @@ func preloadMaterial(q *gorm.DB) *gorm.DB {
}
// listMaterials returns one page of the published materials userID may
-// see, newest first.
-func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, page, perPage int) ([]materialListJSON, int64, error) {
+// see. With ids nil the page is newest first. With ids non-nil (search
+// candidates in the engine's rank order) both the count and the page are
+// also limited to ids and the page follows their order; the ids never widen
+// what visibleMaterials and the filter allow. An empty ids is an empty page.
+func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, ids []uint, page, perPage int) ([]materialListJSON, []models.Material, int64, error) {
+ if ids != nil && len(ids) == 0 {
+ return []materialListJSON{}, nil, 0, nil
+ }
q := f.apply(visibleMaterials(db.Model(&models.Material{}), now, userID))
+ if ids != nil {
+ q = q.Where("jz_bm_materials.id IN ?", ids)
+ }
var total int64
if err := q.Count(&total).Error; err != nil {
- return nil, 0, err
+ return nil, nil, 0, err
+ }
+ rowsQ := preloadMaterial(f.apply(visibleMaterials(db, now, userID)))
+ if ids != nil {
+ rowsQ = rowsQ.Where("jz_bm_materials.id IN ?", ids).Order(rankOrder(ids))
+ } else {
+ rowsQ = rowsQ.Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC")
}
var rows []models.Material
- err := preloadMaterial(f.apply(visibleMaterials(db, now, userID))).
- Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC").
- Offset((page - 1) * perPage).Limit(perPage).
- Find(&rows).Error
+ err := rowsQ.Offset((page - 1) * perPage).Limit(perPage).Find(&rows).Error
if err != nil {
- return nil, 0, err
+ return nil, nil, 0, err
}
out, err := materialsJSON(db, rows)
- return out, total, err
+ return out, rows, total, err
}
// materialsJSON builds list rows with their file counts.
@@ -99,31 +113,69 @@ func materialsJSON(db *gorm.DB, rows []models.Material) ([]materialListJSON, err
return out, nil
}
-// Materials answers GET /materials?course=&meeting=&tag=&type=&page=&per_page=:
+// Materials answers GET /materials?q=&course=&meeting=&tag=&type=&page=&per_page=:
// a page of the published materials the caller may see, in the {data, meta}
-// shape of lagoon.Paginate.
+// shape of lagoon.Paginate. With q (2 to 100 characters) the engine answers
+// candidates, which SQL re-gates, counts and pages in rank order, each row
+// with at most one snippet; when the engine cannot answer the request is a
+// 503. Without q the list is newest first and never calls the engine.
func (h *Handlers) Materials(w http.ResponseWriter, r *http.Request) {
userID, ok := h.caller(w, r)
if !ok {
return
}
- in := query(r, "course", "meeting", "tag", "type", "page", "per_page")
+ in := query(r, "q", "course", "meeting", "tag", "type", "page", "per_page")
if !h.validate(w, r, in, materialsRules) {
return
}
+ filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
+ var cands *classes.MaterialCandidates
+ if q := searchQuery(in); q != "" {
+ c, err := h.MaterialIndex.Search(r.Context(), classes.MaterialSearch{Q: q, CourseID: filter.CourseID, MeetingID: filter.MeetingID, Type: filter.Type, Tag: filter.Tag})
+ if err != nil {
+ writeSearchUnavailable(w)
+ return
+ }
+ cands = &c
+ }
db, ok := h.db(w, r)
if !ok {
return
}
- filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
page := min(intParam(in, "page", 1), 100000)
perPage := intParam(in, "per_page", h.perPage())
- rows, total, err := listMaterials(db, h.now(), userID, filter, page, perPage)
+ var ids []uint
+ if cands != nil {
+ ids = cands.IDs
+ if ids == nil {
+ ids = []uint{}
+ }
+ }
+ out, rows, total, err := listMaterials(db, h.now(), userID, filter, ids, page, perPage)
if err != nil {
writeOpaque500(w)
return
}
- writeJSONPage(w, lagoon.Paginate(rows, page, perPage, total))
+ if cands != nil {
+ attachSnippets(out, rows, cands.Hits, viaMCP(r))
+ }
+ writeJSONPage(w, lagoon.Paginate(out, page, perPage, total))
+}
+
+// attachSnippets sets the snippet of each list row from its Hit. rows are
+// the rows SQL returned (they passed the re-gate), in the order of out; the
+// Hit of an id SQL did not return is never read.
+func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, mcp bool) {
+ for i := range out {
+ if i >= len(rows) || rows[i].ID != out[i].ID {
+ continue
+ }
+ hit, ok := hits[rows[i].ID]
+ if !ok {
+ continue
+ }
+ out[i].Snippet = snippetValue(pickSnippet(hit, rows[i]), mcp)
+ }
}
// Material answers GET /materials/{id}: one published material the caller
diff --git a/controllers/api/params.go b/controllers/api/params.go
index 34c95a2..515d259 100644
--- a/controllers/api/params.go
+++ b/controllers/api/params.go
@@ -37,6 +37,7 @@ var (
{Field: "type", Rules: append(lagoon.ParseRules("nullable"), lagoon.In(models.MaterialTypes...))},
{Field: "page", Rules: lagoon.ParseRules("nullable|integer|min:1")},
{Field: "per_page", Rules: lagoon.ParseRules("nullable|integer|between:1,100")},
+ {Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")},
}
)
diff --git a/go.mod b/go.mod
index dd5259d..ef91cfd 100644
--- a/go.mod
+++ b/go.mod
@@ -6,19 +6,19 @@ require (
git.golem15.com/golem15/sm-chat-plugin v0.1.3
git.golem15.com/golem15/sm-chatchannels-plugin v0.0.0
git.golem15.com/golem15/sm-user-plugin v0.1.2
- git.golem15.com/golem15/summercms v0.5.0
+ git.golem15.com/golem15/summercms v0.6.4
github.com/go-gormigrate/gormigrate/v2 v2.1.7
github.com/jackc/pgx/v5 v5.10.0
github.com/testcontainers/testcontainers-go v0.44.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
gocloud.dev v0.46.0
+ gorm.io/driver/postgres v1.6.3
gorm.io/gorm v1.31.2
)
require (
charm.land/bubbles/v2 v2.2.1 // indirect
charm.land/bubbletea/v2 v2.1.0 // indirect
- charm.land/fang/v2 v2.0.1 // indirect
charm.land/huh/v2 v2.0.3 // indirect
charm.land/lipgloss/v2 v2.0.6 // indirect
dario.cat/mergo v1.0.2 // indirect
@@ -52,7 +52,6 @@ require (
github.com/charmbracelet/harmonica v0.2.0 // indirect
github.com/charmbracelet/ultraviolet v0.0.0-20261008173134-6b8d4baf91b4 // indirect
github.com/charmbracelet/x/ansi v0.11.9 // indirect
- github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 // indirect
github.com/charmbracelet/x/exp/ordered v0.1.0 // indirect
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
github.com/charmbracelet/x/term v0.2.2 // indirect
@@ -123,10 +122,6 @@ require (
github.com/moby/term v0.5.2 // indirect
github.com/modelcontextprotocol/go-sdk v1.8.0 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
- github.com/muesli/mango v0.1.0 // indirect
- github.com/muesli/mango-cobra v1.2.0 // indirect
- github.com/muesli/mango-pflag v0.1.0 // indirect
- github.com/muesli/roff v0.1.0 // indirect
github.com/nicksnyder/go-i18n/v2 v2.6.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
@@ -180,7 +175,6 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect
google.golang.org/grpc v1.79.3 // indirect
google.golang.org/protobuf v1.36.11 // indirect
- gorm.io/driver/postgres v1.6.3 // indirect
)
replace git.golem15.com/golem15/summercms => ../../../../summercms.go
diff --git a/go.sum b/go.sum
index abcd5a0..d61206e 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,6 @@ charm.land/bubbles/v2 v2.2.1 h1:Fq1+qm5hV6GkvzLQDhCBpXXE5tLgvh1PRriCLwSvIQU=
charm.land/bubbles/v2 v2.2.1/go.mod h1:wdMgn+sje1KNXdwFizIWjbf328fIUBxqEmJ/vYPo8yc=
charm.land/bubbletea/v2 v2.1.0 h1:A9fj9h5LFrvQhQHFNggp8K8rvY6CUfgn9dMbb6p2HIw=
charm.land/bubbletea/v2 v2.1.0/go.mod h1:xWHeyxIX9/wiFU2kgPW04ojOZyTzrWf+8lGjUCaftiU=
-charm.land/fang/v2 v2.0.1 h1:zQCM8JQJ1JnQX/66B5jlCYBUxL2as5JXQZ2KJ6EL0mY=
-charm.land/fang/v2 v2.0.1/go.mod h1:S1GmkpcvK+OB5w9caywUnJcsMew45Ot8FXqoz8ALrII=
charm.land/huh/v2 v2.0.3 h1:2cJsMqEPwSywGHvdlKsJyQKPtSJLVnFKyFbsYZTlLkU=
charm.land/huh/v2 v2.0.3/go.mod h1:93eEveeeqn47MwiC3tf+2atZ2l7Is88rAtmZNZ8x9Wc=
charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ=
@@ -110,8 +108,6 @@ github.com/charmbracelet/x/conpty v0.1.1 h1:s1bUxjoi7EpqiXysVtC+a8RrvPPNcNvAjfi4
github.com/charmbracelet/x/conpty v0.1.1/go.mod h1:OmtR77VODEFbiTzGE9G1XiRJAga6011PIm4u5fTNZpk=
github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA=
github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0=
-github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 h1:IJDiTgVE56gkAGfq0lBEloWgkXMk4hl/bmuPoicI4R0=
-github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444/go.mod h1:T9jr8CzFpjhFVHjNjKwbAD7KwBNyFnj2pntAO7F2zw0=
github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA=
github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I=
github.com/charmbracelet/x/exp/ordered v0.1.0 h1:55/qLwjIh0gL0Vni+QAWk7T/qRVP6sBf+2agPBgnOFE=
@@ -299,14 +295,6 @@ github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtw
github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts=
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
-github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI=
-github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4=
-github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg=
-github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA=
-github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg=
-github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0=
-github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
-github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
diff --git a/internal/searchtest/typesense.go b/internal/searchtest/typesense.go
new file mode 100644
index 0000000..f559a26
--- /dev/null
+++ b/internal/searchtest/typesense.go
@@ -0,0 +1,198 @@
+// Package searchtest is a Typesense stand-in for the jz.bm tests: an
+// httptest server that answers the collection, import, delete and search
+// endpoints the beachcomber typesense engine calls, and an app configured to
+// talk to it. Only tests import it.
+package searchtest
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ "git.golem15.com/golem15/summercms/modules/backpack"
+ _ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+ "git.golem15.com/golem15/summercms/modules/compass"
+)
+
+// Index is the collection of the materials index.
+const Index = "jz_bm_materials"
+
+// HighlightStart and HighlightEnd are the highlight tags the typesense
+// engine asks for (two private-use characters).
+const (
+ HighlightStart = ""
+ HighlightEnd = ""
+)
+
+// Hit is one search answer: a document id and, optionally, the snippet of
+// one field with the matched words between HighlightStart and HighlightEnd.
+type Hit struct {
+ ID string
+ Field string
+ Snippet string
+}
+
+// Request is one request the stub received.
+type Request struct {
+ Method string
+ Path string
+ Query url.Values
+}
+
+// Stub stands in for a Typesense server with one collection.
+type Stub struct {
+ mu sync.Mutex
+ // Hits answer every search, in rank order.
+ Hits []Hit
+ // Status, when set, answers every request with that code.
+ Status int
+
+ exists bool
+ requests []Request
+ imported []map[string]any
+ removed []string
+}
+
+// ServeHTTP implements http.Handler.
+func (s *Stub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query()})
+ if s.Status != 0 {
+ w.WriteHeader(s.Status)
+ _, _ = io.WriteString(w, `{"message":"stub failure"}`)
+ return
+ }
+ coll := "/collections/" + Index
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == coll:
+ if !s.exists {
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = io.WriteString(w, `{"message":"Not Found"}`)
+ return
+ }
+ _, _ = io.WriteString(w, `{}`)
+ case r.Method == http.MethodPost && r.URL.Path == "/collections":
+ s.exists = true
+ w.WriteHeader(http.StatusCreated)
+ _, _ = io.WriteString(w, `{}`)
+ case r.Method == http.MethodDelete && r.URL.Path == coll:
+ if !s.exists {
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = io.WriteString(w, `{"message":"Not Found"}`)
+ return
+ }
+ s.exists = false
+ _, _ = io.WriteString(w, `{}`)
+ case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import":
+ s.exists = true
+ sc := bufio.NewScanner(r.Body)
+ sc.Buffer(make([]byte, 0, 64*1024), 16<<20)
+ var answer []string
+ for sc.Scan() {
+ var doc map[string]any
+ if err := json.Unmarshal(sc.Bytes(), &doc); err == nil {
+ s.imported = append(s.imported, doc)
+ }
+ answer = append(answer, `{"success":true}`)
+ }
+ _, _ = io.WriteString(w, strings.Join(answer, "\n"))
+ case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"):
+ s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/"))
+ _, _ = io.WriteString(w, `{}`)
+ case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search":
+ hits := make([]string, 0, len(s.Hits))
+ for _, h := range s.Hits {
+ id, _ := json.Marshal(h.ID)
+ highlights := "[]"
+ if h.Field != "" {
+ field, _ := json.Marshal(h.Field)
+ snippet, _ := json.Marshal(h.Snippet)
+ highlights = fmt.Sprintf(`[{"field":%s,"snippet":%s}]`, field, snippet)
+ }
+ hits = append(hits, fmt.Sprintf(`{"document":{"id":%s},"highlights":%s}`, id, highlights))
+ }
+ // found is deliberately larger than the hits, so a caller that
+ // exposed it would show.
+ _, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.Hits)+100, strings.Join(hits, ","))
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = io.WriteString(w, `{"message":"Not Found"}`)
+ }
+}
+
+// Requests returns the requests received so far.
+func (s *Stub) Requests() []Request {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]Request(nil), s.requests...)
+}
+
+// Searches returns the search requests received so far.
+func (s *Stub) Searches() []Request {
+ var out []Request
+ for _, r := range s.Requests() {
+ if r.Method == http.MethodGet && strings.HasSuffix(r.Path, "/documents/search") {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// Imported returns the documents upserted so far.
+func (s *Stub) Imported() []map[string]any {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]map[string]any(nil), s.imported...)
+}
+
+// Removed returns the document ids deleted so far.
+func (s *Stub) Removed() []string {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]string(nil), s.removed...)
+}
+
+// Start serves stub on an httptest server closed with the test, and returns
+// the SUMMER_SEARCH__* environment that points the typesense engine at it
+// with a test-only key.
+func Start(t testing.TB, stub *Stub) []string {
+ t.Helper()
+ srv := httptest.NewServer(stub)
+ t.Cleanup(srv.Close)
+ u, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return []string{
+ "SUMMER_SEARCH__DRIVER=typesense",
+ "SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key",
+ "SUMMER_SEARCH__TYPESENSE__HOST=" + u.Hostname(),
+ "SUMMER_SEARCH__TYPESENSE__PORT=" + u.Port(),
+ "SUMMER_SEARCH__TYPESENSE__PROTOCOL=" + u.Scheme,
+ }
+}
+
+// App returns a bare application whose config holds env (and SUMMER_ENV
+// testing) and an empty config directory.
+func App(t testing.TB, env ...string) *backpack.App {
+ t.Helper()
+ dir := t.TempDir()
+ if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: jz-bm-test\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg, err := compass.Open(compass.Options{Dir: dir, Environ: append([]string{"SUMMER_ENV=testing"}, env...)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return backpack.New(cfg)
+}
diff --git a/models/material_search.go b/models/material_search.go
new file mode 100644
index 0000000..5c145b1
--- /dev/null
+++ b/models/material_search.go
@@ -0,0 +1,173 @@
+package models
+
+import (
+ "context"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "gorm.io/gorm"
+)
+
+// A material is kept in the search index jz_bm_materials (beachcomber). The
+// index holds every material, drafts and private ones included: visibility
+// is enforced only in SQL, where the engine's ids are candidates that the
+// student queries re-gate. This file is hand-written; material.go is
+// generated.
+
+// SearchableAs is the index name without the search.prefix.
+func (Material) SearchableAs() string { return "jz_bm_materials" }
+
+// ShouldBeSearchable is true for every material: the index carries no
+// visibility, so drafts and private materials are indexed too.
+func (m *Material) ShouldBeSearchable() bool { return m != nil }
+
+// ToSearchableArray builds the document of the material. beachcomber reloads
+// the row by primary key without preloads, so the course, the meeting number
+// and the tags are loaded here.
+func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
+ db = db.WithContext(ctx)
+ var course *Course
+ if m.CourseID != nil {
+ var c Course
+ res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c)
+ if res.Error != nil {
+ return nil, res.Error
+ }
+ if res.RowsAffected > 0 {
+ course = &c
+ }
+ }
+ meetingNumber := 0
+ if m.MeetingID != nil {
+ var mt Meeting
+ res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt)
+ if res.Error != nil {
+ return nil, res.Error
+ }
+ if res.RowsAffected > 0 {
+ meetingNumber = mt.Number
+ }
+ }
+ var tags []Tag
+ err := db.Model(&Tag{}).
+ Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id").
+ Where("mt.material_id = ?", m.ID).
+ Order("jz_bm_tags.name").
+ Find(&tags).Error
+ if err != nil {
+ return nil, err
+ }
+ return MaterialDocument(*m, course, meetingNumber, tags, nil), nil
+}
+
+// MaterialDocument is the search document of m with its course, meeting
+// number (0 when none), tags in order and the texts of its attached files.
+// Absent numbers are 0, so filters and the sort tiebreak never see a
+// missing value.
+func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any {
+ doc := map[string]any{
+ "id": strconv.FormatUint(uint64(m.ID), 10),
+ "title": m.Title,
+ "type": m.Type,
+ "course_id": int64(0),
+ "meeting_id": int64(0),
+ "published_at": int64(0),
+ }
+ if m.Source != nil && strings.TrimSpace(*m.Source) != "" {
+ doc["source"] = *m.Source
+ }
+ if m.URL != nil && strings.TrimSpace(*m.URL) != "" {
+ doc["url"] = *m.URL
+ }
+ if m.CourseID != nil {
+ doc["course_id"] = int64(*m.CourseID)
+ }
+ if m.MeetingID != nil {
+ doc["meeting_id"] = int64(*m.MeetingID)
+ }
+ if m.PublishedAt != nil {
+ doc["published_at"] = m.PublishedAt.Unix()
+ }
+ if course != nil {
+ label := course.Name
+ if course.Code != nil && strings.TrimSpace(*course.Code) != "" {
+ label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code))
+ }
+ doc["course"] = label
+ }
+ if meetingNumber > 0 {
+ doc["meeting_number"] = meetingNumber
+ }
+ names := make([]string, 0, len(tags))
+ slugs := make([]string, 0, len(tags))
+ for _, t := range tags {
+ names = append(names, t.Name)
+ slugs = append(slugs, t.Slug)
+ }
+ doc["tags"] = names
+ doc["tag_slugs"] = slugs
+ if m.Body != nil {
+ if body := MaterialPlainBody(*m.Body); body != "" {
+ doc["body"] = body
+ }
+ }
+ if len(fileTexts) > 0 {
+ doc["file_texts"] = fileTexts
+ }
+ return doc
+}
+
+// SearchIndexSchema is the Typesense collection schema. It applies only when
+// the collection is created; bm:materials-reindex --fresh recreates it.
+// Every field is stored (a search leaves body and file_texts out of the
+// answer instead), and no field sets a locale: the default one folds
+// diacritics, so lodz finds Łódź.
+func (Material) SearchIndexSchema() map[string]any {
+ return map[string]any{
+ "fields": []map[string]any{
+ {"name": "id", "type": "string"},
+ {"name": "title", "type": "string"},
+ {"name": "source", "type": "string", "optional": true},
+ {"name": "url", "type": "string", "optional": true},
+ {"name": "tags", "type": "string[]", "optional": true},
+ {"name": "tag_slugs", "type": "string[]", "optional": true},
+ {"name": "course", "type": "string", "optional": true},
+ {"name": "course_id", "type": "int64"},
+ {"name": "meeting_id", "type": "int64"},
+ {"name": "meeting_number", "type": "int32", "optional": true},
+ {"name": "type", "type": "string"},
+ {"name": "published_at", "type": "int64"},
+ {"name": "body", "type": "string", "optional": true},
+ {"name": "file_texts", "type": "string[]", "optional": true},
+ },
+ }
+}
+
+var (
+ mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
+ mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
+ mdHeading = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`)
+ mdQuote = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`)
+ mdList = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`)
+ mdEmphasis = regexp.MustCompile(`\*+|~~`)
+ mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`)
+ mdUnderEnd = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`)
+)
+
+// MaterialPlainBody drops the Markdown syntax of a material body (heading
+// hashes, blockquote and list markers, emphasis, inline code ticks, link
+// and image syntax down to their text) and collapses white space, so the
+// index holds and highlights only words.
+func MaterialPlainBody(body string) string {
+ s := strings.ReplaceAll(body, "`", "")
+ s = mdImage.ReplaceAllString(s, "$1")
+ s = mdLink.ReplaceAllString(s, "$1")
+ s = mdHeading.ReplaceAllString(s, "")
+ s = mdQuote.ReplaceAllString(s, "")
+ s = mdList.ReplaceAllString(s, "")
+ s = mdEmphasis.ReplaceAllString(s, "")
+ s = mdUnderStart.ReplaceAllString(s, "$1")
+ s = mdUnderEnd.ReplaceAllString(s, "$1")
+ return strings.Join(strings.Fields(s), " ")
+}
diff --git a/models/material_search_test.go b/models/material_search_test.go
new file mode 100644
index 0000000..9b32ae6
--- /dev/null
+++ b/models/material_search_test.go
@@ -0,0 +1,124 @@
+package models_test
+
+import (
+ "slices"
+ "testing"
+ "time"
+
+ "git.golem15.com/jakub/sm-bm-plugin/models"
+)
+
+func TestMaterialSearchSchema(t *testing.T) {
+ m := models.Material{}
+ if got := m.SearchableAs(); got != "jz_bm_materials" {
+ t.Errorf("SearchableAs = %q, want jz_bm_materials", got)
+ }
+ fields, ok := m.SearchIndexSchema()["fields"].([]map[string]any)
+ if !ok || len(fields) == 0 {
+ t.Fatalf("schema fields = %#v", m.SearchIndexSchema()["fields"])
+ }
+ byName := map[string]map[string]any{}
+ for _, f := range fields {
+ name, _ := f["name"].(string)
+ typ, _ := f["type"].(string)
+ if name == "" || typ == "" {
+ t.Errorf("field without a name or type: %v", f)
+ }
+ if _, ok := f["locale"]; ok {
+ t.Errorf("field %s sets a locale; the default one folds Polish diacritics", name)
+ }
+ if store, ok := f["store"]; ok && store == false {
+ t.Errorf("field %s turns storage off", name)
+ }
+ byName[name] = f
+ }
+ for _, name := range []string{"body", "file_texts"} {
+ f, ok := byName[name]
+ if !ok {
+ t.Errorf("schema has no %s field", name)
+ continue
+ }
+ if f["optional"] != true {
+ t.Errorf("%s is not optional", name)
+ }
+ }
+ if byName["body"]["type"] != "string" || byName["file_texts"]["type"] != "string[]" {
+ t.Errorf("body %v, file_texts %v: want string and string[]", byName["body"]["type"], byName["file_texts"]["type"])
+ }
+ for _, name := range []string{"id", "title", "source", "url", "tags", "tag_slugs", "course", "course_id", "meeting_id", "meeting_number", "type", "published_at"} {
+ if _, ok := byName[name]; !ok {
+ t.Errorf("schema has no %s field", name)
+ }
+ }
+ if !(&m).ShouldBeSearchable() {
+ t.Error("a material is not searchable")
+ }
+ var nilMaterial *models.Material
+ if nilMaterial.ShouldBeSearchable() {
+ t.Error("a nil material is searchable")
+ }
+}
+
+func TestMaterialDocument(t *testing.T) {
+ plain := models.MaterialPlainBody("# Nagłówek\n\nTekst **pogrubienie** i _kursywa_ oraz [tekst](https://x) i .\n\n> cytat\n> dalej\n\n- punkt `kod` snake_case")
+ want := "Nagłówek Tekst pogrubienie i kursywa oraz tekst i obraz. cytat dalej punkt kod snake_case"
+ if plain != want {
+ t.Errorf("MaterialPlainBody =\n%q\nwant\n%q", plain, want)
+ }
+
+ code := "BM-12"
+ course := &models.Course{ID: 4, Name: "Bezpieczeństwo", Code: &code}
+ courseID, meetingID := uint(4), uint(9)
+ at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
+ body := "## Wstęp\n\n**Globalizacja** w praktyce"
+ source := "Giddens"
+ m := models.Material{ID: 12, Title: "Globalizacja", Type: "notatka", CourseID: &courseID, MeetingID: &meetingID, PublishedAt: &at, Body: &body, Source: &source}
+ tags := []models.Tag{{Name: "Globalizacja", Slug: "globalizacja"}, {Name: "Polityka", Slug: "polityka"}}
+ doc := models.MaterialDocument(m, course, 3, tags, []string{"tekst pliku"})
+ checks := map[string]any{
+ "id": "12",
+ "title": "Globalizacja",
+ "type": "notatka",
+ "source": "Giddens",
+ "course": "Bezpieczeństwo BM-12",
+ "course_id": int64(4),
+ "meeting_id": int64(9),
+ "meeting_number": 3,
+ "published_at": at.Unix(),
+ "body": "Wstęp Globalizacja w praktyce",
+ }
+ for k, v := range checks {
+ if doc[k] != v {
+ t.Errorf("doc[%s] = %#v, want %#v", k, doc[k], v)
+ }
+ }
+ if got, _ := doc["tags"].([]string); !slices.Equal(got, []string{"Globalizacja", "Polityka"}) {
+ t.Errorf("tags = %v", doc["tags"])
+ }
+ if got, _ := doc["tag_slugs"].([]string); !slices.Equal(got, []string{"globalizacja", "polityka"}) {
+ t.Errorf("tag_slugs = %v", doc["tag_slugs"])
+ }
+ if got, _ := doc["file_texts"].([]string); !slices.Equal(got, []string{"tekst pliku"}) {
+ t.Errorf("file_texts = %v", doc["file_texts"])
+ }
+
+ // No code, no course id, no meeting, a draft, no files.
+ bare := models.MaterialDocument(models.Material{ID: 5, Title: "Szkic", Type: "inne"}, &models.Course{Name: "Teoria"}, 0, nil, nil)
+ if bare["course"] != "Teoria" {
+ t.Errorf("course without code = %v", bare["course"])
+ }
+ for _, k := range []string{"course_id", "meeting_id", "published_at"} {
+ if bare[k] != int64(0) {
+ t.Errorf("bare %s = %#v, want 0", k, bare[k])
+ }
+ }
+ for _, k := range []string{"meeting_number", "file_texts", "body", "source", "url"} {
+ if _, ok := bare[k]; ok {
+ t.Errorf("bare document carries %s", k)
+ }
+ }
+ none := models.MaterialDocument(models.Material{ID: 6, Title: "X", Type: "link"}, nil, 0, nil, nil)
+ if _, ok := none["course"]; ok {
+ t.Error("a material without a course carries course")
+ }
+}
diff --git a/plugin.go b/plugin.go
index 586fadd..5d22da3 100644
--- a/plugin.go
+++ b/plugin.go
@@ -59,6 +59,9 @@ type Plugin struct {
// chatMember is the chat-group check of the realtime access guard; nil
// means classes.IsChatMember. Tests inject their own.
chatMember classes.MemberFunc
+ // materialIndex searches the materials index and keeps it current. Nil
+ // when Boot did not run: then search answers 503 and syncs are skipped.
+ materialIndex *classes.MaterialIndexer
}
func (p *Plugin) ID() string { return "jz.bm" }
@@ -74,7 +77,8 @@ func (p *Plugin) Register(app *backpack.App) error {
// Boot keeps the application, so route handlers and commands can reach its
// services, such as the database, when they run, and wires the realtime
-// service and, when golem15.chat is active, the BM chat context. The
+// service, the materials search and, when golem15.chat is active, the BM
+// chat context. The
// management API uses the shared user.api_token middleware owned by
// golem15.user.
func (p *Plugin) Boot(app *backpack.App) error {
@@ -83,6 +87,9 @@ func (p *Plugin) Boot(app *backpack.App) error {
if err := p.wireRealtime(app); err != nil {
return err
}
+ if err := p.wireSearch(app); err != nil {
+ return err
+ }
if err := p.registerChat(app); err != nil {
return err
}
diff --git a/routes.go b/routes.go
index f7bccd5..acf5200 100644
--- a/routes.go
+++ b/routes.go
@@ -304,6 +304,7 @@ func (p *Plugin) mountRealtime(r pact.Router) error {
// publishes it.
func (p *Plugin) handlers() *api.Handlers {
h := &api.Handlers{
+ MaterialIndex: p.materialIndex,
DB: func() *gorm.DB {
if p.app == nil {
return nil
diff --git a/search.go b/search.go
new file mode 100644
index 0000000..8685585
--- /dev/null
+++ b/search.go
@@ -0,0 +1,23 @@
+package bm
+
+import (
+ "git.golem15.com/golem15/summercms/modules/backpack"
+ "git.golem15.com/golem15/summercms/modules/beachcomber"
+ _ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+ "git.golem15.com/jakub/sm-bm-plugin/classes"
+)
+
+// wireSearch builds the app's beachcomber service (search.driver; the
+// typesense engine registers itself through the blank import), installs the
+// jz.bm-only Gate and the material indexer. The Gate is app-wide, so it is
+// on only for contexts jz.bm marks: golem15.chatchannels' message syncs stay
+// off and chat messages never reach the index.
+func (p *Plugin) wireSearch(app *backpack.App) error {
+ svc, err := beachcomber.From(app)
+ if err != nil {
+ return err
+ }
+ svc.SetGate(classes.MaterialSearchGate())
+ p.materialIndex = classes.NewMaterialIndexer(app)
+ return nil
+}