feat: search materials through beachcomber

- Material is a beachcomber Searchable (index jz_bm_materials) in the
  hand-written models/material_search.go; jz.bm requires summercms v0.6.4
- classes.MaterialIndexer asks Typesense for up to 250 candidates with
  body highlights; the app-wide Gate is on only for jz.bm-marked contexts
- GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and
  paged in SQL in rank order, each row with at most one plain snippet
- search failure answers 503 Search is unavailable.; without q the list
  is unchanged

classes/material_index.go

@@ -0,0 +1,256 @@
+package classes
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+	"git.golem15.com/golem15/summercms/modules/diary"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+	"gorm.io/gorm"
+)
+
+// materialSearchKey marks a context whose syncs belong to jz.bm.
+type materialSearchKey struct{}
+
+// MarkMaterialSearch returns ctx marked as a jz.bm material sync. The
+// app-wide beachcomber Gate (MaterialSearchGate) is on only for a marked
+// context, so every jz.bm Sync or Remove passes one.
+func MarkMaterialSearch(ctx context.Context) context.Context {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return context.WithValue(ctx, materialSearchKey{}, true)
+}
+
+// MaterialSearchGate is the app-wide beachcomber Gate jz.bm installs. The
+// Gate sees only (ctx, db), not the model, so it is on only for a context
+// marked by MarkMaterialSearch: golem15.chatchannels syncs its messages
+// with unmarked contexts, and chat messages and DMs never reach the index.
+// The automatic callbacks of material writes are off as well; every jz.bm
+// write path syncs explicitly.
+func MaterialSearchGate() beachcomber.Gate {
+	return beachcomber.GateFunc(func(ctx context.Context, _ *gorm.DB) bool {
+		return ctx != nil && ctx.Value(materialSearchKey{}) != nil
+	})
+}
+
+// MaterialSearchWindow is how many engine candidates one search re-gates in
+// SQL. The list counts and pages over the gated rows, so a page never
+// depends on how the engine paged.
+const MaterialSearchWindow = typesense.MaxPerPage
+
+// MaterialQueryBy are the document fields a query is matched against, and
+// MaterialQueryWeights their weights in the same order: the title over the
+// tags, the source and the course, then the body, the link and file text.
+var (
+	MaterialQueryBy      = []string{"title", "tags", "source", "course", "url", "body", "file_texts"}
+	MaterialQueryWeights = []int{6, 4, 3, 3, 1, 2, 1}
+)
+
+// materialHighlightFields are the fields snippets are cut from.
+var materialHighlightFields = []string{"body", "file_texts"}
+
+// ErrSearchUnavailable is returned by Search when the engine is not
+// configured or the search failed. The API answers it with a 503.
+var ErrSearchUnavailable = errors.New("jz.bm: search is unavailable")
+
+// MaterialSearch is one material search: the query text and the validated
+// filters of the list.
+type MaterialSearch struct {
+	Q         string
+	CourseID  uint
+	MeetingID uint
+	Type      string
+	Tag       string
+}
+
+// MaterialCandidates are the engine's answer: the candidate ids in rank
+// order and the Hit of each. Both are candidates only: the caller re-gates
+// the ids in SQL and reads the Hit of a row only once SQL returned it.
+// Highlights reports whether the engine was asked for snippets.
+type MaterialCandidates struct {
+	IDs        []uint
+	Hits       map[uint]beachcomber.Hit
+	Highlights bool
+}
+
+// MaterialIndexer searches the materials index and keeps it current. A nil
+// *MaterialIndexer is valid: Search answers ErrSearchUnavailable and the
+// syncs do nothing.
+type MaterialIndexer struct {
+	// Service returns the app's beachcomber service, or nil.
+	Service func() *beachcomber.Service
+	// Log is where failures are logged; nil means slog.Default.
+	Log *slog.Logger
+}
+
+// NewMaterialIndexer returns the indexer of app. The service is resolved on
+// each call, so it may be built after the indexer.
+func NewMaterialIndexer(app *backpack.App) *MaterialIndexer {
+	return &MaterialIndexer{
+		Service: func() *beachcomber.Service {
+			if app == nil {
+				return nil
+			}
+			svc, err := beachcomber.From(app)
+			if err != nil {
+				return nil
+			}
+			return svc
+		},
+		Log: diary.For("jz.bm"),
+	}
+}
+
+func (ix *MaterialIndexer) service() *beachcomber.Service {
+	if ix == nil || ix.Service == nil {
+		return nil
+	}
+	return ix.Service()
+}
+
+func (ix *MaterialIndexer) log() *slog.Logger {
+	if ix == nil || ix.Log == nil {
+		return slog.Default()
+	}
+	return ix.Log
+}
+
+// Search asks the engine for up to MaterialSearchWindow candidates of req.
+// It answers ErrSearchUnavailable (wrapping the cause) when the service or
+// its engine is missing, the engine is not configured or the call fails.
+// The query text is never logged.
+func (ix *MaterialIndexer) Search(ctx context.Context, req MaterialSearch) (MaterialCandidates, error) {
+	svc := ix.service()
+	if svc == nil {
+		return MaterialCandidates{}, fmt.Errorf("%w: no search service", ErrSearchUnavailable)
+	}
+	engine := svc.Engine()
+	if engine == nil || !engine.Configured() {
+		return MaterialCandidates{}, fmt.Errorf("%w: the search engine is not configured", ErrSearchUnavailable)
+	}
+	q := beachcomber.Query{
+		Q:              req.Q,
+		QueryBy:        MaterialQueryBy,
+		QueryByWeights: MaterialQueryWeights,
+		FilterBy:       materialFilterBy(req),
+		SortBy:         "_text_match:desc,published_at:desc",
+		Page:           1,
+		PerPage:        MaterialSearchWindow,
+		ExcludeFields:  []string{"body", "file_texts"},
+	}
+	highlights := beachcomber.SupportsHighlights(engine)
+	if highlights {
+		q.Highlight = &beachcomber.Highlight{Fields: materialHighlightFields, AffixTokens: 12, MaxSnippetsPerArray: 1}
+	}
+	res, err := beachcomber.SearchPage(ctx, engine, svc.IndexName(&models.Material{}), q)
+	if err != nil {
+		ix.log().Warn("jz.bm: material search failed", "err", err)
+		return MaterialCandidates{}, fmt.Errorf("%w: %w", ErrSearchUnavailable, err)
+	}
+	out := MaterialCandidates{IDs: make([]uint, 0, len(res.IDs)), Hits: make(map[uint]beachcomber.Hit, len(res.IDs)), Highlights: highlights}
+	for i, raw := range res.IDs {
+		n, err := strconv.ParseUint(raw, 10, 64)
+		if err != nil || n == 0 {
+			continue
+		}
+		id := uint(n)
+		if _, seen := out.Hits[id]; seen {
+			continue
+		}
+		out.IDs = append(out.IDs, id)
+		hit := beachcomber.Hit{ID: raw}
+		if i < len(res.Hits) {
+			hit = res.Hits[i]
+		}
+		out.Hits[id] = hit
+	}
+	return out, nil
+}
+
+// materialTagSlug is the shape of a tag slug the engine filter may carry.
+var materialTagSlug = regexp.MustCompile(`^[a-z0-9-]{1,100}$`)
+
+// materialFilterBy narrows the engine's candidates to the list filters. It
+// is built only from parsed integers, an enumerated type and a tag that is
+// a plain slug; any other value is left out and applied by SQL alone, which
+// applies every filter anyway.
+func materialFilterBy(req MaterialSearch) string {
+	var parts []string
+	if req.CourseID != 0 {
+		parts = append(parts, "course_id:="+strconv.FormatUint(uint64(req.CourseID), 10))
+	}
+	if req.MeetingID != 0 {
+		parts = append(parts, "meeting_id:="+strconv.FormatUint(uint64(req.MeetingID), 10))
+	}
+	if slices.Contains(models.MaterialTypes, req.Type) {
+		parts = append(parts, "type:=`"+req.Type+"`")
+	}
+	if materialTagSlug.MatchString(req.Tag) {
+		parts = append(parts, "tag_slugs:=`"+req.Tag+"`")
+	}
+	return strings.Join(parts, " && ")
+}
+
+// Sync upserts the document of material id now. A failure is logged with
+// the id and never returned: a write never fails because of the index.
+func (ix *MaterialIndexer) Sync(ctx context.Context, db *gorm.DB, id uint) {
+	ix.run(ctx, db, id, false)
+}
+
+// Remove deletes the document of material id now, logging a failure like
+// Sync.
+func (ix *MaterialIndexer) Remove(ctx context.Context, db *gorm.DB, id uint) {
+	ix.run(ctx, db, id, true)
+}
+
+// SyncAfterCommit runs Sync once tx commits; with a nil tx, or outside a
+// transaction, it runs at once.
+func (ix *MaterialIndexer) SyncAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
+	ix.afterCommit(ctx, tx, id, false)
+}
+
+// RemoveAfterCommit runs Remove once tx commits, like SyncAfterCommit.
+func (ix *MaterialIndexer) RemoveAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
+	ix.afterCommit(ctx, tx, id, true)
+}
+
+func (ix *MaterialIndexer) afterCommit(ctx context.Context, tx *gorm.DB, id uint, remove bool) {
+	if ix == nil || id == 0 {
+		return
+	}
+	if tx == nil {
+		ix.run(ctx, nil, id, remove)
+		return
+	}
+	lagoon.AfterCommit(ctx, tx, func(ctx context.Context, db *gorm.DB) {
+		ix.run(ctx, db, id, remove)
+	})
+}
+
+func (ix *MaterialIndexer) run(ctx context.Context, db *gorm.DB, id uint, remove bool) {
+	svc := ix.service()
+	if svc == nil || id == 0 {
+		return
+	}
+	ctx = MarkMaterialSearch(ctx)
+	var err error
+	if remove {
+		err = svc.Remove(ctx, db, &models.Material{ID: id})
+	} else {
+		err = svc.Sync(ctx, db, &models.Material{ID: id})
+	}
+	if err != nil {
+		ix.log().Warn("jz.bm: material index sync failed", "material_id", id, "remove", remove, "err", err)
+	}
+}

classes/material_index_test.go

@@ -0,0 +1,121 @@
+package classes_test
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"slices"
+	"strings"
+	"testing"
+
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+)
+
+func TestMaterialSearchGate(t *testing.T) {
+	gate := classes.MaterialSearchGate()
+	if !gate.Enabled(classes.MarkMaterialSearch(context.Background()), nil) {
+		t.Error("a marked context is off")
+	}
+	if gate.Enabled(context.Background(), nil) {
+		t.Error("an unmarked context (a chat message sync) is on")
+	}
+}
+
+func TestMaterialSearchRequest(t *testing.T) {
+	t.Run("unavailable", materialSearchUnavailable)
+	stub := &searchtest.Stub{Hits: []searchtest.Hit{
+		{ID: "12", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd + " dziś"},
+		{ID: "abc"},
+		{ID: "0"},
+		{ID: "5"},
+	}}
+	ix := classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...))
+	got, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "globalizacja", CourseID: 7, MeetingID: 3, Type: "pdf", Tag: "globalizacja"})
+	if err != nil {
+		t.Fatalf("Search: %v", err)
+	}
+	searches := stub.Searches()
+	if len(searches) != 1 {
+		t.Fatalf("searches = %d, want 1", len(searches))
+	}
+	q := searches[0].Query
+	want := map[string]string{
+		"q":                "globalizacja",
+		"query_by":         "title,tags,source,course,url,body,file_texts",
+		"query_by_weights": "6,4,3,3,1,2,1",
+		"filter_by":        "course_id:=7 && meeting_id:=3 && type:=`pdf` && tag_slugs:=`globalizacja`",
+		"sort_by":          "_text_match:desc,published_at:desc",
+		"page":             "1",
+		"per_page":         "250",
+		"exclude_fields":   "body,file_texts",
+		"highlight_fields": "body,file_texts",
+	}
+	for k, v := range want {
+		if q.Get(k) != v {
+			t.Errorf("%s = %q, want %q", k, q.Get(k), v)
+		}
+	}
+	if !slices.Equal(got.IDs, []uint{12, 5}) {
+		t.Errorf("ids = %v, want [12 5]", got.IDs)
+	}
+	if !got.Highlights {
+		t.Error("Highlights is false against the typesense engine")
+	}
+	if h := got.Hits[12]; len(h.Highlights) != 1 || h.Highlights[0].Field != "body" {
+		t.Errorf("hit 12 = %+v", h)
+	} else if segs := h.Highlights[0].Segments; len(segs) != 3 || segs[1].Text != "globalizacji" || !segs[1].Match {
+		t.Errorf("hit 12 segments = %+v", segs)
+	}
+	if _, ok := got.Hits[5]; !ok {
+		t.Error("hit 5 is missing")
+	}
+	if len(got.Hits) != 2 {
+		t.Errorf("hits = %v, want only 12 and 5", got.Hits)
+	}
+
+	// A tag that is not a plain slug and an unknown type stay out of the
+	// engine filter; SQL applies them.
+	if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "x`y", Type: "exe"}); err != nil {
+		t.Fatal(err)
+	}
+	searches = stub.Searches()
+	if f := searches[len(searches)-1].Query.Get("filter_by"); f != "" {
+		t.Errorf("filter_by = %q, want empty", f)
+	}
+	if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "zła-Wielka"}); err != nil {
+		t.Fatal(err)
+	}
+	searches = stub.Searches()
+	if f := searches[len(searches)-1].Query.Get("filter_by"); strings.Contains(f, "tag_slugs") {
+		t.Errorf("filter_by = %q carries a tag that is not a slug", f)
+	}
+}
+
+// materialSearchUnavailable runs under TestMaterialSearchRequest.
+func materialSearchUnavailable(t *testing.T) {
+	// No key: the engine is not configured and nothing is sent.
+	stub := &searchtest.Stub{}
+	env := searchtest.Start(t, stub)
+	noKey := slices.DeleteFunc(slices.Clone(env), func(s string) bool { return strings.Contains(s, "API_KEY") })
+	ix := classes.NewMaterialIndexer(searchtest.App(t, noKey...))
+	if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+		t.Errorf("unconfigured engine: err = %v, want ErrSearchUnavailable", err)
+	}
+	if n := len(stub.Requests()); n != 0 {
+		t.Errorf("unconfigured engine sent %d requests", n)
+	}
+
+	failing := &searchtest.Stub{Status: http.StatusInternalServerError}
+	ix = classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, failing)...))
+	if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+		t.Errorf("failing engine: err = %v, want ErrSearchUnavailable", err)
+	}
+
+	var none *classes.MaterialIndexer
+	if _, err := none.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
+		t.Errorf("nil indexer: err = %v, want ErrSearchUnavailable", err)
+	}
+	none.Sync(context.Background(), nil, 1)
+	none.RemoveAfterCommit(context.Background(), nil, 1)
+}

controllers/api/api.go

@@ -17,6 +17,7 @@ import (
 	"git.golem15.com/golem15/summercms/modules/cooler"
 	"git.golem15.com/golem15/summercms/modules/phrasebook"
 	"git.golem15.com/golem15/summercms/modules/wire"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
 	"gocloud.dev/blob"
 	"gorm.io/gorm"
 )
@@ -73,6 +74,10 @@ type Handlers struct {
 	// Cache returns the application cache that holds the issued upload
 	// tickets until they are used, or nil while none is available.
 	Cache func() cooler.Cache
+	// MaterialIndex searches the materials index and keeps it current. Nil
+	// means a search (q) answers 503 and the syncs after a write are
+	// skipped.
+	MaterialIndex *classes.MaterialIndexer
 }
 
 func (h *Handlers) now() time.Time {

controllers/api/export_test.go

@@ -2,3 +2,11 @@ package api
 
 // DecodeInline exposes decodeInline to the api_test package.
 var DecodeInline = decodeInline
+
+// PickSnippet, FlattenSnippet and RankOrder expose the search helpers to the
+// api_test package.
+var (
+	PickSnippet    = pickSnippet
+	FlattenSnippet = flattenSnippet
+	RankOrder      = rankOrder
+)

controllers/api/handlers_test.go

@@ -152,6 +152,7 @@ func TestQueryValidation(t *testing.T) {
 		{"materials course negative", h.Materials, "/?course=-1", "course"},
 		{"materials meeting zero", h.Materials, "/?meeting=0", "meeting"},
 		{"materials tag too long", h.Materials, "/?tag=" + longTag(), "tag"},
+		{"materials q too long", h.Materials, "/?q=" + longTag(), "q"},
 	}
 	for _, tc := range cases {
 		code, body := serve(t, tc.fn, http.MethodGet, tc.target, user, nil)

controllers/api/json.go

@@ -213,6 +213,10 @@ type materialListJSON struct {
 	Tags        []tagJSON  `json:"tags"`
 	FilesCount  int64      `json:"files_count"`
 	IsPrivate   bool       `json:"is_private"`
+	// Snippet is the matched passage of a search result (*snippetJSON, or
+	// a plain string through MCP). It is nil, and the key absent, without
+	// q or without a highlight.
+	Snippet any `json:"snippet,omitempty"`
 }
 
 func newMaterialListJSON(m models.Material, files int64) materialListJSON {

controllers/api/material_search.go

@@ -0,0 +1,123 @@
+package api
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/wire"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+)
+
+// Material search: the engine (beachcomber, Typesense) answers candidate ids
+// only. Visibility, the filters, the total and the paging stay in SQL, and a
+// row's snippet is read only once SQL returned that row, so a material the
+// caller may not see never reaches the JSON, its count or a snippet.
+
+// minSearchRunes is the shortest query that searches; a shorter one is
+// ignored, as the student app does.
+const minSearchRunes = 2
+
+// searchQuery returns the trimmed q of in, or "" when it is shorter than
+// minSearchRunes.
+func searchQuery(in map[string]any) string {
+	q := strings.TrimSpace(str(in, "q"))
+	if utf8.RuneCountInString(q) < minSearchRunes {
+		return ""
+	}
+	return q
+}
+
+// writeSearchUnavailable answers a search the engine cannot serve: it is not
+// configured or the call failed. The list without q keeps working.
+func writeSearchUnavailable(w http.ResponseWriter) {
+	wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]any{"error": true, "message": "Search is unavailable."})
+}
+
+// rankOrder orders rows by their position in ids, the engine's rank. The ids
+// are parsed integers, so the array literal carries no caller text.
+func rankOrder(ids []uint) string {
+	parts := make([]string, len(ids))
+	for i, id := range ids {
+		parts[i] = strconv.FormatUint(uint64(id), 10)
+	}
+	return "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC"
+}
+
+// segmentJSON is one run of a snippet: the engine's plain text, verbatim,
+// and whether it matched the query. It is never markup; the client renders
+// it as text.
+type segmentJSON struct {
+	Text string `json:"text"`
+	Hit  bool   `json:"hit"`
+}
+
+// snippetJSON is the one excerpt a search result shows. Source is "body" or
+// "file" (FileName set); Lead and Trail say whether text was cut before or
+// after the excerpt.
+type snippetJSON struct {
+	Source   string        `json:"source"`
+	FileName *string       `json:"file_name"`
+	Lead     bool          `json:"lead"`
+	Trail    bool          `json:"trail"`
+	Segments []segmentJSON `json:"segments"`
+}
+
+// pickSnippet returns the snippet of m from hit: the first body snippet,
+// mapped one to one. It is nil when the hit has none. Call it only for a
+// row the SQL re-gate returned.
+func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON {
+	for _, sn := range hit.Highlights {
+		if sn.Field != "body" || len(sn.Segments) == 0 {
+			continue
+		}
+		out := &snippetJSON{Source: "body", Segments: make([]segmentJSON, 0, len(sn.Segments))}
+		var joined strings.Builder
+		for _, seg := range sn.Segments {
+			out.Segments = append(out.Segments, segmentJSON{Text: seg.Text, Hit: seg.Match})
+			joined.WriteString(seg.Text)
+		}
+		body := ""
+		if m.Body != nil {
+			body = models.MaterialPlainBody(*m.Body)
+		}
+		text := strings.TrimSpace(joined.String())
+		out.Lead = !strings.HasPrefix(body, text)
+		out.Trail = !strings.HasSuffix(body, text)
+		return out
+	}
+	return nil
+}
+
+// flattenSnippet is a snippet as plain text, for MCP: the segment texts
+// joined, with an ellipsis where text was cut.
+func flattenSnippet(s *snippetJSON) string {
+	if s == nil {
+		return ""
+	}
+	var b strings.Builder
+	if s.Lead {
+		b.WriteString("…")
+	}
+	for _, seg := range s.Segments {
+		b.WriteString(seg.Text)
+	}
+	if s.Trail {
+		b.WriteString("…")
+	}
+	return b.String()
+}
+
+// snippetValue is the snippet key of a row: the struct, a plain string
+// through MCP, or nil when there is none (the key is then left out).
+func snippetValue(s *snippetJSON, mcp bool) any {
+	if s == nil {
+		return nil
+	}
+	if mcp {
+		return flattenSnippet(s)
+	}
+	return s
+}

controllers/api/material_search_test.go

@@ -0,0 +1,200 @@
+package api_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/bouncer"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/controllers/api"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func segments(parts ...any) []beachcomber.Segment {
+	out := make([]beachcomber.Segment, 0, len(parts)/2)
+	for i := 0; i+1 < len(parts); i += 2 {
+		out = append(out, beachcomber.Segment{Text: parts[i].(string), Match: parts[i+1].(bool)})
+	}
+	return out
+}
+
+func TestMaterialSnippet(t *testing.T) {
+	body := "# Wstęp\n\nNa początku. Proces **globalizacji** zmienia państwa. Koniec tekstu."
+	m := models.Material{ID: 3, Body: &body}
+	hit := beachcomber.Hit{ID: "3", Highlights: []beachcomber.Snippet{
+		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
+		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia <b>państwa</b>.", false)},
+	}}
+	s := api.PickSnippet(hit, m)
+	if s == nil {
+		t.Fatal("no snippet")
+	}
+	if s.Source != "body" || s.FileName != nil {
+		t.Errorf("source %q file %v, want body and no file", s.Source, s.FileName)
+	}
+	if len(s.Segments) != 3 || s.Segments[1].Text != "globalizacji" || !s.Segments[1].Hit || s.Segments[0].Hit {
+		t.Errorf("segments = %+v", s.Segments)
+	}
+	if s.Segments[2].Text != " zmienia <b>państwa</b>." {
+		t.Errorf("segment text was changed: %q", s.Segments[2].Text)
+	}
+	if !s.Lead || !s.Trail {
+		t.Errorf("lead %v trail %v, want both (cut inside the body)", s.Lead, s.Trail)
+	}
+	b, err := json.Marshal(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(b); !strings.Contains(got, `"source":"body","file_name":null,"lead":true,"trail":true,"segments":[{"text":"Proces ","hit":false}`) {
+		t.Errorf("JSON = %s", got)
+	}
+
+	// The whole body: neither lead nor trail.
+	short := "Krótka **notatka**"
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
+	if whole == nil || whole.Lead || whole.Trail {
+		t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
+	}
+	// Start of the body only: trail but no lead.
+	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
+	if start == nil || start.Lead || !start.Trail {
+		t.Errorf("start snippet = %+v, want trail only", start)
+	}
+
+	// A snippet on another field only, or none at all, gives no snippet.
+	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
+		t.Errorf("title-only hit gave %+v", got)
+	}
+	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
+		t.Errorf("hit without highlights gave %+v", got)
+	}
+
+	if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia <b>państwa</b>.…" {
+		t.Errorf("FlattenSnippet = %q", got)
+	}
+	if got := api.FlattenSnippet(nil); got != "" {
+		t.Errorf("FlattenSnippet(nil) = %q", got)
+	}
+	if got := api.RankOrder([]uint{9, 3}); got != "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" {
+		t.Errorf("RankOrder = %q", got)
+	}
+}
+
+func TestMaterialsSearchUnavailable(t *testing.T) {
+	h := noDB()
+	user := &bouncer.Principal{ID: 7}
+	code, body := serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+	if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." {
+		t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body)
+	}
+	// One rune is no search: the list answers as without q (no database).
+	code, body = serve(t, h.Materials, http.MethodGet, "/?q=%20g%20", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+		t.Errorf("one-rune q = %d %v, want the no-database answer", code, body)
+	}
+	code, body = serve(t, h.Materials, http.MethodGet, "/", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+		t.Errorf("no q = %d %v, want the no-database answer", code, body)
+	}
+	// An unconfigured engine is the same 503.
+	h.MaterialIndex = classes.NewMaterialIndexer(searchtest.App(t))
+	code, body = serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Search is unavailable." {
+		t.Errorf("q with a null engine = %d %v, want 503 Search is unavailable.", code, body)
+	}
+}
+
+// sqlRecorder is a dry-run postgres handle that records every query it
+// builds, with its arguments inlined.
+type sqlRecorder struct {
+	mu  sync.Mutex
+	sql []string
+}
+
+func (r *sqlRecorder) queries() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.sql...)
+}
+
+func dryDB(t *testing.T) (*gorm.DB, *sqlRecorder) {
+	t.Helper()
+	db, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := &sqlRecorder{}
+	err = db.Callback().Query().After("gorm:query").Register("test:record_sql", func(tx *gorm.DB) {
+		rec.mu.Lock()
+		defer rec.mu.Unlock()
+		rec.sql = append(rec.sql, tx.Dialector.Explain(tx.Statement.SQL.String(), tx.Statement.Vars...))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db, rec
+}
+
+func TestMaterialsSearchTrace(t *testing.T) {
+	stub := &searchtest.Stub{Hits: []searchtest.Hit{
+		{ID: "9"},
+		{ID: "3", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd},
+	}}
+	db, rec := dryDB(t)
+	h := &api.Handlers{
+		DB:            func() *gorm.DB { return db },
+		MaterialIndex: classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)),
+	}
+	req := httptest.NewRequest(http.MethodGet, "/materials?q=globalizacja&course=7", nil)
+	req = req.WithContext(bouncer.WithUser(req.Context(), &bouncer.Principal{ID: 5}))
+	w := httptest.NewRecorder()
+	h.Materials(w, req)
+	if w.Code != http.StatusOK {
+		t.Fatalf("status %d: %s", w.Code, w.Body.String())
+	}
+	if n := len(stub.Searches()); n != 1 {
+		t.Errorf("searches = %d, want 1", n)
+	}
+	var count, page string
+	for _, q := range rec.queries() {
+		switch {
+		case strings.HasPrefix(q, "SELECT count(*)"):
+			count = q
+		case strings.HasPrefix(q, "SELECT") && strings.Contains(q, `FROM "jz_bm_materials"`):
+			page = q
+		}
+	}
+	if count == "" || page == "" {
+		t.Fatalf("queries = %q, want a count and a page", rec.queries())
+	}
+	for name, q := range map[string]string{"count": count, "page": page} {
+		for _, want := range []string{"jz_bm_materials.is_private = FALSE", "jz_bm_materials.published_at IS NOT NULL", "jz_bm_materials.course_id = 7", "jz_bm_materials.id IN (9,3)"} {
+			if !strings.Contains(q, want) {
+				t.Errorf("%s query lacks %q:\n%s", name, want, q)
+			}
+		}
+	}
+	if !strings.Contains(page, "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id)") {
+		t.Errorf("page query is not in rank order:\n%s", page)
+	}
+	var out struct {
+		Data []map[string]any `json:"data"`
+		Meta struct {
+			Total int `json:"total"`
+		} `json:"meta"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	if out.Meta.Total != 0 || len(out.Data) != 0 {
+		t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data)
+	}
+}

controllers/api/materials.go

@@ -6,9 +6,11 @@ import (
 	"strings"
 	"time"
 
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
 	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/lagoon/attach"
 	"git.golem15.com/golem15/summercms/modules/surf"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
 	"git.golem15.com/jakub/sm-bm-plugin/models"
 	"gorm.io/gorm"
 )
@@ -49,23 +51,35 @@ func preloadMaterial(q *gorm.DB) *gorm.DB {
 }
 
 // listMaterials returns one page of the published materials userID may
-// see, newest first.
+// see. With ids nil the page is newest first. With ids non-nil (search
-func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, page, perPage int) ([]materialListJSON, int64, error) {
+// candidates in the engine's rank order) both the count and the page are
+// also limited to ids and the page follows their order; the ids never widen
+// what visibleMaterials and the filter allow. An empty ids is an empty page.
+func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, ids []uint, page, perPage int) ([]materialListJSON, []models.Material, int64, error) {
+	if ids != nil && len(ids) == 0 {
+		return []materialListJSON{}, nil, 0, nil
+	}
 	q := f.apply(visibleMaterials(db.Model(&models.Material{}), now, userID))
+	if ids != nil {
+		q = q.Where("jz_bm_materials.id IN ?", ids)
+	}
 	var total int64
 	if err := q.Count(&total).Error; err != nil {
-		return nil, 0, err
+		return nil, nil, 0, err
+	}
+	rowsQ := preloadMaterial(f.apply(visibleMaterials(db, now, userID)))
+	if ids != nil {
+		rowsQ = rowsQ.Where("jz_bm_materials.id IN ?", ids).Order(rankOrder(ids))
+	} else {
+		rowsQ = rowsQ.Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC")
 	}
 	var rows []models.Material
-	err := preloadMaterial(f.apply(visibleMaterials(db, now, userID))).
+	err := rowsQ.Offset((page - 1) * perPage).Limit(perPage).Find(&rows).Error
-		Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC").
-		Offset((page - 1) * perPage).Limit(perPage).
-		Find(&rows).Error
 	if err != nil {
-		return nil, 0, err
+		return nil, nil, 0, err
 	}
 	out, err := materialsJSON(db, rows)
-	return out, total, err
+	return out, rows, total, err
 }
 
 // materialsJSON builds list rows with their file counts.
@@ -99,31 +113,69 @@ func materialsJSON(db *gorm.DB, rows []models.Material) ([]materialListJSON, err
 	return out, nil
 }
 
-// Materials answers GET /materials?course=&meeting=&tag=&type=&page=&per_page=:
+// Materials answers GET /materials?q=&course=&meeting=&tag=&type=&page=&per_page=:
 // a page of the published materials the caller may see, in the {data, meta}
-// shape of lagoon.Paginate.
+// shape of lagoon.Paginate. With q (2 to 100 characters) the engine answers
+// candidates, which SQL re-gates, counts and pages in rank order, each row
+// with at most one snippet; when the engine cannot answer the request is a
+// 503. Without q the list is newest first and never calls the engine.
 func (h *Handlers) Materials(w http.ResponseWriter, r *http.Request) {
 	userID, ok := h.caller(w, r)
 	if !ok {
 		return
 	}
-	in := query(r, "course", "meeting", "tag", "type", "page", "per_page")
+	in := query(r, "q", "course", "meeting", "tag", "type", "page", "per_page")
 	if !h.validate(w, r, in, materialsRules) {
 		return
 	}
+	filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
+	var cands *classes.MaterialCandidates
+	if q := searchQuery(in); q != "" {
+		c, err := h.MaterialIndex.Search(r.Context(), classes.MaterialSearch{Q: q, CourseID: filter.CourseID, MeetingID: filter.MeetingID, Type: filter.Type, Tag: filter.Tag})
+		if err != nil {
+			writeSearchUnavailable(w)
+			return
+		}
+		cands = &c
+	}
 	db, ok := h.db(w, r)
 	if !ok {
 		return
 	}
-	filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
 	page := min(intParam(in, "page", 1), 100000)
 	perPage := intParam(in, "per_page", h.perPage())
-	rows, total, err := listMaterials(db, h.now(), userID, filter, page, perPage)
+	var ids []uint
+	if cands != nil {
+		ids = cands.IDs
+		if ids == nil {
+			ids = []uint{}
+		}
+	}
+	out, rows, total, err := listMaterials(db, h.now(), userID, filter, ids, page, perPage)
 	if err != nil {
 		writeOpaque500(w)
 		return
 	}
-	writeJSONPage(w, lagoon.Paginate(rows, page, perPage, total))
+	if cands != nil {
+		attachSnippets(out, rows, cands.Hits, viaMCP(r))
+	}
+	writeJSONPage(w, lagoon.Paginate(out, page, perPage, total))
+}
+
+// attachSnippets sets the snippet of each list row from its Hit. rows are
+// the rows SQL returned (they passed the re-gate), in the order of out; the
+// Hit of an id SQL did not return is never read.
+func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, mcp bool) {
+	for i := range out {
+		if i >= len(rows) || rows[i].ID != out[i].ID {
+			continue
+		}
+		hit, ok := hits[rows[i].ID]
+		if !ok {
+			continue
+		}
+		out[i].Snippet = snippetValue(pickSnippet(hit, rows[i]), mcp)
+	}
 }
 
 // Material answers GET /materials/{id}: one published material the caller

controllers/api/params.go

@@ -37,6 +37,7 @@ var (
 		{Field: "type", Rules: append(lagoon.ParseRules("nullable"), lagoon.In(models.MaterialTypes...))},
 		{Field: "page", Rules: lagoon.ParseRules("nullable|integer|min:1")},
 		{Field: "per_page", Rules: lagoon.ParseRules("nullable|integer|between:1,100")},
+		{Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")},
 	}
 )
 

go.mod

@@ -6,19 +6,19 @@ require (
 	git.golem15.com/golem15/sm-chat-plugin v0.1.3
 	git.golem15.com/golem15/sm-chatchannels-plugin v0.0.0
 	git.golem15.com/golem15/sm-user-plugin v0.1.2
-	git.golem15.com/golem15/summercms v0.5.0
+	git.golem15.com/golem15/summercms v0.6.4
 	github.com/go-gormigrate/gormigrate/v2 v2.1.7
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/testcontainers/testcontainers-go v0.44.0
 	github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0
 	gocloud.dev v0.46.0
+	gorm.io/driver/postgres v1.6.3
 	gorm.io/gorm v1.31.2
 )
 
 require (
 	charm.land/bubbles/v2 v2.2.1 // indirect
 	charm.land/bubbletea/v2 v2.1.0 // indirect
-	charm.land/fang/v2 v2.0.1 // indirect
 	charm.land/huh/v2 v2.0.3 // indirect
 	charm.land/lipgloss/v2 v2.0.6 // indirect
 	dario.cat/mergo v1.0.2 // indirect
@@ -52,7 +52,6 @@ require (
 	github.com/charmbracelet/harmonica v0.2.0 // indirect
 	github.com/charmbracelet/ultraviolet v0.0.0-20261008173134-6b8d4baf91b4 // indirect
 	github.com/charmbracelet/x/ansi v0.11.9 // indirect
-	github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 // indirect
 	github.com/charmbracelet/x/exp/ordered v0.1.0 // indirect
 	github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
 	github.com/charmbracelet/x/term v0.2.2 // indirect
@@ -123,10 +122,6 @@ require (
 	github.com/moby/term v0.5.2 // indirect
 	github.com/modelcontextprotocol/go-sdk v1.8.0 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
-	github.com/muesli/mango v0.1.0 // indirect
-	github.com/muesli/mango-cobra v1.2.0 // indirect
-	github.com/muesli/mango-pflag v0.1.0 // indirect
-	github.com/muesli/roff v0.1.0 // indirect
 	github.com/nicksnyder/go-i18n/v2 v2.6.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
@@ -180,7 +175,6 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
-	gorm.io/driver/postgres v1.6.3 // indirect
 )
 
 replace git.golem15.com/golem15/summercms => ../../../../summercms.go

go.sum

@@ -4,8 +4,6 @@ charm.land/bubbles/v2 v2.2.1 h1:Fq1+qm5hV6GkvzLQDhCBpXXE5tLgvh1PRriCLwSvIQU=
 charm.land/bubbles/v2 v2.2.1/go.mod h1:wdMgn+sje1KNXdwFizIWjbf328fIUBxqEmJ/vYPo8yc=
 charm.land/bubbletea/v2 v2.1.0 h1:A9fj9h5LFrvQhQHFNggp8K8rvY6CUfgn9dMbb6p2HIw=
 charm.land/bubbletea/v2 v2.1.0/go.mod h1:xWHeyxIX9/wiFU2kgPW04ojOZyTzrWf+8lGjUCaftiU=
-charm.land/fang/v2 v2.0.1 h1:zQCM8JQJ1JnQX/66B5jlCYBUxL2as5JXQZ2KJ6EL0mY=
-charm.land/fang/v2 v2.0.1/go.mod h1:S1GmkpcvK+OB5w9caywUnJcsMew45Ot8FXqoz8ALrII=
 charm.land/huh/v2 v2.0.3 h1:2cJsMqEPwSywGHvdlKsJyQKPtSJLVnFKyFbsYZTlLkU=
 charm.land/huh/v2 v2.0.3/go.mod h1:93eEveeeqn47MwiC3tf+2atZ2l7Is88rAtmZNZ8x9Wc=
 charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ=
@@ -110,8 +108,6 @@ github.com/charmbracelet/x/conpty v0.1.1 h1:s1bUxjoi7EpqiXysVtC+a8RrvPPNcNvAjfi4
 github.com/charmbracelet/x/conpty v0.1.1/go.mod h1:OmtR77VODEFbiTzGE9G1XiRJAga6011PIm4u5fTNZpk=
 github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA=
 github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0=
-github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444 h1:IJDiTgVE56gkAGfq0lBEloWgkXMk4hl/bmuPoicI4R0=
-github.com/charmbracelet/x/exp/charmtone v0.0.0-20250603201427-c31516f43444/go.mod h1:T9jr8CzFpjhFVHjNjKwbAD7KwBNyFnj2pntAO7F2zw0=
 github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA=
 github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I=
 github.com/charmbracelet/x/exp/ordered v0.1.0 h1:55/qLwjIh0gL0Vni+QAWk7T/qRVP6sBf+2agPBgnOFE=
@@ -299,14 +295,6 @@ github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtw
 github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts=
 github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
-github.com/muesli/mango v0.1.0 h1:DZQK45d2gGbql1arsYA4vfg4d7I9Hfx5rX/GCmzsAvI=
-github.com/muesli/mango v0.1.0/go.mod h1:5XFpbC8jY5UUv89YQciiXNlbi+iJgt29VDC5xbzrLL4=
-github.com/muesli/mango-cobra v1.2.0 h1:DQvjzAM0PMZr85Iv9LIMaYISpTOliMEg+uMFtNbYvWg=
-github.com/muesli/mango-cobra v1.2.0/go.mod h1:vMJL54QytZAJhCT13LPVDfkvCUJ5/4jNUKF/8NC2UjA=
-github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe7Sg=
-github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0=
-github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8=
-github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig=
 github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ=
 github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=

internal/searchtest/typesense.go

@@ -0,0 +1,198 @@
+// Package searchtest is a Typesense stand-in for the jz.bm tests: an
+// httptest server that answers the collection, import, delete and search
+// endpoints the beachcomber typesense engine calls, and an app configured to
+// talk to it. Only tests import it.
+package searchtest
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	_ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+	"git.golem15.com/golem15/summercms/modules/compass"
+)
+
+// Index is the collection of the materials index.
+const Index = "jz_bm_materials"
+
+// HighlightStart and HighlightEnd are the highlight tags the typesense
+// engine asks for (two private-use characters).
+const (
+	HighlightStart = ""
+	HighlightEnd   = ""
+)
+
+// Hit is one search answer: a document id and, optionally, the snippet of
+// one field with the matched words between HighlightStart and HighlightEnd.
+type Hit struct {
+	ID      string
+	Field   string
+	Snippet string
+}
+
+// Request is one request the stub received.
+type Request struct {
+	Method string
+	Path   string
+	Query  url.Values
+}
+
+// Stub stands in for a Typesense server with one collection.
+type Stub struct {
+	mu sync.Mutex
+	// Hits answer every search, in rank order.
+	Hits []Hit
+	// Status, when set, answers every request with that code.
+	Status int
+
+	exists   bool
+	requests []Request
+	imported []map[string]any
+	removed  []string
+}
+
+// ServeHTTP implements http.Handler.
+func (s *Stub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query()})
+	if s.Status != 0 {
+		w.WriteHeader(s.Status)
+		_, _ = io.WriteString(w, `{"message":"stub failure"}`)
+		return
+	}
+	coll := "/collections/" + Index
+	switch {
+	case r.Method == http.MethodGet && r.URL.Path == coll:
+		if !s.exists {
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = io.WriteString(w, `{"message":"Not Found"}`)
+			return
+		}
+		_, _ = io.WriteString(w, `{}`)
+	case r.Method == http.MethodPost && r.URL.Path == "/collections":
+		s.exists = true
+		w.WriteHeader(http.StatusCreated)
+		_, _ = io.WriteString(w, `{}`)
+	case r.Method == http.MethodDelete && r.URL.Path == coll:
+		if !s.exists {
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = io.WriteString(w, `{"message":"Not Found"}`)
+			return
+		}
+		s.exists = false
+		_, _ = io.WriteString(w, `{}`)
+	case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import":
+		s.exists = true
+		sc := bufio.NewScanner(r.Body)
+		sc.Buffer(make([]byte, 0, 64*1024), 16<<20)
+		var answer []string
+		for sc.Scan() {
+			var doc map[string]any
+			if err := json.Unmarshal(sc.Bytes(), &doc); err == nil {
+				s.imported = append(s.imported, doc)
+			}
+			answer = append(answer, `{"success":true}`)
+		}
+		_, _ = io.WriteString(w, strings.Join(answer, "\n"))
+	case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"):
+		s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/"))
+		_, _ = io.WriteString(w, `{}`)
+	case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search":
+		hits := make([]string, 0, len(s.Hits))
+		for _, h := range s.Hits {
+			id, _ := json.Marshal(h.ID)
+			highlights := "[]"
+			if h.Field != "" {
+				field, _ := json.Marshal(h.Field)
+				snippet, _ := json.Marshal(h.Snippet)
+				highlights = fmt.Sprintf(`[{"field":%s,"snippet":%s}]`, field, snippet)
+			}
+			hits = append(hits, fmt.Sprintf(`{"document":{"id":%s},"highlights":%s}`, id, highlights))
+		}
+		// found is deliberately larger than the hits, so a caller that
+		// exposed it would show.
+		_, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.Hits)+100, strings.Join(hits, ","))
+	default:
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = io.WriteString(w, `{"message":"Not Found"}`)
+	}
+}
+
+// Requests returns the requests received so far.
+func (s *Stub) Requests() []Request {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]Request(nil), s.requests...)
+}
+
+// Searches returns the search requests received so far.
+func (s *Stub) Searches() []Request {
+	var out []Request
+	for _, r := range s.Requests() {
+		if r.Method == http.MethodGet && strings.HasSuffix(r.Path, "/documents/search") {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+// Imported returns the documents upserted so far.
+func (s *Stub) Imported() []map[string]any {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]map[string]any(nil), s.imported...)
+}
+
+// Removed returns the document ids deleted so far.
+func (s *Stub) Removed() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.removed...)
+}
+
+// Start serves stub on an httptest server closed with the test, and returns
+// the SUMMER_SEARCH__* environment that points the typesense engine at it
+// with a test-only key.
+func Start(t testing.TB, stub *Stub) []string {
+	t.Helper()
+	srv := httptest.NewServer(stub)
+	t.Cleanup(srv.Close)
+	u, err := url.Parse(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return []string{
+		"SUMMER_SEARCH__DRIVER=typesense",
+		"SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key",
+		"SUMMER_SEARCH__TYPESENSE__HOST=" + u.Hostname(),
+		"SUMMER_SEARCH__TYPESENSE__PORT=" + u.Port(),
+		"SUMMER_SEARCH__TYPESENSE__PROTOCOL=" + u.Scheme,
+	}
+}
+
+// App returns a bare application whose config holds env (and SUMMER_ENV
+// testing) and an empty config directory.
+func App(t testing.TB, env ...string) *backpack.App {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: jz-bm-test\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := compass.Open(compass.Options{Dir: dir, Environ: append([]string{"SUMMER_ENV=testing"}, env...)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return backpack.New(cfg)
+}

models/material_search.go

@@ -0,0 +1,173 @@
+package models
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"gorm.io/gorm"
+)
+
+// A material is kept in the search index jz_bm_materials (beachcomber). The
+// index holds every material, drafts and private ones included: visibility
+// is enforced only in SQL, where the engine's ids are candidates that the
+// student queries re-gate. This file is hand-written; material.go is
+// generated.
+
+// SearchableAs is the index name without the search.prefix.
+func (Material) SearchableAs() string { return "jz_bm_materials" }
+
+// ShouldBeSearchable is true for every material: the index carries no
+// visibility, so drafts and private materials are indexed too.
+func (m *Material) ShouldBeSearchable() bool { return m != nil }
+
+// ToSearchableArray builds the document of the material. beachcomber reloads
+// the row by primary key without preloads, so the course, the meeting number
+// and the tags are loaded here.
+func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
+	db = db.WithContext(ctx)
+	var course *Course
+	if m.CourseID != nil {
+		var c Course
+		res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c)
+		if res.Error != nil {
+			return nil, res.Error
+		}
+		if res.RowsAffected > 0 {
+			course = &c
+		}
+	}
+	meetingNumber := 0
+	if m.MeetingID != nil {
+		var mt Meeting
+		res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt)
+		if res.Error != nil {
+			return nil, res.Error
+		}
+		if res.RowsAffected > 0 {
+			meetingNumber = mt.Number
+		}
+	}
+	var tags []Tag
+	err := db.Model(&Tag{}).
+		Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id").
+		Where("mt.material_id = ?", m.ID).
+		Order("jz_bm_tags.name").
+		Find(&tags).Error
+	if err != nil {
+		return nil, err
+	}
+	return MaterialDocument(*m, course, meetingNumber, tags, nil), nil
+}
+
+// MaterialDocument is the search document of m with its course, meeting
+// number (0 when none), tags in order and the texts of its attached files.
+// Absent numbers are 0, so filters and the sort tiebreak never see a
+// missing value.
+func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any {
+	doc := map[string]any{
+		"id":           strconv.FormatUint(uint64(m.ID), 10),
+		"title":        m.Title,
+		"type":         m.Type,
+		"course_id":    int64(0),
+		"meeting_id":   int64(0),
+		"published_at": int64(0),
+	}
+	if m.Source != nil && strings.TrimSpace(*m.Source) != "" {
+		doc["source"] = *m.Source
+	}
+	if m.URL != nil && strings.TrimSpace(*m.URL) != "" {
+		doc["url"] = *m.URL
+	}
+	if m.CourseID != nil {
+		doc["course_id"] = int64(*m.CourseID)
+	}
+	if m.MeetingID != nil {
+		doc["meeting_id"] = int64(*m.MeetingID)
+	}
+	if m.PublishedAt != nil {
+		doc["published_at"] = m.PublishedAt.Unix()
+	}
+	if course != nil {
+		label := course.Name
+		if course.Code != nil && strings.TrimSpace(*course.Code) != "" {
+			label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code))
+		}
+		doc["course"] = label
+	}
+	if meetingNumber > 0 {
+		doc["meeting_number"] = meetingNumber
+	}
+	names := make([]string, 0, len(tags))
+	slugs := make([]string, 0, len(tags))
+	for _, t := range tags {
+		names = append(names, t.Name)
+		slugs = append(slugs, t.Slug)
+	}
+	doc["tags"] = names
+	doc["tag_slugs"] = slugs
+	if m.Body != nil {
+		if body := MaterialPlainBody(*m.Body); body != "" {
+			doc["body"] = body
+		}
+	}
+	if len(fileTexts) > 0 {
+		doc["file_texts"] = fileTexts
+	}
+	return doc
+}
+
+// SearchIndexSchema is the Typesense collection schema. It applies only when
+// the collection is created; bm:materials-reindex --fresh recreates it.
+// Every field is stored (a search leaves body and file_texts out of the
+// answer instead), and no field sets a locale: the default one folds
+// diacritics, so lodz finds Łódź.
+func (Material) SearchIndexSchema() map[string]any {
+	return map[string]any{
+		"fields": []map[string]any{
+			{"name": "id", "type": "string"},
+			{"name": "title", "type": "string"},
+			{"name": "source", "type": "string", "optional": true},
+			{"name": "url", "type": "string", "optional": true},
+			{"name": "tags", "type": "string[]", "optional": true},
+			{"name": "tag_slugs", "type": "string[]", "optional": true},
+			{"name": "course", "type": "string", "optional": true},
+			{"name": "course_id", "type": "int64"},
+			{"name": "meeting_id", "type": "int64"},
+			{"name": "meeting_number", "type": "int32", "optional": true},
+			{"name": "type", "type": "string"},
+			{"name": "published_at", "type": "int64"},
+			{"name": "body", "type": "string", "optional": true},
+			{"name": "file_texts", "type": "string[]", "optional": true},
+		},
+	}
+}
+
+var (
+	mdImage      = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
+	mdLink       = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
+	mdHeading    = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`)
+	mdQuote      = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`)
+	mdList       = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`)
+	mdEmphasis   = regexp.MustCompile(`\*+|~~`)
+	mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`)
+	mdUnderEnd   = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`)
+)
+
+// MaterialPlainBody drops the Markdown syntax of a material body (heading
+// hashes, blockquote and list markers, emphasis, inline code ticks, link
+// and image syntax down to their text) and collapses white space, so the
+// index holds and highlights only words.
+func MaterialPlainBody(body string) string {
+	s := strings.ReplaceAll(body, "`", "")
+	s = mdImage.ReplaceAllString(s, "$1")
+	s = mdLink.ReplaceAllString(s, "$1")
+	s = mdHeading.ReplaceAllString(s, "")
+	s = mdQuote.ReplaceAllString(s, "")
+	s = mdList.ReplaceAllString(s, "")
+	s = mdEmphasis.ReplaceAllString(s, "")
+	s = mdUnderStart.ReplaceAllString(s, "$1")
+	s = mdUnderEnd.ReplaceAllString(s, "$1")
+	return strings.Join(strings.Fields(s), " ")
+}

models/material_search_test.go

@@ -0,0 +1,124 @@
+package models_test
+
+import (
+	"slices"
+	"testing"
+	"time"
+
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+)
+
+func TestMaterialSearchSchema(t *testing.T) {
+	m := models.Material{}
+	if got := m.SearchableAs(); got != "jz_bm_materials" {
+		t.Errorf("SearchableAs = %q, want jz_bm_materials", got)
+	}
+	fields, ok := m.SearchIndexSchema()["fields"].([]map[string]any)
+	if !ok || len(fields) == 0 {
+		t.Fatalf("schema fields = %#v", m.SearchIndexSchema()["fields"])
+	}
+	byName := map[string]map[string]any{}
+	for _, f := range fields {
+		name, _ := f["name"].(string)
+		typ, _ := f["type"].(string)
+		if name == "" || typ == "" {
+			t.Errorf("field without a name or type: %v", f)
+		}
+		if _, ok := f["locale"]; ok {
+			t.Errorf("field %s sets a locale; the default one folds Polish diacritics", name)
+		}
+		if store, ok := f["store"]; ok && store == false {
+			t.Errorf("field %s turns storage off", name)
+		}
+		byName[name] = f
+	}
+	for _, name := range []string{"body", "file_texts"} {
+		f, ok := byName[name]
+		if !ok {
+			t.Errorf("schema has no %s field", name)
+			continue
+		}
+		if f["optional"] != true {
+			t.Errorf("%s is not optional", name)
+		}
+	}
+	if byName["body"]["type"] != "string" || byName["file_texts"]["type"] != "string[]" {
+		t.Errorf("body %v, file_texts %v: want string and string[]", byName["body"]["type"], byName["file_texts"]["type"])
+	}
+	for _, name := range []string{"id", "title", "source", "url", "tags", "tag_slugs", "course", "course_id", "meeting_id", "meeting_number", "type", "published_at"} {
+		if _, ok := byName[name]; !ok {
+			t.Errorf("schema has no %s field", name)
+		}
+	}
+	if !(&m).ShouldBeSearchable() {
+		t.Error("a material is not searchable")
+	}
+	var nilMaterial *models.Material
+	if nilMaterial.ShouldBeSearchable() {
+		t.Error("a nil material is searchable")
+	}
+}
+
+func TestMaterialDocument(t *testing.T) {
+	plain := models.MaterialPlainBody("# Nagłówek\n\nTekst **pogrubienie** i _kursywa_ oraz [tekst](https://x) i ![obraz](a.png).\n\n> cytat\n> dalej\n\n- punkt `kod` snake_case")
+	want := "Nagłówek Tekst pogrubienie i kursywa oraz tekst i obraz. cytat dalej punkt kod snake_case"
+	if plain != want {
+		t.Errorf("MaterialPlainBody =\n%q\nwant\n%q", plain, want)
+	}
+
+	code := "BM-12"
+	course := &models.Course{ID: 4, Name: "Bezpieczeństwo", Code: &code}
+	courseID, meetingID := uint(4), uint(9)
+	at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
+	body := "## Wstęp\n\n**Globalizacja** w praktyce"
+	source := "Giddens"
+	m := models.Material{ID: 12, Title: "Globalizacja", Type: "notatka", CourseID: &courseID, MeetingID: &meetingID, PublishedAt: &at, Body: &body, Source: &source}
+	tags := []models.Tag{{Name: "Globalizacja", Slug: "globalizacja"}, {Name: "Polityka", Slug: "polityka"}}
+	doc := models.MaterialDocument(m, course, 3, tags, []string{"tekst pliku"})
+	checks := map[string]any{
+		"id":             "12",
+		"title":          "Globalizacja",
+		"type":           "notatka",
+		"source":         "Giddens",
+		"course":         "Bezpieczeństwo BM-12",
+		"course_id":      int64(4),
+		"meeting_id":     int64(9),
+		"meeting_number": 3,
+		"published_at":   at.Unix(),
+		"body":           "Wstęp Globalizacja w praktyce",
+	}
+	for k, v := range checks {
+		if doc[k] != v {
+			t.Errorf("doc[%s] = %#v, want %#v", k, doc[k], v)
+		}
+	}
+	if got, _ := doc["tags"].([]string); !slices.Equal(got, []string{"Globalizacja", "Polityka"}) {
+		t.Errorf("tags = %v", doc["tags"])
+	}
+	if got, _ := doc["tag_slugs"].([]string); !slices.Equal(got, []string{"globalizacja", "polityka"}) {
+		t.Errorf("tag_slugs = %v", doc["tag_slugs"])
+	}
+	if got, _ := doc["file_texts"].([]string); !slices.Equal(got, []string{"tekst pliku"}) {
+		t.Errorf("file_texts = %v", doc["file_texts"])
+	}
+
+	// No code, no course id, no meeting, a draft, no files.
+	bare := models.MaterialDocument(models.Material{ID: 5, Title: "Szkic", Type: "inne"}, &models.Course{Name: "Teoria"}, 0, nil, nil)
+	if bare["course"] != "Teoria" {
+		t.Errorf("course without code = %v", bare["course"])
+	}
+	for _, k := range []string{"course_id", "meeting_id", "published_at"} {
+		if bare[k] != int64(0) {
+			t.Errorf("bare %s = %#v, want 0", k, bare[k])
+		}
+	}
+	for _, k := range []string{"meeting_number", "file_texts", "body", "source", "url"} {
+		if _, ok := bare[k]; ok {
+			t.Errorf("bare document carries %s", k)
+		}
+	}
+	none := models.MaterialDocument(models.Material{ID: 6, Title: "X", Type: "link"}, nil, 0, nil, nil)
+	if _, ok := none["course"]; ok {
+		t.Error("a material without a course carries course")
+	}
+}

plugin.go

@@ -59,6 +59,9 @@ type Plugin struct {
 	// chatMember is the chat-group check of the realtime access guard; nil
 	// means classes.IsChatMember. Tests inject their own.
 	chatMember classes.MemberFunc
+	// materialIndex searches the materials index and keeps it current. Nil
+	// when Boot did not run: then search answers 503 and syncs are skipped.
+	materialIndex *classes.MaterialIndexer
 }
 
 func (p *Plugin) ID() string { return "jz.bm" }
@@ -74,7 +77,8 @@ func (p *Plugin) Register(app *backpack.App) error {
 
 // Boot keeps the application, so route handlers and commands can reach its
 // services, such as the database, when they run, and wires the realtime
-// service and, when golem15.chat is active, the BM chat context. The
+// service, the materials search and, when golem15.chat is active, the BM
+// chat context. The
 // management API uses the shared user.api_token middleware owned by
 // golem15.user.
 func (p *Plugin) Boot(app *backpack.App) error {
@@ -83,6 +87,9 @@ func (p *Plugin) Boot(app *backpack.App) error {
 		if err := p.wireRealtime(app); err != nil {
 			return err
 		}
+		if err := p.wireSearch(app); err != nil {
+			return err
+		}
 		if err := p.registerChat(app); err != nil {
 			return err
 		}

routes.go

@@ -304,6 +304,7 @@ func (p *Plugin) mountRealtime(r pact.Router) error {
 // publishes it.
 func (p *Plugin) handlers() *api.Handlers {
 	h := &api.Handlers{
+		MaterialIndex: p.materialIndex,
 		DB: func() *gorm.DB {
 			if p.app == nil {
 				return nil

search.go

@@ -0,0 +1,23 @@
+package bm
+
+import (
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	_ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+)
+
+// wireSearch builds the app's beachcomber service (search.driver; the
+// typesense engine registers itself through the blank import), installs the
+// jz.bm-only Gate and the material indexer. The Gate is app-wide, so it is
+// on only for contexts jz.bm marks: golem15.chatchannels' message syncs stay
+// off and chat messages never reach the index.
+func (p *Plugin) wireSearch(app *backpack.App) error {
+	svc, err := beachcomber.From(app)
+	if err != nil {
+		return err
+	}
+	svc.SetGate(classes.MaterialSearchGate())
+	p.materialIndex = classes.NewMaterialIndexer(app)
+	return nil
+}