feat: search materials through beachcomber

- Material is a beachcomber Searchable (index jz_bm_materials) in the
  hand-written models/material_search.go; jz.bm requires summercms v0.6.4
- classes.MaterialIndexer asks Typesense for up to 250 candidates with
  body highlights; the app-wide Gate is on only for jz.bm-marked contexts
- GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and
  paged in SQL in rank order, each row with at most one plain snippet
- search failure answers 503 Search is unavailable.; without q the list
  is unchanged

controllers/api/api.go

@@ -17,6 +17,7 @@ import (
 	"git.golem15.com/golem15/summercms/modules/cooler"
 	"git.golem15.com/golem15/summercms/modules/phrasebook"
 	"git.golem15.com/golem15/summercms/modules/wire"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
 	"gocloud.dev/blob"
 	"gorm.io/gorm"
 )
@@ -73,6 +74,10 @@ type Handlers struct {
 	// Cache returns the application cache that holds the issued upload
 	// tickets until they are used, or nil while none is available.
 	Cache func() cooler.Cache
+	// MaterialIndex searches the materials index and keeps it current. Nil
+	// means a search (q) answers 503 and the syncs after a write are
+	// skipped.
+	MaterialIndex *classes.MaterialIndexer
 }
 
 func (h *Handlers) now() time.Time {

controllers/api/export_test.go

@@ -2,3 +2,11 @@ package api
 
 // DecodeInline exposes decodeInline to the api_test package.
 var DecodeInline = decodeInline
+
+// PickSnippet, FlattenSnippet and RankOrder expose the search helpers to the
+// api_test package.
+var (
+	PickSnippet    = pickSnippet
+	FlattenSnippet = flattenSnippet
+	RankOrder      = rankOrder
+)

controllers/api/handlers_test.go

@@ -152,6 +152,7 @@ func TestQueryValidation(t *testing.T) {
 		{"materials course negative", h.Materials, "/?course=-1", "course"},
 		{"materials meeting zero", h.Materials, "/?meeting=0", "meeting"},
 		{"materials tag too long", h.Materials, "/?tag=" + longTag(), "tag"},
+		{"materials q too long", h.Materials, "/?q=" + longTag(), "q"},
 	}
 	for _, tc := range cases {
 		code, body := serve(t, tc.fn, http.MethodGet, tc.target, user, nil)

controllers/api/json.go

@@ -213,6 +213,10 @@ type materialListJSON struct {
 	Tags        []tagJSON  `json:"tags"`
 	FilesCount  int64      `json:"files_count"`
 	IsPrivate   bool       `json:"is_private"`
+	// Snippet is the matched passage of a search result (*snippetJSON, or
+	// a plain string through MCP). It is nil, and the key absent, without
+	// q or without a highlight.
+	Snippet any `json:"snippet,omitempty"`
 }
 
 func newMaterialListJSON(m models.Material, files int64) materialListJSON {

controllers/api/material_search.go

@@ -0,0 +1,123 @@
+package api
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/wire"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+)
+
+// Material search: the engine (beachcomber, Typesense) answers candidate ids
+// only. Visibility, the filters, the total and the paging stay in SQL, and a
+// row's snippet is read only once SQL returned that row, so a material the
+// caller may not see never reaches the JSON, its count or a snippet.
+
+// minSearchRunes is the shortest query that searches; a shorter one is
+// ignored, as the student app does.
+const minSearchRunes = 2
+
+// searchQuery returns the trimmed q of in, or "" when it is shorter than
+// minSearchRunes.
+func searchQuery(in map[string]any) string {
+	q := strings.TrimSpace(str(in, "q"))
+	if utf8.RuneCountInString(q) < minSearchRunes {
+		return ""
+	}
+	return q
+}
+
+// writeSearchUnavailable answers a search the engine cannot serve: it is not
+// configured or the call failed. The list without q keeps working.
+func writeSearchUnavailable(w http.ResponseWriter) {
+	wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]any{"error": true, "message": "Search is unavailable."})
+}
+
+// rankOrder orders rows by their position in ids, the engine's rank. The ids
+// are parsed integers, so the array literal carries no caller text.
+func rankOrder(ids []uint) string {
+	parts := make([]string, len(ids))
+	for i, id := range ids {
+		parts[i] = strconv.FormatUint(uint64(id), 10)
+	}
+	return "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC"
+}
+
+// segmentJSON is one run of a snippet: the engine's plain text, verbatim,
+// and whether it matched the query. It is never markup; the client renders
+// it as text.
+type segmentJSON struct {
+	Text string `json:"text"`
+	Hit  bool   `json:"hit"`
+}
+
+// snippetJSON is the one excerpt a search result shows. Source is "body" or
+// "file" (FileName set); Lead and Trail say whether text was cut before or
+// after the excerpt.
+type snippetJSON struct {
+	Source   string        `json:"source"`
+	FileName *string       `json:"file_name"`
+	Lead     bool          `json:"lead"`
+	Trail    bool          `json:"trail"`
+	Segments []segmentJSON `json:"segments"`
+}
+
+// pickSnippet returns the snippet of m from hit: the first body snippet,
+// mapped one to one. It is nil when the hit has none. Call it only for a
+// row the SQL re-gate returned.
+func pickSnippet(hit beachcomber.Hit, m models.Material) *snippetJSON {
+	for _, sn := range hit.Highlights {
+		if sn.Field != "body" || len(sn.Segments) == 0 {
+			continue
+		}
+		out := &snippetJSON{Source: "body", Segments: make([]segmentJSON, 0, len(sn.Segments))}
+		var joined strings.Builder
+		for _, seg := range sn.Segments {
+			out.Segments = append(out.Segments, segmentJSON{Text: seg.Text, Hit: seg.Match})
+			joined.WriteString(seg.Text)
+		}
+		body := ""
+		if m.Body != nil {
+			body = models.MaterialPlainBody(*m.Body)
+		}
+		text := strings.TrimSpace(joined.String())
+		out.Lead = !strings.HasPrefix(body, text)
+		out.Trail = !strings.HasSuffix(body, text)
+		return out
+	}
+	return nil
+}
+
+// flattenSnippet is a snippet as plain text, for MCP: the segment texts
+// joined, with an ellipsis where text was cut.
+func flattenSnippet(s *snippetJSON) string {
+	if s == nil {
+		return ""
+	}
+	var b strings.Builder
+	if s.Lead {
+		b.WriteString("…")
+	}
+	for _, seg := range s.Segments {
+		b.WriteString(seg.Text)
+	}
+	if s.Trail {
+		b.WriteString("…")
+	}
+	return b.String()
+}
+
+// snippetValue is the snippet key of a row: the struct, a plain string
+// through MCP, or nil when there is none (the key is then left out).
+func snippetValue(s *snippetJSON, mcp bool) any {
+	if s == nil {
+		return nil
+	}
+	if mcp {
+		return flattenSnippet(s)
+	}
+	return s
+}

controllers/api/material_search_test.go

@@ -0,0 +1,200 @@
+package api_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/bouncer"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
+	"git.golem15.com/jakub/sm-bm-plugin/controllers/api"
+	"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
+	"git.golem15.com/jakub/sm-bm-plugin/models"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func segments(parts ...any) []beachcomber.Segment {
+	out := make([]beachcomber.Segment, 0, len(parts)/2)
+	for i := 0; i+1 < len(parts); i += 2 {
+		out = append(out, beachcomber.Segment{Text: parts[i].(string), Match: parts[i+1].(bool)})
+	}
+	return out
+}
+
+func TestMaterialSnippet(t *testing.T) {
+	body := "# Wstęp\n\nNa początku. Proces **globalizacji** zmienia państwa. Koniec tekstu."
+	m := models.Material{ID: 3, Body: &body}
+	hit := beachcomber.Hit{ID: "3", Highlights: []beachcomber.Snippet{
+		{Field: "title", ArrayIndex: -1, Segments: segments("Tytuł", true)},
+		{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia <b>państwa</b>.", false)},
+	}}
+	s := api.PickSnippet(hit, m)
+	if s == nil {
+		t.Fatal("no snippet")
+	}
+	if s.Source != "body" || s.FileName != nil {
+		t.Errorf("source %q file %v, want body and no file", s.Source, s.FileName)
+	}
+	if len(s.Segments) != 3 || s.Segments[1].Text != "globalizacji" || !s.Segments[1].Hit || s.Segments[0].Hit {
+		t.Errorf("segments = %+v", s.Segments)
+	}
+	if s.Segments[2].Text != " zmienia <b>państwa</b>." {
+		t.Errorf("segment text was changed: %q", s.Segments[2].Text)
+	}
+	if !s.Lead || !s.Trail {
+		t.Errorf("lead %v trail %v, want both (cut inside the body)", s.Lead, s.Trail)
+	}
+	b, err := json.Marshal(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := string(b); !strings.Contains(got, `"source":"body","file_name":null,"lead":true,"trail":true,"segments":[{"text":"Proces ","hit":false}`) {
+		t.Errorf("JSON = %s", got)
+	}
+
+	// The whole body: neither lead nor trail.
+	short := "Krótka **notatka**"
+	whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Krótka ", false, "notatka", true)}}}, models.Material{Body: &short})
+	if whole == nil || whole.Lead || whole.Trail {
+		t.Errorf("whole body snippet = %+v, want no lead or trail", whole)
+	}
+	// Start of the body only: trail but no lead.
+	start := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp ", false, "Na", true)}}}, m)
+	if start == nil || start.Lead || !start.Trail {
+		t.Errorf("start snippet = %+v, want trail only", start)
+	}
+
+	// A snippet on another field only, or none at all, gives no snippet.
+	if got := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "title", ArrayIndex: -1, Segments: segments("x", true)}}}, m); got != nil {
+		t.Errorf("title-only hit gave %+v", got)
+	}
+	if got := api.PickSnippet(beachcomber.Hit{ID: "3"}, m); got != nil {
+		t.Errorf("hit without highlights gave %+v", got)
+	}
+
+	if got := api.FlattenSnippet(s); got != "…Proces globalizacji zmienia <b>państwa</b>.…" {
+		t.Errorf("FlattenSnippet = %q", got)
+	}
+	if got := api.FlattenSnippet(nil); got != "" {
+		t.Errorf("FlattenSnippet(nil) = %q", got)
+	}
+	if got := api.RankOrder([]uint{9, 3}); got != "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id), jz_bm_materials.id DESC" {
+		t.Errorf("RankOrder = %q", got)
+	}
+}
+
+func TestMaterialsSearchUnavailable(t *testing.T) {
+	h := noDB()
+	user := &bouncer.Principal{ID: 7}
+	code, body := serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+	if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." {
+		t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body)
+	}
+	// One rune is no search: the list answers as without q (no database).
+	code, body = serve(t, h.Materials, http.MethodGet, "/?q=%20g%20", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+		t.Errorf("one-rune q = %d %v, want the no-database answer", code, body)
+	}
+	code, body = serve(t, h.Materials, http.MethodGet, "/", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." {
+		t.Errorf("no q = %d %v, want the no-database answer", code, body)
+	}
+	// An unconfigured engine is the same 503.
+	h.MaterialIndex = classes.NewMaterialIndexer(searchtest.App(t))
+	code, body = serve(t, h.Materials, http.MethodGet, "/?q=globalizacja", user, nil)
+	if code != http.StatusServiceUnavailable || body["message"] != "Search is unavailable." {
+		t.Errorf("q with a null engine = %d %v, want 503 Search is unavailable.", code, body)
+	}
+}
+
+// sqlRecorder is a dry-run postgres handle that records every query it
+// builds, with its arguments inlined.
+type sqlRecorder struct {
+	mu  sync.Mutex
+	sql []string
+}
+
+func (r *sqlRecorder) queries() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.sql...)
+}
+
+func dryDB(t *testing.T) (*gorm.DB, *sqlRecorder) {
+	t.Helper()
+	db, err := gorm.Open(postgres.New(postgres.Config{DSN: "host=127.0.0.1"}), &gorm.Config{DryRun: true, DisableAutomaticPing: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := &sqlRecorder{}
+	err = db.Callback().Query().After("gorm:query").Register("test:record_sql", func(tx *gorm.DB) {
+		rec.mu.Lock()
+		defer rec.mu.Unlock()
+		rec.sql = append(rec.sql, tx.Dialector.Explain(tx.Statement.SQL.String(), tx.Statement.Vars...))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db, rec
+}
+
+func TestMaterialsSearchTrace(t *testing.T) {
+	stub := &searchtest.Stub{Hits: []searchtest.Hit{
+		{ID: "9"},
+		{ID: "3", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd},
+	}}
+	db, rec := dryDB(t)
+	h := &api.Handlers{
+		DB:            func() *gorm.DB { return db },
+		MaterialIndex: classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...)),
+	}
+	req := httptest.NewRequest(http.MethodGet, "/materials?q=globalizacja&course=7", nil)
+	req = req.WithContext(bouncer.WithUser(req.Context(), &bouncer.Principal{ID: 5}))
+	w := httptest.NewRecorder()
+	h.Materials(w, req)
+	if w.Code != http.StatusOK {
+		t.Fatalf("status %d: %s", w.Code, w.Body.String())
+	}
+	if n := len(stub.Searches()); n != 1 {
+		t.Errorf("searches = %d, want 1", n)
+	}
+	var count, page string
+	for _, q := range rec.queries() {
+		switch {
+		case strings.HasPrefix(q, "SELECT count(*)"):
+			count = q
+		case strings.HasPrefix(q, "SELECT") && strings.Contains(q, `FROM "jz_bm_materials"`):
+			page = q
+		}
+	}
+	if count == "" || page == "" {
+		t.Fatalf("queries = %q, want a count and a page", rec.queries())
+	}
+	for name, q := range map[string]string{"count": count, "page": page} {
+		for _, want := range []string{"jz_bm_materials.is_private = FALSE", "jz_bm_materials.published_at IS NOT NULL", "jz_bm_materials.course_id = 7", "jz_bm_materials.id IN (9,3)"} {
+			if !strings.Contains(q, want) {
+				t.Errorf("%s query lacks %q:\n%s", name, want, q)
+			}
+		}
+	}
+	if !strings.Contains(page, "array_position(ARRAY[9,3]::bigint[], jz_bm_materials.id)") {
+		t.Errorf("page query is not in rank order:\n%s", page)
+	}
+	var out struct {
+		Data []map[string]any `json:"data"`
+		Meta struct {
+			Total int `json:"total"`
+		} `json:"meta"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	if out.Meta.Total != 0 || len(out.Data) != 0 {
+		t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data)
+	}
+}

controllers/api/materials.go

@@ -6,9 +6,11 @@ import (
 	"strings"
 	"time"
 
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
 	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/lagoon/attach"
 	"git.golem15.com/golem15/summercms/modules/surf"
+	"git.golem15.com/jakub/sm-bm-plugin/classes"
 	"git.golem15.com/jakub/sm-bm-plugin/models"
 	"gorm.io/gorm"
 )
@@ -49,23 +51,35 @@ func preloadMaterial(q *gorm.DB) *gorm.DB {
 }
 
 // listMaterials returns one page of the published materials userID may
-// see, newest first.
-func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, page, perPage int) ([]materialListJSON, int64, error) {
+// see. With ids nil the page is newest first. With ids non-nil (search
+// candidates in the engine's rank order) both the count and the page are
+// also limited to ids and the page follows their order; the ids never widen
+// what visibleMaterials and the filter allow. An empty ids is an empty page.
+func listMaterials(db *gorm.DB, now time.Time, userID uint, f materialFilter, ids []uint, page, perPage int) ([]materialListJSON, []models.Material, int64, error) {
+	if ids != nil && len(ids) == 0 {
+		return []materialListJSON{}, nil, 0, nil
+	}
 	q := f.apply(visibleMaterials(db.Model(&models.Material{}), now, userID))
+	if ids != nil {
+		q = q.Where("jz_bm_materials.id IN ?", ids)
+	}
 	var total int64
 	if err := q.Count(&total).Error; err != nil {
-		return nil, 0, err
+		return nil, nil, 0, err
+	}
+	rowsQ := preloadMaterial(f.apply(visibleMaterials(db, now, userID)))
+	if ids != nil {
+		rowsQ = rowsQ.Where("jz_bm_materials.id IN ?", ids).Order(rankOrder(ids))
+	} else {
+		rowsQ = rowsQ.Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC")
 	}
 	var rows []models.Material
-	err := preloadMaterial(f.apply(visibleMaterials(db, now, userID))).
-		Order("jz_bm_materials.published_at DESC, jz_bm_materials.id DESC").
-		Offset((page - 1) * perPage).Limit(perPage).
-		Find(&rows).Error
+	err := rowsQ.Offset((page - 1) * perPage).Limit(perPage).Find(&rows).Error
 	if err != nil {
-		return nil, 0, err
+		return nil, nil, 0, err
 	}
 	out, err := materialsJSON(db, rows)
-	return out, total, err
+	return out, rows, total, err
 }
 
 // materialsJSON builds list rows with their file counts.
@@ -99,31 +113,69 @@ func materialsJSON(db *gorm.DB, rows []models.Material) ([]materialListJSON, err
 	return out, nil
 }
 
-// Materials answers GET /materials?course=&meeting=&tag=&type=&page=&per_page=:
+// Materials answers GET /materials?q=&course=&meeting=&tag=&type=&page=&per_page=:
 // a page of the published materials the caller may see, in the {data, meta}
-// shape of lagoon.Paginate.
+// shape of lagoon.Paginate. With q (2 to 100 characters) the engine answers
+// candidates, which SQL re-gates, counts and pages in rank order, each row
+// with at most one snippet; when the engine cannot answer the request is a
+// 503. Without q the list is newest first and never calls the engine.
 func (h *Handlers) Materials(w http.ResponseWriter, r *http.Request) {
 	userID, ok := h.caller(w, r)
 	if !ok {
 		return
 	}
-	in := query(r, "course", "meeting", "tag", "type", "page", "per_page")
+	in := query(r, "q", "course", "meeting", "tag", "type", "page", "per_page")
 	if !h.validate(w, r, in, materialsRules) {
 		return
 	}
+	filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
+	var cands *classes.MaterialCandidates
+	if q := searchQuery(in); q != "" {
+		c, err := h.MaterialIndex.Search(r.Context(), classes.MaterialSearch{Q: q, CourseID: filter.CourseID, MeetingID: filter.MeetingID, Type: filter.Type, Tag: filter.Tag})
+		if err != nil {
+			writeSearchUnavailable(w)
+			return
+		}
+		cands = &c
+	}
 	db, ok := h.db(w, r)
 	if !ok {
 		return
 	}
-	filter := materialFilter{CourseID: uintParam(in, "course"), MeetingID: uintParam(in, "meeting"), Tag: strings.TrimSpace(str(in, "tag")), Type: str(in, "type")}
 	page := min(intParam(in, "page", 1), 100000)
 	perPage := intParam(in, "per_page", h.perPage())
-	rows, total, err := listMaterials(db, h.now(), userID, filter, page, perPage)
+	var ids []uint
+	if cands != nil {
+		ids = cands.IDs
+		if ids == nil {
+			ids = []uint{}
+		}
+	}
+	out, rows, total, err := listMaterials(db, h.now(), userID, filter, ids, page, perPage)
 	if err != nil {
 		writeOpaque500(w)
 		return
 	}
-	writeJSONPage(w, lagoon.Paginate(rows, page, perPage, total))
+	if cands != nil {
+		attachSnippets(out, rows, cands.Hits, viaMCP(r))
+	}
+	writeJSONPage(w, lagoon.Paginate(out, page, perPage, total))
+}
+
+// attachSnippets sets the snippet of each list row from its Hit. rows are
+// the rows SQL returned (they passed the re-gate), in the order of out; the
+// Hit of an id SQL did not return is never read.
+func attachSnippets(out []materialListJSON, rows []models.Material, hits map[uint]beachcomber.Hit, mcp bool) {
+	for i := range out {
+		if i >= len(rows) || rows[i].ID != out[i].ID {
+			continue
+		}
+		hit, ok := hits[rows[i].ID]
+		if !ok {
+			continue
+		}
+		out[i].Snippet = snippetValue(pickSnippet(hit, rows[i]), mcp)
+	}
 }
 
 // Material answers GET /materials/{id}: one published material the caller

controllers/api/params.go

@@ -37,6 +37,7 @@ var (
 		{Field: "type", Rules: append(lagoon.ParseRules("nullable"), lagoon.In(models.MaterialTypes...))},
 		{Field: "page", Rules: lagoon.ParseRules("nullable|integer|min:1")},
 		{Field: "per_page", Rules: lagoon.ParseRules("nullable|integer|between:1,100")},
+		{Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")},
 	}
 )