From dd13795e6fa999931a9f3be30f116f11724d374d Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Sun, 8 Nov 2026 19:53:26 +0100 Subject: [PATCH] feat: search the management list and the MCP material tools - GET /api/v1/bm/materials takes q: candidates over every material, in rank order, with the snippet of the student list; 503 when the engine cannot answer - bm_materials and bm_manage_list_materials take q; under MCP the snippet is plain text with ellipses where text was cut - MCPMaterialPrivacy no longer claims no file text is ever returned - README documents q, the snippet, the 503, the Search section and bm:materials-reindex --- README.md | 37 ++++++++++++++++--- controllers/api/manage.go | 48 +++++++++++++++++++++---- controllers/api/material_search_test.go | 40 +++++++++++++++++++++ controllers/api/params.go | 4 +++ mcp_manage.go | 5 +-- mcp_student.go | 5 +-- mcp_test.go | 4 +-- 7 files changed, 126 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index e4dbfc0..dad46a6 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Every route except the calendar feed itself needs `Authorization: Bearer `) before the bearer and IP keys. ## Student app @@ -360,8 +370,27 @@ Defaults in `config/config.yaml`, read as `jz.bm.`: | `jz.bm.dashboard_limit` | `5` | Sessions and deadlines on `/dashboard`. | | `jz.bm.public_url` | `""` | Absolute base URL of the application (`https://bm.jakubzych.com`), used in the calendar feed URL and the course links inside the feed. Empty means the scheme and Host of the request; set it behind a TLS-terminating proxy. The application overrides it in `config/jz.bm.yaml`. | +## Search + +Materials are searched through the framework's `beachcomber` module with the Typesense engine (`search.driver: typesense`; the blank import of `beachcomber/typesense` is in `search.go`). `models.Material` is a `beachcomber.Searchable` (`models/material_search.go`) kept in the index `jz_bm_materials` (after `search.prefix`). A document holds `id`, `title`, `source`, `url`, `tags` and `tag_slugs`, `course` (name and code), `course_id`, `meeting_id`, `meeting_number`, `type`, `published_at` (Unix seconds, 0 for a draft), the body with its Markdown syntax removed and `file_texts` (the text of attached files). No field sets a locale, so the default one folds diacritics and `lodz` finds `Łódź`. + +The index holds every material, drafts and private ones included, and no visibility: the engine's ids are candidates only. A search asks for at most 250 of them (title weighted over tags, source and course, then body, link and file text), and SQL re-gates them with the same visibility rule and filters as the list, counts them and pages over them in the engine's order. A snippet is read only for a row SQL returned, so a material the caller may not see never reaches an answer, its count or a snippet. + +The beachcomber service is app-wide, and so is its Gate: jz.bm installs one that is on only for contexts it marks (`classes.MarkMaterialSearch`), so `golem15.chatchannels` messages and DMs are never indexed. Every material write syncs explicitly after it commits: the admin form (create, update, delete), `POST`, `PUT` and `DELETE /api/v1/bm/materials`. A failed sync is logged with the material id and never fails the write. + +Configuration, in the app's `config/search.yaml`: `search.driver`, `search.prefix`, `search.typesense.host`, `port`, `protocol` and `connection_timeout_seconds`. The API key goes only in the environment, `SUMMER_SEARCH__TYPESENSE__API_KEY`; with no key the engine is not configured, nothing is sent and every search answers 503. + ## CLI commands +### bm:materials-reindex + +```sh +./bin/bm bm:materials-reindex +./bin/bm bm:materials-reindex --fresh +``` + +Ensures the `jz_bm_materials` collection and syncs every material through the same path as the sync after a save, ending with `Reindexed: N synced, M failed`; it exits non-zero when any material failed. `--fresh` first drops the collection and recreates it from `models.Material.SearchIndexSchema`: use it on the first deploy and after a schema change. Without an engine the command refuses with an error naming `search.driver` and `search.typesense.api_key`; `--if-configured` makes that a notice and a zero exit. The plugin schedules `bm:materials-reindex --if-configured` every night at 04:15 (`Schedule`, run by `serve`'s scheduler), because course renames, meeting renumbering, failed syncs and Typesense restarts can leave the index stale. Run it by hand after a Typesense restart. + ### bm:seed ```sh @@ -407,7 +436,7 @@ The file format (every top-level array is optional, and nullable fields may be l ## Dependencies -- SummerCMS framework modules: `pact`, `party`, `backpack`, `surf`, `wire`, `bouncer`, `lagoon` (with `lagoon/attach`), `cabana`, `bonfire`, `phrasebook`. +- SummerCMS framework modules (v0.6.4): `pact`, `party`, `backpack`, `surf`, `wire`, `bouncer`, `lagoon` (with `lagoon/attach`), `cabana`, `bonfire`, `phrasebook`, `beachcomber` (with `beachcomber/typesense`). - The `golem15.user` plugin (`git.golem15.com/golem15/sm-user-plugin`): the `users` table and the `jwt.auth` middleware. `jz.bm` declares it in `Requires`, so its migrations run first. - GORM and gormigrate. Tests also use testcontainers-go and pgx. diff --git a/controllers/api/manage.go b/controllers/api/manage.go index d732fbb..dc828ab 100644 --- a/controllers/api/manage.go +++ b/controllers/api/manage.go @@ -155,6 +155,9 @@ type managedMaterialJSON struct { AuthorUserID *uint `json:"author_user_id"` Author *userRefJSON `json:"author"` Shares []userRefJSON `json:"shares"` + // Snippet is the matched passage of a search result, as on the + // student list; nil, and the key absent, without q. + Snippet any `json:"snippet,omitempty"` } // materialPeople are the authors and shares of a batch of materials, as @@ -206,27 +209,58 @@ func newManagedMaterialJSON(m models.Material, people materialPeople) managedMat return out } -// ManagedMaterials answers GET /materials: every material, drafts -// included, newest first. An MCP call gets the bodies only with -// include_body (see managedMaterialView). +// ManagedMaterials answers GET /materials?q=&include_body=: every material, +// drafts and private ones included, newest first. With q (2 to 100 +// characters) the engine answers candidates, which are loaded in rank order +// with at most one snippet each; when the engine cannot answer the request +// is a 503. An MCP call gets the bodies only with include_body (see +// managedMaterialView) and the snippets as plain text. func (h *Handlers) ManagedMaterials(w http.ResponseWriter, r *http.Request) { + in := query(r, "q", "include_body") + if !h.validate(w, r, in, manageMaterialsRules) { + return + } + var cands *classes.MaterialCandidates + if q := searchQuery(in); q != "" { + c, err := h.MaterialIndex.Search(r.Context(), classes.MaterialSearch{Q: q}) + if err != nil { + writeSearchUnavailable(w) + return + } + cands = &c + } db, ok := h.db(w, r) if !ok { return } var rows []models.Material - if err := preloadMaterial(db).Order("jz_bm_materials.id DESC").Find(&rows).Error; err != nil { - writeOpaque500(w) - return + if cands == nil || len(cands.IDs) > 0 { + q := preloadMaterial(db) + if cands != nil { + q = q.Where("jz_bm_materials.id IN ?", cands.IDs).Order(rankOrder(cands.IDs)) + } else { + q = q.Order("jz_bm_materials.id DESC") + } + if err := q.Find(&rows).Error; err != nil { + writeOpaque500(w) + return + } } people, err := loadMaterialPeople(db, rows...) if err != nil { writeOpaque500(w) return } + mcp := viaMCP(r) out := make([]any, 0, len(rows)) for _, m := range rows { - out = append(out, managedMaterialView(newManagedMaterialJSON(m, people), viaMCP(r), includeBody(r))) + row := newManagedMaterialJSON(m, people) + if cands != nil { + if hit, ok := cands.Hits[m.ID]; ok { + row.Snippet = snippetValue(pickSnippet(hit, m), mcp) + } + } + out = append(out, managedMaterialView(row, mcp, includeBody(r))) } writeData(w, out) } diff --git a/controllers/api/material_search_test.go b/controllers/api/material_search_test.go index 2681351..d8034c5 100644 --- a/controllers/api/material_search_test.go +++ b/controllers/api/material_search_test.go @@ -198,3 +198,43 @@ func TestMaterialsSearchTrace(t *testing.T) { t.Errorf("total %d data %v, want the dry SQL's 0 (never the engine's found)", out.Meta.Total, out.Data) } } + +func TestManagedMaterialsSearch(t *testing.T) { + h := noDB() + user := &bouncer.Principal{ID: 7} + code, body := serve(t, h.ManagedMaterials, http.MethodGet, "/?q=globalizacja", user, nil) + if code != http.StatusServiceUnavailable || body["error"] != true || body["message"] != "Search is unavailable." { + t.Errorf("q without an index = %d %v, want 503 Search is unavailable.", code, body) + } + code, body = serve(t, h.ManagedMaterials, http.MethodGet, "/?q="+strings.Repeat("a", 101), user, nil) + if code != http.StatusUnprocessableEntity { + t.Errorf("q over 100 characters = %d %v, want 422", code, body) + } else if errs, _ := body["errors"].(map[string]any); errs["q"] == nil { + t.Errorf("422 errors %v have no q message", errs) + } + code, body = serve(t, h.ManagedMaterials, http.MethodGet, "/?q=a&include_body=true", user, nil) + if code != http.StatusServiceUnavailable || body["message"] != "Database unavailable." { + t.Errorf("one-rune q = %d %v, want the no-database answer", code, body) + } +} + +func TestMaterialSnippetMCP(t *testing.T) { + body := "Wstęp. Proces globalizacji zmienia państwa. Koniec." + m := models.Material{Body: &body} + mid := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Proces ", false, "globalizacji", true, " zmienia", false)}}}, m) + if got := api.FlattenSnippet(mid); got != "…Proces globalizacji zmienia…" { + t.Errorf("middle = %q, want ellipses on both sides", got) + } + head := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("Wstęp", true, ". Proces", false)}}}, m) + if got := api.FlattenSnippet(head); got != "Wstęp. Proces…" { + t.Errorf("head = %q, want a trailing ellipsis only", got) + } + tail := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments("państwa. ", false, "Koniec", true, ".", false)}}}, m) + if got := api.FlattenSnippet(tail); got != "…państwa. Koniec." { + t.Errorf("tail = %q, want a leading ellipsis only", got) + } + whole := api.PickSnippet(beachcomber.Hit{Highlights: []beachcomber.Snippet{{Field: "body", ArrayIndex: -1, Segments: segments(body, false)}}}, m) + if got := api.FlattenSnippet(whole); got != body { + t.Errorf("whole = %q, want no ellipsis", got) + } +} diff --git a/controllers/api/params.go b/controllers/api/params.go index 515d259..5467273 100644 --- a/controllers/api/params.go +++ b/controllers/api/params.go @@ -39,6 +39,10 @@ var ( {Field: "per_page", Rules: lagoon.ParseRules("nullable|integer|between:1,100")}, {Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")}, } + // manageMaterialsRules check the search of the management list. + manageMaterialsRules = []lagoon.RequestRule{ + {Field: "q", Rules: lagoon.ParseRules("nullable|string|max:100")}, + } ) // query collects the named query parameters that are present into the input diff --git a/mcp_manage.go b/mcp_manage.go index 4396eec..e538bec 100644 --- a/mcp_manage.go +++ b/mcp_manage.go @@ -185,7 +185,8 @@ type updateDeadlineInput struct { // listManagedMaterialsInput is the input of bm_manage_list_materials. type listManagedMaterialsInput struct { - IncludeBody bool `json:"include_body,omitempty" query:"include_body" jsonschema:"true to include each material's body (it may hold transcript text)"` + Q string `json:"q,omitempty" query:"q" jsonschema:"full-text search over the title, tags, source, course, link, body and the text of attached PDFs, 2 to 100 characters; results then come by relevance, each with a short plain-text snippet when the match is in the body or a file"` + IncludeBody bool `json:"include_body,omitempty" query:"include_body" jsonschema:"true to include each material's body (it may hold transcript text)"` } // showManagedMaterialInput is the input of bm_manage_show_material. @@ -327,7 +328,7 @@ func manageTools() []pact.MCPTool { d, "/deadlines/{id}", idInput{}), tool("manage_list_materials", "List materials (manage)", - "Lists every material, drafts included, newest first, with course, zjazd, type, source, link, publication time and tags. "+MCPMaterialPrivacy+readMaterials, + "Lists every material, drafts and private ones included, newest first, or by relevance with q, with course, zjazd, type, source, link, publication time and tags (and the snippet with q); 503 when search is unavailable. "+MCPMaterialPrivacy+readMaterials, g, "/materials", listManagedMaterialsInput{}), tool("manage_show_material", "Show a material (manage)", "Returns one material, draft or not, with its public files (id, absolute link, name, size and type); 404 when unknown. "+MCPMaterialPrivacy+readMaterials, diff --git a/mcp_student.go b/mcp_student.go index 57614f2..46bd512 100644 --- a/mcp_student.go +++ b/mcp_student.go @@ -8,7 +8,7 @@ import ( // MCPMaterialPrivacy is the sentence every material tool description // carries (D-17, D-21). -const MCPMaterialPrivacy = "Returns metadata and links only, never file contents; a material's body (possible transcript text) is left out unless include_body is true." +const MCPMaterialPrivacy = "Returns metadata and links, never whole file contents; with q a material may carry a short plain-text snippet of the matched passage, possibly from an attached file; a material's body (possible transcript text) is left out unless include_body is true." // The sentences every student tool description ends with (D-15). const ( @@ -47,6 +47,7 @@ type readingsInput struct { // materialsInput is the input of bm_materials. type materialsInput struct { + Q string `json:"q,omitempty" query:"q" jsonschema:"full-text search over the title, tags, source, course, link, body and the text of attached PDFs, 2 to 100 characters; results then come by relevance, each with a short plain-text snippet when the match is in the body or a file"` Course uint `json:"course,omitempty" query:"course" jsonschema:"only materials of this course id"` Meeting uint `json:"meeting,omitempty" query:"meeting" jsonschema:"only materials of this zjazd (meeting) id"` Tag string `json:"tag,omitempty" query:"tag" jsonschema:"only materials with this tag slug, at most 100 characters"` @@ -175,7 +176,7 @@ func studentTools() []pact.MCPTool { { Name: "materials", Title: "BM materials", - Description: "Lists a page of the published materials the student may see (public ones and private ones they wrote or that are shared with them), newest first, with title, type, course, zjazd, source, link, tags and file count, plus the page meta. " + MCPMaterialPrivacy + needsScopeRead, + Description: "Lists a page of the published materials the student may see (public ones and private ones they wrote or that are shared with them), newest first, or by relevance with q, with title, type, course, zjazd, source, link, tags and file count (and the snippet with q), plus the page meta. " + MCPMaterialPrivacy + needsScopeRead, Guard: "user", Method: http.MethodGet, Path: TokenPrefix + "/materials", diff --git a/mcp_test.go b/mcp_test.go index 5e6a86d..45084d2 100644 --- a/mcp_test.go +++ b/mcp_test.go @@ -120,14 +120,14 @@ var queryKeys = map[string][]string{ "manage_list_class_sessions": {"course_id", "meeting_id"}, "manage_list_readings": {"course_id", "meeting_id", "is_required"}, "manage_list_deadlines": {"course_id", "kind"}, - "manage_list_materials": {"include_body"}, + "manage_list_materials": {"q", "include_body"}, "manage_show_material": {"include_body"}, "courses": {"semester"}, "schedule": {"from", "to"}, "deadlines": {"from", "to"}, "readings": {"course"}, "checklists": {"meeting"}, - "materials": {"course", "meeting", "tag", "type", "page", "per_page"}, + "materials": {"q", "course", "meeting", "tag", "type", "page", "per_page"}, "material": {"include_body"}, }