feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
Jakub Zych
173
models/material_search.go
Normal file
173
models/material_search.go
Normal file
@@ -0,0 +1,173 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// A material is kept in the search index jz_bm_materials (beachcomber). The
|
||||
// index holds every material, drafts and private ones included: visibility
|
||||
// is enforced only in SQL, where the engine's ids are candidates that the
|
||||
// student queries re-gate. This file is hand-written; material.go is
|
||||
// generated.
|
||||
|
||||
// SearchableAs is the index name without the search.prefix.
|
||||
func (Material) SearchableAs() string { return "jz_bm_materials" }
|
||||
|
||||
// ShouldBeSearchable is true for every material: the index carries no
|
||||
// visibility, so drafts and private materials are indexed too.
|
||||
func (m *Material) ShouldBeSearchable() bool { return m != nil }
|
||||
|
||||
// ToSearchableArray builds the document of the material. beachcomber reloads
|
||||
// the row by primary key without preloads, so the course, the meeting number
|
||||
// and the tags are loaded here.
|
||||
func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
|
||||
db = db.WithContext(ctx)
|
||||
var course *Course
|
||||
if m.CourseID != nil {
|
||||
var c Course
|
||||
res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c)
|
||||
if res.Error != nil {
|
||||
return nil, res.Error
|
||||
}
|
||||
if res.RowsAffected > 0 {
|
||||
course = &c
|
||||
}
|
||||
}
|
||||
meetingNumber := 0
|
||||
if m.MeetingID != nil {
|
||||
var mt Meeting
|
||||
res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt)
|
||||
if res.Error != nil {
|
||||
return nil, res.Error
|
||||
}
|
||||
if res.RowsAffected > 0 {
|
||||
meetingNumber = mt.Number
|
||||
}
|
||||
}
|
||||
var tags []Tag
|
||||
err := db.Model(&Tag{}).
|
||||
Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id").
|
||||
Where("mt.material_id = ?", m.ID).
|
||||
Order("jz_bm_tags.name").
|
||||
Find(&tags).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return MaterialDocument(*m, course, meetingNumber, tags, nil), nil
|
||||
}
|
||||
|
||||
// MaterialDocument is the search document of m with its course, meeting
|
||||
// number (0 when none), tags in order and the texts of its attached files.
|
||||
// Absent numbers are 0, so filters and the sort tiebreak never see a
|
||||
// missing value.
|
||||
func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any {
|
||||
doc := map[string]any{
|
||||
"id": strconv.FormatUint(uint64(m.ID), 10),
|
||||
"title": m.Title,
|
||||
"type": m.Type,
|
||||
"course_id": int64(0),
|
||||
"meeting_id": int64(0),
|
||||
"published_at": int64(0),
|
||||
}
|
||||
if m.Source != nil && strings.TrimSpace(*m.Source) != "" {
|
||||
doc["source"] = *m.Source
|
||||
}
|
||||
if m.URL != nil && strings.TrimSpace(*m.URL) != "" {
|
||||
doc["url"] = *m.URL
|
||||
}
|
||||
if m.CourseID != nil {
|
||||
doc["course_id"] = int64(*m.CourseID)
|
||||
}
|
||||
if m.MeetingID != nil {
|
||||
doc["meeting_id"] = int64(*m.MeetingID)
|
||||
}
|
||||
if m.PublishedAt != nil {
|
||||
doc["published_at"] = m.PublishedAt.Unix()
|
||||
}
|
||||
if course != nil {
|
||||
label := course.Name
|
||||
if course.Code != nil && strings.TrimSpace(*course.Code) != "" {
|
||||
label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code))
|
||||
}
|
||||
doc["course"] = label
|
||||
}
|
||||
if meetingNumber > 0 {
|
||||
doc["meeting_number"] = meetingNumber
|
||||
}
|
||||
names := make([]string, 0, len(tags))
|
||||
slugs := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
names = append(names, t.Name)
|
||||
slugs = append(slugs, t.Slug)
|
||||
}
|
||||
doc["tags"] = names
|
||||
doc["tag_slugs"] = slugs
|
||||
if m.Body != nil {
|
||||
if body := MaterialPlainBody(*m.Body); body != "" {
|
||||
doc["body"] = body
|
||||
}
|
||||
}
|
||||
if len(fileTexts) > 0 {
|
||||
doc["file_texts"] = fileTexts
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
// SearchIndexSchema is the Typesense collection schema. It applies only when
|
||||
// the collection is created; bm:materials-reindex --fresh recreates it.
|
||||
// Every field is stored (a search leaves body and file_texts out of the
|
||||
// answer instead), and no field sets a locale: the default one folds
|
||||
// diacritics, so lodz finds Łódź.
|
||||
func (Material) SearchIndexSchema() map[string]any {
|
||||
return map[string]any{
|
||||
"fields": []map[string]any{
|
||||
{"name": "id", "type": "string"},
|
||||
{"name": "title", "type": "string"},
|
||||
{"name": "source", "type": "string", "optional": true},
|
||||
{"name": "url", "type": "string", "optional": true},
|
||||
{"name": "tags", "type": "string[]", "optional": true},
|
||||
{"name": "tag_slugs", "type": "string[]", "optional": true},
|
||||
{"name": "course", "type": "string", "optional": true},
|
||||
{"name": "course_id", "type": "int64"},
|
||||
{"name": "meeting_id", "type": "int64"},
|
||||
{"name": "meeting_number", "type": "int32", "optional": true},
|
||||
{"name": "type", "type": "string"},
|
||||
{"name": "published_at", "type": "int64"},
|
||||
{"name": "body", "type": "string", "optional": true},
|
||||
{"name": "file_texts", "type": "string[]", "optional": true},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
|
||||
mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
|
||||
mdHeading = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`)
|
||||
mdQuote = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`)
|
||||
mdList = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`)
|
||||
mdEmphasis = regexp.MustCompile(`\*+|~~`)
|
||||
mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`)
|
||||
mdUnderEnd = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`)
|
||||
)
|
||||
|
||||
// MaterialPlainBody drops the Markdown syntax of a material body (heading
|
||||
// hashes, blockquote and list markers, emphasis, inline code ticks, link
|
||||
// and image syntax down to their text) and collapses white space, so the
|
||||
// index holds and highlights only words.
|
||||
func MaterialPlainBody(body string) string {
|
||||
s := strings.ReplaceAll(body, "`", "")
|
||||
s = mdImage.ReplaceAllString(s, "$1")
|
||||
s = mdLink.ReplaceAllString(s, "$1")
|
||||
s = mdHeading.ReplaceAllString(s, "")
|
||||
s = mdQuote.ReplaceAllString(s, "")
|
||||
s = mdList.ReplaceAllString(s, "")
|
||||
s = mdEmphasis.ReplaceAllString(s, "")
|
||||
s = mdUnderStart.ReplaceAllString(s, "$1")
|
||||
s = mdUnderEnd.ReplaceAllString(s, "$1")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
124
models/material_search_test.go
Normal file
124
models/material_search_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/jakub/sm-bm-plugin/models"
|
||||
)
|
||||
|
||||
func TestMaterialSearchSchema(t *testing.T) {
|
||||
m := models.Material{}
|
||||
if got := m.SearchableAs(); got != "jz_bm_materials" {
|
||||
t.Errorf("SearchableAs = %q, want jz_bm_materials", got)
|
||||
}
|
||||
fields, ok := m.SearchIndexSchema()["fields"].([]map[string]any)
|
||||
if !ok || len(fields) == 0 {
|
||||
t.Fatalf("schema fields = %#v", m.SearchIndexSchema()["fields"])
|
||||
}
|
||||
byName := map[string]map[string]any{}
|
||||
for _, f := range fields {
|
||||
name, _ := f["name"].(string)
|
||||
typ, _ := f["type"].(string)
|
||||
if name == "" || typ == "" {
|
||||
t.Errorf("field without a name or type: %v", f)
|
||||
}
|
||||
if _, ok := f["locale"]; ok {
|
||||
t.Errorf("field %s sets a locale; the default one folds Polish diacritics", name)
|
||||
}
|
||||
if store, ok := f["store"]; ok && store == false {
|
||||
t.Errorf("field %s turns storage off", name)
|
||||
}
|
||||
byName[name] = f
|
||||
}
|
||||
for _, name := range []string{"body", "file_texts"} {
|
||||
f, ok := byName[name]
|
||||
if !ok {
|
||||
t.Errorf("schema has no %s field", name)
|
||||
continue
|
||||
}
|
||||
if f["optional"] != true {
|
||||
t.Errorf("%s is not optional", name)
|
||||
}
|
||||
}
|
||||
if byName["body"]["type"] != "string" || byName["file_texts"]["type"] != "string[]" {
|
||||
t.Errorf("body %v, file_texts %v: want string and string[]", byName["body"]["type"], byName["file_texts"]["type"])
|
||||
}
|
||||
for _, name := range []string{"id", "title", "source", "url", "tags", "tag_slugs", "course", "course_id", "meeting_id", "meeting_number", "type", "published_at"} {
|
||||
if _, ok := byName[name]; !ok {
|
||||
t.Errorf("schema has no %s field", name)
|
||||
}
|
||||
}
|
||||
if !(&m).ShouldBeSearchable() {
|
||||
t.Error("a material is not searchable")
|
||||
}
|
||||
var nilMaterial *models.Material
|
||||
if nilMaterial.ShouldBeSearchable() {
|
||||
t.Error("a nil material is searchable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterialDocument(t *testing.T) {
|
||||
plain := models.MaterialPlainBody("# Nagłówek\n\nTekst **pogrubienie** i _kursywa_ oraz [tekst](https://x) i .\n\n> cytat\n> dalej\n\n- punkt `kod` snake_case")
|
||||
want := "Nagłówek Tekst pogrubienie i kursywa oraz tekst i obraz. cytat dalej punkt kod snake_case"
|
||||
if plain != want {
|
||||
t.Errorf("MaterialPlainBody =\n%q\nwant\n%q", plain, want)
|
||||
}
|
||||
|
||||
code := "BM-12"
|
||||
course := &models.Course{ID: 4, Name: "Bezpieczeństwo", Code: &code}
|
||||
courseID, meetingID := uint(4), uint(9)
|
||||
at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
body := "## Wstęp\n\n**Globalizacja** w praktyce"
|
||||
source := "Giddens"
|
||||
m := models.Material{ID: 12, Title: "Globalizacja", Type: "notatka", CourseID: &courseID, MeetingID: &meetingID, PublishedAt: &at, Body: &body, Source: &source}
|
||||
tags := []models.Tag{{Name: "Globalizacja", Slug: "globalizacja"}, {Name: "Polityka", Slug: "polityka"}}
|
||||
doc := models.MaterialDocument(m, course, 3, tags, []string{"tekst pliku"})
|
||||
checks := map[string]any{
|
||||
"id": "12",
|
||||
"title": "Globalizacja",
|
||||
"type": "notatka",
|
||||
"source": "Giddens",
|
||||
"course": "Bezpieczeństwo BM-12",
|
||||
"course_id": int64(4),
|
||||
"meeting_id": int64(9),
|
||||
"meeting_number": 3,
|
||||
"published_at": at.Unix(),
|
||||
"body": "Wstęp Globalizacja w praktyce",
|
||||
}
|
||||
for k, v := range checks {
|
||||
if doc[k] != v {
|
||||
t.Errorf("doc[%s] = %#v, want %#v", k, doc[k], v)
|
||||
}
|
||||
}
|
||||
if got, _ := doc["tags"].([]string); !slices.Equal(got, []string{"Globalizacja", "Polityka"}) {
|
||||
t.Errorf("tags = %v", doc["tags"])
|
||||
}
|
||||
if got, _ := doc["tag_slugs"].([]string); !slices.Equal(got, []string{"globalizacja", "polityka"}) {
|
||||
t.Errorf("tag_slugs = %v", doc["tag_slugs"])
|
||||
}
|
||||
if got, _ := doc["file_texts"].([]string); !slices.Equal(got, []string{"tekst pliku"}) {
|
||||
t.Errorf("file_texts = %v", doc["file_texts"])
|
||||
}
|
||||
|
||||
// No code, no course id, no meeting, a draft, no files.
|
||||
bare := models.MaterialDocument(models.Material{ID: 5, Title: "Szkic", Type: "inne"}, &models.Course{Name: "Teoria"}, 0, nil, nil)
|
||||
if bare["course"] != "Teoria" {
|
||||
t.Errorf("course without code = %v", bare["course"])
|
||||
}
|
||||
for _, k := range []string{"course_id", "meeting_id", "published_at"} {
|
||||
if bare[k] != int64(0) {
|
||||
t.Errorf("bare %s = %#v, want 0", k, bare[k])
|
||||
}
|
||||
}
|
||||
for _, k := range []string{"meeting_number", "file_texts", "body", "source", "url"} {
|
||||
if _, ok := bare[k]; ok {
|
||||
t.Errorf("bare document carries %s", k)
|
||||
}
|
||||
}
|
||||
none := models.MaterialDocument(models.Material{ID: 6, Title: "X", Type: "link"}, nil, 0, nil, nil)
|
||||
if _, ok := none["course"]; ok {
|
||||
t.Error("a material without a course carries course")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user