feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
173
models/material_search.go
Normal file
173
models/material_search.go
Normal file
@@ -0,0 +1,173 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// A material is kept in the search index jz_bm_materials (beachcomber). The
|
||||
// index holds every material, drafts and private ones included: visibility
|
||||
// is enforced only in SQL, where the engine's ids are candidates that the
|
||||
// student queries re-gate. This file is hand-written; material.go is
|
||||
// generated.
|
||||
|
||||
// SearchableAs is the index name without the search.prefix.
|
||||
func (Material) SearchableAs() string { return "jz_bm_materials" }
|
||||
|
||||
// ShouldBeSearchable is true for every material: the index carries no
|
||||
// visibility, so drafts and private materials are indexed too.
|
||||
func (m *Material) ShouldBeSearchable() bool { return m != nil }
|
||||
|
||||
// ToSearchableArray builds the document of the material. beachcomber reloads
|
||||
// the row by primary key without preloads, so the course, the meeting number
|
||||
// and the tags are loaded here.
|
||||
func (m *Material) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
|
||||
db = db.WithContext(ctx)
|
||||
var course *Course
|
||||
if m.CourseID != nil {
|
||||
var c Course
|
||||
res := db.Where("id = ?", *m.CourseID).Limit(1).Find(&c)
|
||||
if res.Error != nil {
|
||||
return nil, res.Error
|
||||
}
|
||||
if res.RowsAffected > 0 {
|
||||
course = &c
|
||||
}
|
||||
}
|
||||
meetingNumber := 0
|
||||
if m.MeetingID != nil {
|
||||
var mt Meeting
|
||||
res := db.Where("id = ?", *m.MeetingID).Limit(1).Find(&mt)
|
||||
if res.Error != nil {
|
||||
return nil, res.Error
|
||||
}
|
||||
if res.RowsAffected > 0 {
|
||||
meetingNumber = mt.Number
|
||||
}
|
||||
}
|
||||
var tags []Tag
|
||||
err := db.Model(&Tag{}).
|
||||
Joins("JOIN jz_bm_material_tag mt ON mt.tag_id = jz_bm_tags.id").
|
||||
Where("mt.material_id = ?", m.ID).
|
||||
Order("jz_bm_tags.name").
|
||||
Find(&tags).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return MaterialDocument(*m, course, meetingNumber, tags, nil), nil
|
||||
}
|
||||
|
||||
// MaterialDocument is the search document of m with its course, meeting
|
||||
// number (0 when none), tags in order and the texts of its attached files.
|
||||
// Absent numbers are 0, so filters and the sort tiebreak never see a
|
||||
// missing value.
|
||||
func MaterialDocument(m Material, course *Course, meetingNumber int, tags []Tag, fileTexts []string) map[string]any {
|
||||
doc := map[string]any{
|
||||
"id": strconv.FormatUint(uint64(m.ID), 10),
|
||||
"title": m.Title,
|
||||
"type": m.Type,
|
||||
"course_id": int64(0),
|
||||
"meeting_id": int64(0),
|
||||
"published_at": int64(0),
|
||||
}
|
||||
if m.Source != nil && strings.TrimSpace(*m.Source) != "" {
|
||||
doc["source"] = *m.Source
|
||||
}
|
||||
if m.URL != nil && strings.TrimSpace(*m.URL) != "" {
|
||||
doc["url"] = *m.URL
|
||||
}
|
||||
if m.CourseID != nil {
|
||||
doc["course_id"] = int64(*m.CourseID)
|
||||
}
|
||||
if m.MeetingID != nil {
|
||||
doc["meeting_id"] = int64(*m.MeetingID)
|
||||
}
|
||||
if m.PublishedAt != nil {
|
||||
doc["published_at"] = m.PublishedAt.Unix()
|
||||
}
|
||||
if course != nil {
|
||||
label := course.Name
|
||||
if course.Code != nil && strings.TrimSpace(*course.Code) != "" {
|
||||
label = strings.TrimSpace(label + " " + strings.TrimSpace(*course.Code))
|
||||
}
|
||||
doc["course"] = label
|
||||
}
|
||||
if meetingNumber > 0 {
|
||||
doc["meeting_number"] = meetingNumber
|
||||
}
|
||||
names := make([]string, 0, len(tags))
|
||||
slugs := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
names = append(names, t.Name)
|
||||
slugs = append(slugs, t.Slug)
|
||||
}
|
||||
doc["tags"] = names
|
||||
doc["tag_slugs"] = slugs
|
||||
if m.Body != nil {
|
||||
if body := MaterialPlainBody(*m.Body); body != "" {
|
||||
doc["body"] = body
|
||||
}
|
||||
}
|
||||
if len(fileTexts) > 0 {
|
||||
doc["file_texts"] = fileTexts
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
// SearchIndexSchema is the Typesense collection schema. It applies only when
|
||||
// the collection is created; bm:materials-reindex --fresh recreates it.
|
||||
// Every field is stored (a search leaves body and file_texts out of the
|
||||
// answer instead), and no field sets a locale: the default one folds
|
||||
// diacritics, so lodz finds Łódź.
|
||||
func (Material) SearchIndexSchema() map[string]any {
|
||||
return map[string]any{
|
||||
"fields": []map[string]any{
|
||||
{"name": "id", "type": "string"},
|
||||
{"name": "title", "type": "string"},
|
||||
{"name": "source", "type": "string", "optional": true},
|
||||
{"name": "url", "type": "string", "optional": true},
|
||||
{"name": "tags", "type": "string[]", "optional": true},
|
||||
{"name": "tag_slugs", "type": "string[]", "optional": true},
|
||||
{"name": "course", "type": "string", "optional": true},
|
||||
{"name": "course_id", "type": "int64"},
|
||||
{"name": "meeting_id", "type": "int64"},
|
||||
{"name": "meeting_number", "type": "int32", "optional": true},
|
||||
{"name": "type", "type": "string"},
|
||||
{"name": "published_at", "type": "int64"},
|
||||
{"name": "body", "type": "string", "optional": true},
|
||||
{"name": "file_texts", "type": "string[]", "optional": true},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
|
||||
mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
|
||||
mdHeading = regexp.MustCompile(`(?m)^[ \t]{0,3}#{1,6}[ \t]+`)
|
||||
mdQuote = regexp.MustCompile(`(?m)^([ \t]{0,3}>[ \t]?)+`)
|
||||
mdList = regexp.MustCompile(`(?m)^[ \t]*([-+*]|\d+[.)])[ \t]+`)
|
||||
mdEmphasis = regexp.MustCompile(`\*+|~~`)
|
||||
mdUnderStart = regexp.MustCompile(`(^|[^\p{L}\p{N}_])_+`)
|
||||
mdUnderEnd = regexp.MustCompile(`_+([^\p{L}\p{N}_]|$)`)
|
||||
)
|
||||
|
||||
// MaterialPlainBody drops the Markdown syntax of a material body (heading
|
||||
// hashes, blockquote and list markers, emphasis, inline code ticks, link
|
||||
// and image syntax down to their text) and collapses white space, so the
|
||||
// index holds and highlights only words.
|
||||
func MaterialPlainBody(body string) string {
|
||||
s := strings.ReplaceAll(body, "`", "")
|
||||
s = mdImage.ReplaceAllString(s, "$1")
|
||||
s = mdLink.ReplaceAllString(s, "$1")
|
||||
s = mdHeading.ReplaceAllString(s, "")
|
||||
s = mdQuote.ReplaceAllString(s, "")
|
||||
s = mdList.ReplaceAllString(s, "")
|
||||
s = mdEmphasis.ReplaceAllString(s, "")
|
||||
s = mdUnderStart.ReplaceAllString(s, "$1")
|
||||
s = mdUnderEnd.ReplaceAllString(s, "$1")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
Reference in New Issue
Block a user