feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
Jakub Zych
256
classes/material_index.go
Normal file
256
classes/material_index.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/beachcomber"
|
||||
"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// materialSearchKey marks a context whose syncs belong to jz.bm.
|
||||
type materialSearchKey struct{}
|
||||
|
||||
// MarkMaterialSearch returns ctx marked as a jz.bm material sync. The
|
||||
// app-wide beachcomber Gate (MaterialSearchGate) is on only for a marked
|
||||
// context, so every jz.bm Sync or Remove passes one.
|
||||
func MarkMaterialSearch(ctx context.Context) context.Context {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return context.WithValue(ctx, materialSearchKey{}, true)
|
||||
}
|
||||
|
||||
// MaterialSearchGate is the app-wide beachcomber Gate jz.bm installs. The
|
||||
// Gate sees only (ctx, db), not the model, so it is on only for a context
|
||||
// marked by MarkMaterialSearch: golem15.chatchannels syncs its messages
|
||||
// with unmarked contexts, and chat messages and DMs never reach the index.
|
||||
// The automatic callbacks of material writes are off as well; every jz.bm
|
||||
// write path syncs explicitly.
|
||||
func MaterialSearchGate() beachcomber.Gate {
|
||||
return beachcomber.GateFunc(func(ctx context.Context, _ *gorm.DB) bool {
|
||||
return ctx != nil && ctx.Value(materialSearchKey{}) != nil
|
||||
})
|
||||
}
|
||||
|
||||
// MaterialSearchWindow is how many engine candidates one search re-gates in
|
||||
// SQL. The list counts and pages over the gated rows, so a page never
|
||||
// depends on how the engine paged.
|
||||
const MaterialSearchWindow = typesense.MaxPerPage
|
||||
|
||||
// MaterialQueryBy are the document fields a query is matched against, and
|
||||
// MaterialQueryWeights their weights in the same order: the title over the
|
||||
// tags, the source and the course, then the body, the link and file text.
|
||||
var (
|
||||
MaterialQueryBy = []string{"title", "tags", "source", "course", "url", "body", "file_texts"}
|
||||
MaterialQueryWeights = []int{6, 4, 3, 3, 1, 2, 1}
|
||||
)
|
||||
|
||||
// materialHighlightFields are the fields snippets are cut from.
|
||||
var materialHighlightFields = []string{"body", "file_texts"}
|
||||
|
||||
// ErrSearchUnavailable is returned by Search when the engine is not
|
||||
// configured or the search failed. The API answers it with a 503.
|
||||
var ErrSearchUnavailable = errors.New("jz.bm: search is unavailable")
|
||||
|
||||
// MaterialSearch is one material search: the query text and the validated
|
||||
// filters of the list.
|
||||
type MaterialSearch struct {
|
||||
Q string
|
||||
CourseID uint
|
||||
MeetingID uint
|
||||
Type string
|
||||
Tag string
|
||||
}
|
||||
|
||||
// MaterialCandidates are the engine's answer: the candidate ids in rank
|
||||
// order and the Hit of each. Both are candidates only: the caller re-gates
|
||||
// the ids in SQL and reads the Hit of a row only once SQL returned it.
|
||||
// Highlights reports whether the engine was asked for snippets.
|
||||
type MaterialCandidates struct {
|
||||
IDs []uint
|
||||
Hits map[uint]beachcomber.Hit
|
||||
Highlights bool
|
||||
}
|
||||
|
||||
// MaterialIndexer searches the materials index and keeps it current. A nil
|
||||
// *MaterialIndexer is valid: Search answers ErrSearchUnavailable and the
|
||||
// syncs do nothing.
|
||||
type MaterialIndexer struct {
|
||||
// Service returns the app's beachcomber service, or nil.
|
||||
Service func() *beachcomber.Service
|
||||
// Log is where failures are logged; nil means slog.Default.
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// NewMaterialIndexer returns the indexer of app. The service is resolved on
|
||||
// each call, so it may be built after the indexer.
|
||||
func NewMaterialIndexer(app *backpack.App) *MaterialIndexer {
|
||||
return &MaterialIndexer{
|
||||
Service: func() *beachcomber.Service {
|
||||
if app == nil {
|
||||
return nil
|
||||
}
|
||||
svc, err := beachcomber.From(app)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return svc
|
||||
},
|
||||
Log: diary.For("jz.bm"),
|
||||
}
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) service() *beachcomber.Service {
|
||||
if ix == nil || ix.Service == nil {
|
||||
return nil
|
||||
}
|
||||
return ix.Service()
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) log() *slog.Logger {
|
||||
if ix == nil || ix.Log == nil {
|
||||
return slog.Default()
|
||||
}
|
||||
return ix.Log
|
||||
}
|
||||
|
||||
// Search asks the engine for up to MaterialSearchWindow candidates of req.
|
||||
// It answers ErrSearchUnavailable (wrapping the cause) when the service or
|
||||
// its engine is missing, the engine is not configured or the call fails.
|
||||
// The query text is never logged.
|
||||
func (ix *MaterialIndexer) Search(ctx context.Context, req MaterialSearch) (MaterialCandidates, error) {
|
||||
svc := ix.service()
|
||||
if svc == nil {
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: no search service", ErrSearchUnavailable)
|
||||
}
|
||||
engine := svc.Engine()
|
||||
if engine == nil || !engine.Configured() {
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: the search engine is not configured", ErrSearchUnavailable)
|
||||
}
|
||||
q := beachcomber.Query{
|
||||
Q: req.Q,
|
||||
QueryBy: MaterialQueryBy,
|
||||
QueryByWeights: MaterialQueryWeights,
|
||||
FilterBy: materialFilterBy(req),
|
||||
SortBy: "_text_match:desc,published_at:desc",
|
||||
Page: 1,
|
||||
PerPage: MaterialSearchWindow,
|
||||
ExcludeFields: []string{"body", "file_texts"},
|
||||
}
|
||||
highlights := beachcomber.SupportsHighlights(engine)
|
||||
if highlights {
|
||||
q.Highlight = &beachcomber.Highlight{Fields: materialHighlightFields, AffixTokens: 12, MaxSnippetsPerArray: 1}
|
||||
}
|
||||
res, err := beachcomber.SearchPage(ctx, engine, svc.IndexName(&models.Material{}), q)
|
||||
if err != nil {
|
||||
ix.log().Warn("jz.bm: material search failed", "err", err)
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: %w", ErrSearchUnavailable, err)
|
||||
}
|
||||
out := MaterialCandidates{IDs: make([]uint, 0, len(res.IDs)), Hits: make(map[uint]beachcomber.Hit, len(res.IDs)), Highlights: highlights}
|
||||
for i, raw := range res.IDs {
|
||||
n, err := strconv.ParseUint(raw, 10, 64)
|
||||
if err != nil || n == 0 {
|
||||
continue
|
||||
}
|
||||
id := uint(n)
|
||||
if _, seen := out.Hits[id]; seen {
|
||||
continue
|
||||
}
|
||||
out.IDs = append(out.IDs, id)
|
||||
hit := beachcomber.Hit{ID: raw}
|
||||
if i < len(res.Hits) {
|
||||
hit = res.Hits[i]
|
||||
}
|
||||
out.Hits[id] = hit
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// materialTagSlug is the shape of a tag slug the engine filter may carry.
|
||||
var materialTagSlug = regexp.MustCompile(`^[a-z0-9-]{1,100}$`)
|
||||
|
||||
// materialFilterBy narrows the engine's candidates to the list filters. It
|
||||
// is built only from parsed integers, an enumerated type and a tag that is
|
||||
// a plain slug; any other value is left out and applied by SQL alone, which
|
||||
// applies every filter anyway.
|
||||
func materialFilterBy(req MaterialSearch) string {
|
||||
var parts []string
|
||||
if req.CourseID != 0 {
|
||||
parts = append(parts, "course_id:="+strconv.FormatUint(uint64(req.CourseID), 10))
|
||||
}
|
||||
if req.MeetingID != 0 {
|
||||
parts = append(parts, "meeting_id:="+strconv.FormatUint(uint64(req.MeetingID), 10))
|
||||
}
|
||||
if slices.Contains(models.MaterialTypes, req.Type) {
|
||||
parts = append(parts, "type:=`"+req.Type+"`")
|
||||
}
|
||||
if materialTagSlug.MatchString(req.Tag) {
|
||||
parts = append(parts, "tag_slugs:=`"+req.Tag+"`")
|
||||
}
|
||||
return strings.Join(parts, " && ")
|
||||
}
|
||||
|
||||
// Sync upserts the document of material id now. A failure is logged with
|
||||
// the id and never returned: a write never fails because of the index.
|
||||
func (ix *MaterialIndexer) Sync(ctx context.Context, db *gorm.DB, id uint) {
|
||||
ix.run(ctx, db, id, false)
|
||||
}
|
||||
|
||||
// Remove deletes the document of material id now, logging a failure like
|
||||
// Sync.
|
||||
func (ix *MaterialIndexer) Remove(ctx context.Context, db *gorm.DB, id uint) {
|
||||
ix.run(ctx, db, id, true)
|
||||
}
|
||||
|
||||
// SyncAfterCommit runs Sync once tx commits; with a nil tx, or outside a
|
||||
// transaction, it runs at once.
|
||||
func (ix *MaterialIndexer) SyncAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
|
||||
ix.afterCommit(ctx, tx, id, false)
|
||||
}
|
||||
|
||||
// RemoveAfterCommit runs Remove once tx commits, like SyncAfterCommit.
|
||||
func (ix *MaterialIndexer) RemoveAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
|
||||
ix.afterCommit(ctx, tx, id, true)
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) afterCommit(ctx context.Context, tx *gorm.DB, id uint, remove bool) {
|
||||
if ix == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
if tx == nil {
|
||||
ix.run(ctx, nil, id, remove)
|
||||
return
|
||||
}
|
||||
lagoon.AfterCommit(ctx, tx, func(ctx context.Context, db *gorm.DB) {
|
||||
ix.run(ctx, db, id, remove)
|
||||
})
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) run(ctx context.Context, db *gorm.DB, id uint, remove bool) {
|
||||
svc := ix.service()
|
||||
if svc == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
ctx = MarkMaterialSearch(ctx)
|
||||
var err error
|
||||
if remove {
|
||||
err = svc.Remove(ctx, db, &models.Material{ID: id})
|
||||
} else {
|
||||
err = svc.Sync(ctx, db, &models.Material{ID: id})
|
||||
}
|
||||
if err != nil {
|
||||
ix.log().Warn("jz.bm: material index sync failed", "material_id", id, "remove", remove, "err", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user