feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
Jakub Zych
256
classes/material_index.go
Normal file
256
classes/material_index.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/beachcomber"
|
||||
"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// materialSearchKey marks a context whose syncs belong to jz.bm.
|
||||
type materialSearchKey struct{}
|
||||
|
||||
// MarkMaterialSearch returns ctx marked as a jz.bm material sync. The
|
||||
// app-wide beachcomber Gate (MaterialSearchGate) is on only for a marked
|
||||
// context, so every jz.bm Sync or Remove passes one.
|
||||
func MarkMaterialSearch(ctx context.Context) context.Context {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return context.WithValue(ctx, materialSearchKey{}, true)
|
||||
}
|
||||
|
||||
// MaterialSearchGate is the app-wide beachcomber Gate jz.bm installs. The
|
||||
// Gate sees only (ctx, db), not the model, so it is on only for a context
|
||||
// marked by MarkMaterialSearch: golem15.chatchannels syncs its messages
|
||||
// with unmarked contexts, and chat messages and DMs never reach the index.
|
||||
// The automatic callbacks of material writes are off as well; every jz.bm
|
||||
// write path syncs explicitly.
|
||||
func MaterialSearchGate() beachcomber.Gate {
|
||||
return beachcomber.GateFunc(func(ctx context.Context, _ *gorm.DB) bool {
|
||||
return ctx != nil && ctx.Value(materialSearchKey{}) != nil
|
||||
})
|
||||
}
|
||||
|
||||
// MaterialSearchWindow is how many engine candidates one search re-gates in
|
||||
// SQL. The list counts and pages over the gated rows, so a page never
|
||||
// depends on how the engine paged.
|
||||
const MaterialSearchWindow = typesense.MaxPerPage
|
||||
|
||||
// MaterialQueryBy are the document fields a query is matched against, and
|
||||
// MaterialQueryWeights their weights in the same order: the title over the
|
||||
// tags, the source and the course, then the body, the link and file text.
|
||||
var (
|
||||
MaterialQueryBy = []string{"title", "tags", "source", "course", "url", "body", "file_texts"}
|
||||
MaterialQueryWeights = []int{6, 4, 3, 3, 1, 2, 1}
|
||||
)
|
||||
|
||||
// materialHighlightFields are the fields snippets are cut from.
|
||||
var materialHighlightFields = []string{"body", "file_texts"}
|
||||
|
||||
// ErrSearchUnavailable is returned by Search when the engine is not
|
||||
// configured or the search failed. The API answers it with a 503.
|
||||
var ErrSearchUnavailable = errors.New("jz.bm: search is unavailable")
|
||||
|
||||
// MaterialSearch is one material search: the query text and the validated
|
||||
// filters of the list.
|
||||
type MaterialSearch struct {
|
||||
Q string
|
||||
CourseID uint
|
||||
MeetingID uint
|
||||
Type string
|
||||
Tag string
|
||||
}
|
||||
|
||||
// MaterialCandidates are the engine's answer: the candidate ids in rank
|
||||
// order and the Hit of each. Both are candidates only: the caller re-gates
|
||||
// the ids in SQL and reads the Hit of a row only once SQL returned it.
|
||||
// Highlights reports whether the engine was asked for snippets.
|
||||
type MaterialCandidates struct {
|
||||
IDs []uint
|
||||
Hits map[uint]beachcomber.Hit
|
||||
Highlights bool
|
||||
}
|
||||
|
||||
// MaterialIndexer searches the materials index and keeps it current. A nil
|
||||
// *MaterialIndexer is valid: Search answers ErrSearchUnavailable and the
|
||||
// syncs do nothing.
|
||||
type MaterialIndexer struct {
|
||||
// Service returns the app's beachcomber service, or nil.
|
||||
Service func() *beachcomber.Service
|
||||
// Log is where failures are logged; nil means slog.Default.
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// NewMaterialIndexer returns the indexer of app. The service is resolved on
|
||||
// each call, so it may be built after the indexer.
|
||||
func NewMaterialIndexer(app *backpack.App) *MaterialIndexer {
|
||||
return &MaterialIndexer{
|
||||
Service: func() *beachcomber.Service {
|
||||
if app == nil {
|
||||
return nil
|
||||
}
|
||||
svc, err := beachcomber.From(app)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return svc
|
||||
},
|
||||
Log: diary.For("jz.bm"),
|
||||
}
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) service() *beachcomber.Service {
|
||||
if ix == nil || ix.Service == nil {
|
||||
return nil
|
||||
}
|
||||
return ix.Service()
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) log() *slog.Logger {
|
||||
if ix == nil || ix.Log == nil {
|
||||
return slog.Default()
|
||||
}
|
||||
return ix.Log
|
||||
}
|
||||
|
||||
// Search asks the engine for up to MaterialSearchWindow candidates of req.
|
||||
// It answers ErrSearchUnavailable (wrapping the cause) when the service or
|
||||
// its engine is missing, the engine is not configured or the call fails.
|
||||
// The query text is never logged.
|
||||
func (ix *MaterialIndexer) Search(ctx context.Context, req MaterialSearch) (MaterialCandidates, error) {
|
||||
svc := ix.service()
|
||||
if svc == nil {
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: no search service", ErrSearchUnavailable)
|
||||
}
|
||||
engine := svc.Engine()
|
||||
if engine == nil || !engine.Configured() {
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: the search engine is not configured", ErrSearchUnavailable)
|
||||
}
|
||||
q := beachcomber.Query{
|
||||
Q: req.Q,
|
||||
QueryBy: MaterialQueryBy,
|
||||
QueryByWeights: MaterialQueryWeights,
|
||||
FilterBy: materialFilterBy(req),
|
||||
SortBy: "_text_match:desc,published_at:desc",
|
||||
Page: 1,
|
||||
PerPage: MaterialSearchWindow,
|
||||
ExcludeFields: []string{"body", "file_texts"},
|
||||
}
|
||||
highlights := beachcomber.SupportsHighlights(engine)
|
||||
if highlights {
|
||||
q.Highlight = &beachcomber.Highlight{Fields: materialHighlightFields, AffixTokens: 12, MaxSnippetsPerArray: 1}
|
||||
}
|
||||
res, err := beachcomber.SearchPage(ctx, engine, svc.IndexName(&models.Material{}), q)
|
||||
if err != nil {
|
||||
ix.log().Warn("jz.bm: material search failed", "err", err)
|
||||
return MaterialCandidates{}, fmt.Errorf("%w: %w", ErrSearchUnavailable, err)
|
||||
}
|
||||
out := MaterialCandidates{IDs: make([]uint, 0, len(res.IDs)), Hits: make(map[uint]beachcomber.Hit, len(res.IDs)), Highlights: highlights}
|
||||
for i, raw := range res.IDs {
|
||||
n, err := strconv.ParseUint(raw, 10, 64)
|
||||
if err != nil || n == 0 {
|
||||
continue
|
||||
}
|
||||
id := uint(n)
|
||||
if _, seen := out.Hits[id]; seen {
|
||||
continue
|
||||
}
|
||||
out.IDs = append(out.IDs, id)
|
||||
hit := beachcomber.Hit{ID: raw}
|
||||
if i < len(res.Hits) {
|
||||
hit = res.Hits[i]
|
||||
}
|
||||
out.Hits[id] = hit
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// materialTagSlug is the shape of a tag slug the engine filter may carry.
|
||||
var materialTagSlug = regexp.MustCompile(`^[a-z0-9-]{1,100}$`)
|
||||
|
||||
// materialFilterBy narrows the engine's candidates to the list filters. It
|
||||
// is built only from parsed integers, an enumerated type and a tag that is
|
||||
// a plain slug; any other value is left out and applied by SQL alone, which
|
||||
// applies every filter anyway.
|
||||
func materialFilterBy(req MaterialSearch) string {
|
||||
var parts []string
|
||||
if req.CourseID != 0 {
|
||||
parts = append(parts, "course_id:="+strconv.FormatUint(uint64(req.CourseID), 10))
|
||||
}
|
||||
if req.MeetingID != 0 {
|
||||
parts = append(parts, "meeting_id:="+strconv.FormatUint(uint64(req.MeetingID), 10))
|
||||
}
|
||||
if slices.Contains(models.MaterialTypes, req.Type) {
|
||||
parts = append(parts, "type:=`"+req.Type+"`")
|
||||
}
|
||||
if materialTagSlug.MatchString(req.Tag) {
|
||||
parts = append(parts, "tag_slugs:=`"+req.Tag+"`")
|
||||
}
|
||||
return strings.Join(parts, " && ")
|
||||
}
|
||||
|
||||
// Sync upserts the document of material id now. A failure is logged with
|
||||
// the id and never returned: a write never fails because of the index.
|
||||
func (ix *MaterialIndexer) Sync(ctx context.Context, db *gorm.DB, id uint) {
|
||||
ix.run(ctx, db, id, false)
|
||||
}
|
||||
|
||||
// Remove deletes the document of material id now, logging a failure like
|
||||
// Sync.
|
||||
func (ix *MaterialIndexer) Remove(ctx context.Context, db *gorm.DB, id uint) {
|
||||
ix.run(ctx, db, id, true)
|
||||
}
|
||||
|
||||
// SyncAfterCommit runs Sync once tx commits; with a nil tx, or outside a
|
||||
// transaction, it runs at once.
|
||||
func (ix *MaterialIndexer) SyncAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
|
||||
ix.afterCommit(ctx, tx, id, false)
|
||||
}
|
||||
|
||||
// RemoveAfterCommit runs Remove once tx commits, like SyncAfterCommit.
|
||||
func (ix *MaterialIndexer) RemoveAfterCommit(ctx context.Context, tx *gorm.DB, id uint) {
|
||||
ix.afterCommit(ctx, tx, id, true)
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) afterCommit(ctx context.Context, tx *gorm.DB, id uint, remove bool) {
|
||||
if ix == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
if tx == nil {
|
||||
ix.run(ctx, nil, id, remove)
|
||||
return
|
||||
}
|
||||
lagoon.AfterCommit(ctx, tx, func(ctx context.Context, db *gorm.DB) {
|
||||
ix.run(ctx, db, id, remove)
|
||||
})
|
||||
}
|
||||
|
||||
func (ix *MaterialIndexer) run(ctx context.Context, db *gorm.DB, id uint, remove bool) {
|
||||
svc := ix.service()
|
||||
if svc == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
ctx = MarkMaterialSearch(ctx)
|
||||
var err error
|
||||
if remove {
|
||||
err = svc.Remove(ctx, db, &models.Material{ID: id})
|
||||
} else {
|
||||
err = svc.Sync(ctx, db, &models.Material{ID: id})
|
||||
}
|
||||
if err != nil {
|
||||
ix.log().Warn("jz.bm: material index sync failed", "material_id", id, "remove", remove, "err", err)
|
||||
}
|
||||
}
|
||||
121
classes/material_index_test.go
Normal file
121
classes/material_index_test.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package classes_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/jakub/sm-bm-plugin/classes"
|
||||
"git.golem15.com/jakub/sm-bm-plugin/internal/searchtest"
|
||||
)
|
||||
|
||||
func TestMaterialSearchGate(t *testing.T) {
|
||||
gate := classes.MaterialSearchGate()
|
||||
if !gate.Enabled(classes.MarkMaterialSearch(context.Background()), nil) {
|
||||
t.Error("a marked context is off")
|
||||
}
|
||||
if gate.Enabled(context.Background(), nil) {
|
||||
t.Error("an unmarked context (a chat message sync) is on")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterialSearchRequest(t *testing.T) {
|
||||
t.Run("unavailable", materialSearchUnavailable)
|
||||
stub := &searchtest.Stub{Hits: []searchtest.Hit{
|
||||
{ID: "12", Field: "body", Snippet: "o " + searchtest.HighlightStart + "globalizacji" + searchtest.HighlightEnd + " dziś"},
|
||||
{ID: "abc"},
|
||||
{ID: "0"},
|
||||
{ID: "5"},
|
||||
}}
|
||||
ix := classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, stub)...))
|
||||
got, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "globalizacja", CourseID: 7, MeetingID: 3, Type: "pdf", Tag: "globalizacja"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
searches := stub.Searches()
|
||||
if len(searches) != 1 {
|
||||
t.Fatalf("searches = %d, want 1", len(searches))
|
||||
}
|
||||
q := searches[0].Query
|
||||
want := map[string]string{
|
||||
"q": "globalizacja",
|
||||
"query_by": "title,tags,source,course,url,body,file_texts",
|
||||
"query_by_weights": "6,4,3,3,1,2,1",
|
||||
"filter_by": "course_id:=7 && meeting_id:=3 && type:=`pdf` && tag_slugs:=`globalizacja`",
|
||||
"sort_by": "_text_match:desc,published_at:desc",
|
||||
"page": "1",
|
||||
"per_page": "250",
|
||||
"exclude_fields": "body,file_texts",
|
||||
"highlight_fields": "body,file_texts",
|
||||
}
|
||||
for k, v := range want {
|
||||
if q.Get(k) != v {
|
||||
t.Errorf("%s = %q, want %q", k, q.Get(k), v)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got.IDs, []uint{12, 5}) {
|
||||
t.Errorf("ids = %v, want [12 5]", got.IDs)
|
||||
}
|
||||
if !got.Highlights {
|
||||
t.Error("Highlights is false against the typesense engine")
|
||||
}
|
||||
if h := got.Hits[12]; len(h.Highlights) != 1 || h.Highlights[0].Field != "body" {
|
||||
t.Errorf("hit 12 = %+v", h)
|
||||
} else if segs := h.Highlights[0].Segments; len(segs) != 3 || segs[1].Text != "globalizacji" || !segs[1].Match {
|
||||
t.Errorf("hit 12 segments = %+v", segs)
|
||||
}
|
||||
if _, ok := got.Hits[5]; !ok {
|
||||
t.Error("hit 5 is missing")
|
||||
}
|
||||
if len(got.Hits) != 2 {
|
||||
t.Errorf("hits = %v, want only 12 and 5", got.Hits)
|
||||
}
|
||||
|
||||
// A tag that is not a plain slug and an unknown type stay out of the
|
||||
// engine filter; SQL applies them.
|
||||
if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "x`y", Type: "exe"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
searches = stub.Searches()
|
||||
if f := searches[len(searches)-1].Query.Get("filter_by"); f != "" {
|
||||
t.Errorf("filter_by = %q, want empty", f)
|
||||
}
|
||||
if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx", Tag: "zła-Wielka"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
searches = stub.Searches()
|
||||
if f := searches[len(searches)-1].Query.Get("filter_by"); strings.Contains(f, "tag_slugs") {
|
||||
t.Errorf("filter_by = %q carries a tag that is not a slug", f)
|
||||
}
|
||||
}
|
||||
|
||||
// materialSearchUnavailable runs under TestMaterialSearchRequest.
|
||||
func materialSearchUnavailable(t *testing.T) {
|
||||
// No key: the engine is not configured and nothing is sent.
|
||||
stub := &searchtest.Stub{}
|
||||
env := searchtest.Start(t, stub)
|
||||
noKey := slices.DeleteFunc(slices.Clone(env), func(s string) bool { return strings.Contains(s, "API_KEY") })
|
||||
ix := classes.NewMaterialIndexer(searchtest.App(t, noKey...))
|
||||
if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
|
||||
t.Errorf("unconfigured engine: err = %v, want ErrSearchUnavailable", err)
|
||||
}
|
||||
if n := len(stub.Requests()); n != 0 {
|
||||
t.Errorf("unconfigured engine sent %d requests", n)
|
||||
}
|
||||
|
||||
failing := &searchtest.Stub{Status: http.StatusInternalServerError}
|
||||
ix = classes.NewMaterialIndexer(searchtest.App(t, searchtest.Start(t, failing)...))
|
||||
if _, err := ix.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
|
||||
t.Errorf("failing engine: err = %v, want ErrSearchUnavailable", err)
|
||||
}
|
||||
|
||||
var none *classes.MaterialIndexer
|
||||
if _, err := none.Search(context.Background(), classes.MaterialSearch{Q: "xx"}); !errors.Is(err, classes.ErrSearchUnavailable) {
|
||||
t.Errorf("nil indexer: err = %v, want ErrSearchUnavailable", err)
|
||||
}
|
||||
none.Sync(context.Background(), nil, 1)
|
||||
none.RemoveAfterCommit(context.Background(), nil, 1)
|
||||
}
|
||||
Reference in New Issue
Block a user