feat: search materials through beachcomber
- Material is a beachcomber Searchable (index jz_bm_materials) in the hand-written models/material_search.go; jz.bm requires summercms v0.6.4 - classes.MaterialIndexer asks Typesense for up to 250 candidates with body highlights; the app-wide Gate is on only for jz.bm-marked contexts - GET /_bm/api/v1/materials takes q: candidates are re-gated, counted and paged in SQL in rank order, each row with at most one plain snippet - search failure answers 503 Search is unavailable.; without q the list is unchanged
This commit is contained in:
Jakub Zych
198
internal/searchtest/typesense.go
Normal file
198
internal/searchtest/typesense.go
Normal file
@@ -0,0 +1,198 @@
|
||||
// Package searchtest is a Typesense stand-in for the jz.bm tests: an
|
||||
// httptest server that answers the collection, import, delete and search
|
||||
// endpoints the beachcomber typesense engine calls, and an app configured to
|
||||
// talk to it. Only tests import it.
|
||||
package searchtest
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
_ "git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
)
|
||||
|
||||
// Index is the collection of the materials index.
|
||||
const Index = "jz_bm_materials"
|
||||
|
||||
// HighlightStart and HighlightEnd are the highlight tags the typesense
|
||||
// engine asks for (two private-use characters).
|
||||
const (
|
||||
HighlightStart = ""
|
||||
HighlightEnd = ""
|
||||
)
|
||||
|
||||
// Hit is one search answer: a document id and, optionally, the snippet of
|
||||
// one field with the matched words between HighlightStart and HighlightEnd.
|
||||
type Hit struct {
|
||||
ID string
|
||||
Field string
|
||||
Snippet string
|
||||
}
|
||||
|
||||
// Request is one request the stub received.
|
||||
type Request struct {
|
||||
Method string
|
||||
Path string
|
||||
Query url.Values
|
||||
}
|
||||
|
||||
// Stub stands in for a Typesense server with one collection.
|
||||
type Stub struct {
|
||||
mu sync.Mutex
|
||||
// Hits answer every search, in rank order.
|
||||
Hits []Hit
|
||||
// Status, when set, answers every request with that code.
|
||||
Status int
|
||||
|
||||
exists bool
|
||||
requests []Request
|
||||
imported []map[string]any
|
||||
removed []string
|
||||
}
|
||||
|
||||
// ServeHTTP implements http.Handler.
|
||||
func (s *Stub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Query: r.URL.Query()})
|
||||
if s.Status != 0 {
|
||||
w.WriteHeader(s.Status)
|
||||
_, _ = io.WriteString(w, `{"message":"stub failure"}`)
|
||||
return
|
||||
}
|
||||
coll := "/collections/" + Index
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == coll:
|
||||
if !s.exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, `{"message":"Not Found"}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/collections":
|
||||
s.exists = true
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodDelete && r.URL.Path == coll:
|
||||
if !s.exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, `{"message":"Not Found"}`)
|
||||
return
|
||||
}
|
||||
s.exists = false
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import":
|
||||
s.exists = true
|
||||
sc := bufio.NewScanner(r.Body)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 16<<20)
|
||||
var answer []string
|
||||
for sc.Scan() {
|
||||
var doc map[string]any
|
||||
if err := json.Unmarshal(sc.Bytes(), &doc); err == nil {
|
||||
s.imported = append(s.imported, doc)
|
||||
}
|
||||
answer = append(answer, `{"success":true}`)
|
||||
}
|
||||
_, _ = io.WriteString(w, strings.Join(answer, "\n"))
|
||||
case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"):
|
||||
s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/"))
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search":
|
||||
hits := make([]string, 0, len(s.Hits))
|
||||
for _, h := range s.Hits {
|
||||
id, _ := json.Marshal(h.ID)
|
||||
highlights := "[]"
|
||||
if h.Field != "" {
|
||||
field, _ := json.Marshal(h.Field)
|
||||
snippet, _ := json.Marshal(h.Snippet)
|
||||
highlights = fmt.Sprintf(`[{"field":%s,"snippet":%s}]`, field, snippet)
|
||||
}
|
||||
hits = append(hits, fmt.Sprintf(`{"document":{"id":%s},"highlights":%s}`, id, highlights))
|
||||
}
|
||||
// found is deliberately larger than the hits, so a caller that
|
||||
// exposed it would show.
|
||||
_, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.Hits)+100, strings.Join(hits, ","))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, `{"message":"Not Found"}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Requests returns the requests received so far.
|
||||
func (s *Stub) Requests() []Request {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]Request(nil), s.requests...)
|
||||
}
|
||||
|
||||
// Searches returns the search requests received so far.
|
||||
func (s *Stub) Searches() []Request {
|
||||
var out []Request
|
||||
for _, r := range s.Requests() {
|
||||
if r.Method == http.MethodGet && strings.HasSuffix(r.Path, "/documents/search") {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Imported returns the documents upserted so far.
|
||||
func (s *Stub) Imported() []map[string]any {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]map[string]any(nil), s.imported...)
|
||||
}
|
||||
|
||||
// Removed returns the document ids deleted so far.
|
||||
func (s *Stub) Removed() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]string(nil), s.removed...)
|
||||
}
|
||||
|
||||
// Start serves stub on an httptest server closed with the test, and returns
|
||||
// the SUMMER_SEARCH__* environment that points the typesense engine at it
|
||||
// with a test-only key.
|
||||
func Start(t testing.TB, stub *Stub) []string {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(stub)
|
||||
t.Cleanup(srv.Close)
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []string{
|
||||
"SUMMER_SEARCH__DRIVER=typesense",
|
||||
"SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key",
|
||||
"SUMMER_SEARCH__TYPESENSE__HOST=" + u.Hostname(),
|
||||
"SUMMER_SEARCH__TYPESENSE__PORT=" + u.Port(),
|
||||
"SUMMER_SEARCH__TYPESENSE__PROTOCOL=" + u.Scheme,
|
||||
}
|
||||
}
|
||||
|
||||
// App returns a bare application whose config holds env (and SUMMER_ENV
|
||||
// testing) and an empty config directory.
|
||||
func App(t testing.TB, env ...string) *backpack.App {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: jz-bm-test\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := compass.Open(compass.Options{Dir: dir, Environ: append([]string{"SUMMER_ENV=testing"}, env...)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return backpack.New(cfg)
|
||||
}
|
||||
Reference in New Issue
Block a user