feat: relevance-ordered search, suggest rows and journal:reindex
- Engine search asks Typesense for the first 250 candidates, re-gates them in SQL, counts the gated rows and pages over them in rank order; pages after the first are no longer empty. An explicit sort overrides the rank. - The settings row gates engine search, so it works straight after boot. - ?fields=suggest serves id, title, slug, a plain-text excerpt, featured_image_url and published_at. - Content is indexed as plain text; Post declares SearchIndexSchema. - journal:reindex [--fresh] syncs every post into the index.
This commit is contained in:
Jakub Zych
347
search_engine_test.go
Normal file
347
search_engine_test.go
Normal file
@@ -0,0 +1,347 @@
|
||||
package journal_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-journal-plugin/console"
|
||||
"git.golem15.com/golem15/sm-journal-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/bonfire"
|
||||
)
|
||||
|
||||
const searchIndex = "golem15_journal_posts"
|
||||
|
||||
// typesenseStub stands in for a Typesense server: one collection that GET,
|
||||
// POST and DELETE /collections create and drop, imports answered with one
|
||||
// success line per document, and a search that answers with hits, the ids
|
||||
// in rank order.
|
||||
type typesenseStub struct {
|
||||
mu sync.Mutex
|
||||
exists bool
|
||||
hits []string
|
||||
requests []string
|
||||
schema map[string]any
|
||||
imported []string
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (s *typesenseStub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.requests = append(s.requests, r.Method+" "+r.URL.Path)
|
||||
coll := "/collections/" + searchIndex
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == coll:
|
||||
if !s.exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/collections":
|
||||
_ = json.NewDecoder(r.Body).Decode(&s.schema)
|
||||
s.exists = true
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodDelete && r.URL.Path == coll:
|
||||
if !s.exists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
s.exists = false
|
||||
_, _ = io.WriteString(w, `{}`)
|
||||
case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import":
|
||||
sc := bufio.NewScanner(r.Body)
|
||||
var answer []string
|
||||
for sc.Scan() {
|
||||
var doc map[string]any
|
||||
if err := json.Unmarshal(sc.Bytes(), &doc); err == nil {
|
||||
s.imported = append(s.imported, fmt.Sprint(doc["id"]))
|
||||
}
|
||||
answer = append(answer, `{"success":true}`)
|
||||
}
|
||||
_, _ = io.WriteString(w, strings.Join(answer, "\n"))
|
||||
case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"):
|
||||
s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/"))
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search":
|
||||
hits := make([]string, len(s.hits))
|
||||
for i, id := range s.hits {
|
||||
hits[i] = `{"document":{"id":"` + id + `"}}`
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.hits), strings.Join(hits, ","))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *typesenseStub) sent() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]string(nil), s.requests...)
|
||||
}
|
||||
|
||||
func newSearchEnv(t *testing.T) (*adminEnv, *typesenseStub) {
|
||||
t.Helper()
|
||||
stub := &typesenseStub{}
|
||||
srv := httptest.NewServer(stub)
|
||||
t.Cleanup(srv.Close)
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := newPostsEnv(t,
|
||||
"SUMMER_SEARCH__DRIVER=typesense",
|
||||
"SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key",
|
||||
"SUMMER_SEARCH__TYPESENSE__HOST="+u.Hostname(),
|
||||
"SUMMER_SEARCH__TYPESENSE__PORT="+u.Port(),
|
||||
"SUMMER_SEARCH__TYPESENSE__PROTOCOL="+u.Scheme,
|
||||
)
|
||||
t.Cleanup(func() { models.SetSearchGate(false) })
|
||||
return env, stub
|
||||
}
|
||||
|
||||
func setSearchGate(t *testing.T, env *adminEnv, on bool) {
|
||||
t.Helper()
|
||||
if err := env.gdb.Exec(`UPDATE golem15_journal_settings SET search_use_typesense = ? WHERE id = 1`, on).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedSearchPosts creates n published posts titled "Grzyb N" and one draft,
|
||||
// with the gate off so the writes send nothing, and returns their ids.
|
||||
func seedSearchPosts(t *testing.T, env *adminEnv, n int) (published []uint, draft uint) {
|
||||
t.Helper()
|
||||
past := time.Now().UTC().Add(-time.Hour)
|
||||
for i := 1; i <= n; i++ {
|
||||
at := past.Add(-time.Duration(i) * time.Minute)
|
||||
p := models.Post{
|
||||
Title: fmt.Sprintf("Grzyb %02d", i),
|
||||
Slug: fmt.Sprintf("grzyb-%02d", i),
|
||||
Content: "o grzybach",
|
||||
Published: true,
|
||||
PublishedAt: &at,
|
||||
}
|
||||
if err := env.gdb.Create(&p).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
published = append(published, p.ID)
|
||||
}
|
||||
d := models.Post{Title: "Grzyb szkic", Slug: "grzyb-szkic", Content: "o grzybach", Published: false}
|
||||
if err := env.gdb.Create(&d).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return published, d.ID
|
||||
}
|
||||
|
||||
type listBody struct {
|
||||
Data []map[string]any `json:"data"`
|
||||
Meta struct {
|
||||
CurrentPage int `json:"current_page"`
|
||||
LastPage int `json:"last_page"`
|
||||
Total int64 `json:"total"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
|
||||
func getList(t *testing.T, env *adminEnv, path string) listBody {
|
||||
t.Helper()
|
||||
rec := env.call(t, http.MethodGet, path, "", nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
var body listBody
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("GET %s body: %v", path, err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func slugsOf(rows []map[string]any) []string {
|
||||
out := make([]string, len(rows))
|
||||
for i, row := range rows {
|
||||
out[i], _ = row["slug"].(string)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestEngineSearchPages: the engine ranks 25 published posts and a draft;
|
||||
// the list re-gates them in SQL, keeps the engine's order across pages and
|
||||
// counts the gated rows only.
|
||||
func TestEngineSearchPages(t *testing.T) {
|
||||
env, stub := newSearchEnv(t)
|
||||
published, draft := seedSearchPosts(t, env, 25)
|
||||
// Rank: oldest first (the reverse of the date order), the draft in the
|
||||
// middle.
|
||||
var rank []string
|
||||
for i := len(published) - 1; i >= 0; i-- {
|
||||
rank = append(rank, strconv.FormatUint(uint64(published[i]), 10))
|
||||
if i == 12 {
|
||||
rank = append(rank, strconv.FormatUint(uint64(draft), 10))
|
||||
}
|
||||
}
|
||||
stub.hits = rank
|
||||
setSearchGate(t, env, true)
|
||||
|
||||
want := make([]string, 0, 25)
|
||||
for i := 25; i >= 1; i-- {
|
||||
want = append(want, fmt.Sprintf("grzyb-%02d", i))
|
||||
}
|
||||
for page, slice := range map[int][]string{1: want[:10], 2: want[10:20], 3: want[20:]} {
|
||||
body := getList(t, env, fmt.Sprintf("/_journal/api/v1/posts?search=grzyb&per_page=10&page=%d", page))
|
||||
if body.Meta.Total != 25 {
|
||||
t.Fatalf("page %d total = %d, want 25 (draft re-gated out)", page, body.Meta.Total)
|
||||
}
|
||||
if got := slugsOf(body.Data); strings.Join(got, ",") != strings.Join(slice, ",") {
|
||||
t.Fatalf("page %d = %v, want %v", page, got, slice)
|
||||
}
|
||||
}
|
||||
for _, req := range stub.sent() {
|
||||
if strings.HasSuffix(req, "/documents/search") {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("unexpected Typesense request %q", req)
|
||||
}
|
||||
|
||||
// An explicit sort overrides relevance.
|
||||
body := getList(t, env, "/_journal/api/v1/posts?search=grzyb&per_page=3&sort=title+asc")
|
||||
if got := strings.Join(slugsOf(body.Data), ","); got != "grzyb-01,grzyb-02,grzyb-03" {
|
||||
t.Fatalf("sort=title asc = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngineSearchGateOff: with Typesense configured but the gate off, the
|
||||
// list searches in SQL and sends nothing.
|
||||
func TestEngineSearchGateOff(t *testing.T) {
|
||||
env, stub := newSearchEnv(t)
|
||||
seedSearchPosts(t, env, 3)
|
||||
stub.hits = []string{"999"}
|
||||
body := getList(t, env, "/_journal/api/v1/posts?search=grzyb")
|
||||
if body.Meta.Total != 3 || len(body.Data) != 3 {
|
||||
t.Fatalf("total=%d rows=%d, want the 3 SQL matches", body.Meta.Total, len(body.Data))
|
||||
}
|
||||
if got := strings.Join(slugsOf(body.Data), ","); got != "grzyb-01,grzyb-02,grzyb-03" {
|
||||
t.Fatalf("SQL order = %s, want the date order", got)
|
||||
}
|
||||
if n := len(stub.sent()); n != 0 {
|
||||
t.Fatalf("Typesense requests = %v, want none", stub.sent())
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuggestRows: ?fields=suggest serves only the suggestion keys, with a
|
||||
// plain-text excerpt cut on a word boundary or taken from the content.
|
||||
func TestSuggestRows(t *testing.T) {
|
||||
env := newPostsEnv(t)
|
||||
past := time.Now().UTC().Add(-time.Hour)
|
||||
long := strings.Repeat("borowik szlachetny ", 20)
|
||||
for _, p := range []models.Post{
|
||||
{Title: "Z zajawką", Slug: "z-zajawka", Excerpt: "<p>Krótka <b>zajawka</b></p>", Content: "treść", Published: true, PublishedAt: &past},
|
||||
{Title: "Długa", Slug: "dluga", Excerpt: long, Content: "treść", Published: true, PublishedAt: &past},
|
||||
{Title: "Bez zajawki", Slug: "bez-zajawki", Content: "x", ContentHTML: "<h2>Kurki</h2><p>rosną & pachną</p>", Published: true, PublishedAt: &past},
|
||||
} {
|
||||
if err := env.gdb.Create(&p).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
body := getList(t, env, "/_journal/api/v1/posts?fields=suggest&per_page=5&sort=title+asc")
|
||||
if len(body.Data) != 3 || body.Meta.Total != 3 {
|
||||
t.Fatalf("rows=%d total=%d", len(body.Data), body.Meta.Total)
|
||||
}
|
||||
wantKeys := "excerpt,featured_image_url,id,published_at,slug,title"
|
||||
excerpts := map[string]string{}
|
||||
for _, row := range body.Data {
|
||||
keys := make([]string, 0, len(row))
|
||||
for k := range row {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if got := strings.Join(keys, ","); got != wantKeys {
|
||||
t.Fatalf("suggest keys = %s, want %s", got, wantKeys)
|
||||
}
|
||||
excerpts[row["slug"].(string)] = row["excerpt"].(string)
|
||||
}
|
||||
if got := excerpts["z-zajawka"]; got != "Krótka zajawka" {
|
||||
t.Fatalf("excerpt = %q", got)
|
||||
}
|
||||
if got := excerpts["bez-zajawki"]; got != "Kurki rosną & pachną" {
|
||||
t.Fatalf("content fallback = %q", got)
|
||||
}
|
||||
got := excerpts["dluga"]
|
||||
cut, ok := strings.CutSuffix(got, "…")
|
||||
if !ok || len([]rune(cut)) > 160 || !strings.HasPrefix(long, cut+" ") {
|
||||
t.Fatalf("long excerpt = %q, want under 160 runes cut on a word boundary", got)
|
||||
}
|
||||
|
||||
// Without fields=suggest the rows are the full ones.
|
||||
full := getList(t, env, "/_journal/api/v1/posts?per_page=1")
|
||||
if _, ok := full.Data[0]["content"]; !ok {
|
||||
t.Fatal("full rows lost content")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReindexFresh: --fresh drops the collection, recreates it from the
|
||||
// schema, upserts the published posts and removes the draft's document.
|
||||
func TestReindexFresh(t *testing.T) {
|
||||
env, stub := newSearchEnv(t)
|
||||
published, draft := seedSearchPosts(t, env, 3)
|
||||
stub.exists = true
|
||||
setSearchGate(t, env, true)
|
||||
|
||||
var buf strings.Builder
|
||||
out := bonfire.NewOutput(strings.NewReader(""), &buf, &buf)
|
||||
res, err := console.ReindexPosts(t.Context(), env.app, true, out)
|
||||
if err != nil {
|
||||
t.Fatalf("reindex: %v (%s)", err, buf.String())
|
||||
}
|
||||
if res.Synced != 3 || res.Removed != 1 || res.Failed != 0 {
|
||||
t.Fatalf("result = %+v, want 3 synced, 1 removed", res)
|
||||
}
|
||||
sent := stub.sent()
|
||||
if len(sent) < 3 || sent[0] != "DELETE /collections/"+searchIndex ||
|
||||
sent[1] != "GET /collections/"+searchIndex || sent[2] != "POST /collections" {
|
||||
t.Fatalf("requests = %v, want DELETE, GET, POST /collections first", sent)
|
||||
}
|
||||
fields, _ := stub.schema["fields"].([]any)
|
||||
if stub.schema["name"] != searchIndex || len(fields) != len(models.Post{}.SearchIndexSchema()["fields"].([]map[string]any)) {
|
||||
t.Fatalf("created schema = %v", stub.schema)
|
||||
}
|
||||
if len(stub.imported) != 3 {
|
||||
t.Fatalf("imported = %v, want the 3 published posts", stub.imported)
|
||||
}
|
||||
for i, id := range published {
|
||||
if stub.imported[i] != strconv.FormatUint(uint64(id), 10) {
|
||||
t.Fatalf("imported = %v, want %v", stub.imported, published)
|
||||
}
|
||||
}
|
||||
if len(stub.removed) != 1 || stub.removed[0] != strconv.FormatUint(uint64(draft), 10) {
|
||||
t.Fatalf("removed = %v, want the draft %d", stub.removed, draft)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReindexRefuses: with the gate off, or no engine configured, the
|
||||
// command errors and sends nothing.
|
||||
func TestReindexRefuses(t *testing.T) {
|
||||
env, stub := newSearchEnv(t)
|
||||
seedSearchPosts(t, env, 1)
|
||||
if _, err := console.ReindexPosts(t.Context(), env.app, true, nil); err == nil || !strings.Contains(err.Error(), "search_use_typesense") {
|
||||
t.Fatalf("gate off: err = %v", err)
|
||||
}
|
||||
if n := len(stub.sent()); n != 0 {
|
||||
t.Fatalf("gate off sent %v", stub.sent())
|
||||
}
|
||||
|
||||
plain := newPostsEnv(t)
|
||||
setSearchGate(t, plain, true)
|
||||
if _, err := console.ReindexPosts(t.Context(), plain.app, false, nil); err == nil || !strings.Contains(err.Error(), "not configured") {
|
||||
t.Fatalf("null driver: err = %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user