sm-journal-plugin/search_engine_test.go

package journal_test

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"git.golem15.com/golem15/sm-journal-plugin/console"
	"git.golem15.com/golem15/sm-journal-plugin/models"
	"git.golem15.com/golem15/summercms/modules/bonfire"
)

const searchIndex = "golem15_journal_posts"

// typesenseStub stands in for a Typesense server: one collection that GET,
// POST and DELETE /collections create and drop, imports answered with one
// success line per document, and a search that answers with hits, the ids
// in rank order.
type typesenseStub struct {
	mu       sync.Mutex
	exists   bool
	hits     []string
	requests []string
	schema   map[string]any
	imported []string
	removed  []string
}

func (s *typesenseStub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.requests = append(s.requests, r.Method+" "+r.URL.Path)
	coll := "/collections/" + searchIndex
	switch {
	case r.Method == http.MethodGet && r.URL.Path == coll:
		if !s.exists {
			w.WriteHeader(http.StatusNotFound)
			return
		}
		_, _ = io.WriteString(w, `{}`)
	case r.Method == http.MethodPost && r.URL.Path == "/collections":
		_ = json.NewDecoder(r.Body).Decode(&s.schema)
		s.exists = true
		w.WriteHeader(http.StatusCreated)
		_, _ = io.WriteString(w, `{}`)
	case r.Method == http.MethodDelete && r.URL.Path == coll:
		if !s.exists {
			w.WriteHeader(http.StatusNotFound)
			return
		}
		s.exists = false
		_, _ = io.WriteString(w, `{}`)
	case r.Method == http.MethodPost && r.URL.Path == coll+"/documents/import":
		sc := bufio.NewScanner(r.Body)
		var answer []string
		for sc.Scan() {
			var doc map[string]any
			if err := json.Unmarshal(sc.Bytes(), &doc); err == nil {
				s.imported = append(s.imported, fmt.Sprint(doc["id"]))
			}
			answer = append(answer, `{"success":true}`)
		}
		_, _ = io.WriteString(w, strings.Join(answer, "\n"))
	case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, coll+"/documents/"):
		s.removed = append(s.removed, strings.TrimPrefix(r.URL.Path, coll+"/documents/"))
		w.WriteHeader(http.StatusNotFound)
	case r.Method == http.MethodGet && r.URL.Path == coll+"/documents/search":
		hits := make([]string, len(s.hits))
		for i, id := range s.hits {
			hits[i] = `{"document":{"id":"` + id + `"}}`
		}
		_, _ = fmt.Fprintf(w, `{"found":%d,"hits":[%s]}`, len(s.hits), strings.Join(hits, ","))
	default:
		w.WriteHeader(http.StatusNotFound)
	}
}

func (s *typesenseStub) sent() []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return append([]string(nil), s.requests...)
}

func newSearchEnv(t *testing.T) (*adminEnv, *typesenseStub) {
	t.Helper()
	stub := &typesenseStub{}
	srv := httptest.NewServer(stub)
	t.Cleanup(srv.Close)
	u, err := url.Parse(srv.URL)
	if err != nil {
		t.Fatal(err)
	}
	env := newPostsEnv(t,
		"SUMMER_SEARCH__DRIVER=typesense",
		"SUMMER_SEARCH__TYPESENSE__API_KEY=test-only-typesense-key",
		"SUMMER_SEARCH__TYPESENSE__HOST="+u.Hostname(),
		"SUMMER_SEARCH__TYPESENSE__PORT="+u.Port(),
		"SUMMER_SEARCH__TYPESENSE__PROTOCOL="+u.Scheme,
	)
	t.Cleanup(func() { models.SetSearchGate(false) })
	return env, stub
}

func setSearchGate(t *testing.T, env *adminEnv, on bool) {
	t.Helper()
	if err := env.gdb.Exec(`UPDATE golem15_journal_settings SET search_use_typesense = ? WHERE id = 1`, on).Error; err != nil {
		t.Fatal(err)
	}
}

// seedSearchPosts creates n published posts titled "Grzyb N" and one draft,
// with the gate off so the writes send nothing, and returns their ids.
func seedSearchPosts(t *testing.T, env *adminEnv, n int) (published []uint, draft uint) {
	t.Helper()
	past := time.Now().UTC().Add(-time.Hour)
	for i := 1; i <= n; i++ {
		at := past.Add(-time.Duration(i) * time.Minute)
		p := models.Post{
			Title:       fmt.Sprintf("Grzyb %02d", i),
			Slug:        fmt.Sprintf("grzyb-%02d", i),
			Content:     "o grzybach",
			Published:   true,
			PublishedAt: &at,
		}
		if err := env.gdb.Create(&p).Error; err != nil {
			t.Fatal(err)
		}
		published = append(published, p.ID)
	}
	d := models.Post{Title: "Grzyb szkic", Slug: "grzyb-szkic", Content: "o grzybach", Published: false}
	if err := env.gdb.Create(&d).Error; err != nil {
		t.Fatal(err)
	}
	return published, d.ID
}

type listBody struct {
	Data []map[string]any `json:"data"`
	Meta struct {
		CurrentPage int   `json:"current_page"`
		LastPage    int   `json:"last_page"`
		Total       int64 `json:"total"`
	} `json:"meta"`
}

func getList(t *testing.T, env *adminEnv, path string) listBody {
	t.Helper()
	rec := env.call(t, http.MethodGet, path, "", nil)
	if rec.Code != http.StatusOK {
		t.Fatalf("GET %s = %d: %s", path, rec.Code, rec.Body.String())
	}
	var body listBody
	if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
		t.Fatalf("GET %s body: %v", path, err)
	}
	return body
}

func slugsOf(rows []map[string]any) []string {
	out := make([]string, len(rows))
	for i, row := range rows {
		out[i], _ = row["slug"].(string)
	}
	return out
}

// TestEngineSearchPages: the engine ranks 25 published posts and a draft;
// the list re-gates them in SQL, keeps the engine's order across pages and
// counts the gated rows only.
func TestEngineSearchPages(t *testing.T) {
	env, stub := newSearchEnv(t)
	published, draft := seedSearchPosts(t, env, 25)
	// Rank: oldest first (the reverse of the date order), the draft in the
	// middle.
	var rank []string
	for i := len(published) - 1; i >= 0; i-- {
		rank = append(rank, strconv.FormatUint(uint64(published[i]), 10))
		if i == 12 {
			rank = append(rank, strconv.FormatUint(uint64(draft), 10))
		}
	}
	stub.hits = rank
	setSearchGate(t, env, true)

	want := make([]string, 0, 25)
	for i := 25; i >= 1; i-- {
		want = append(want, fmt.Sprintf("grzyb-%02d", i))
	}
	for page, slice := range map[int][]string{1: want[:10], 2: want[10:20], 3: want[20:]} {
		body := getList(t, env, fmt.Sprintf("/_journal/api/v1/posts?search=grzyb&per_page=10&page=%d", page))
		if body.Meta.Total != 25 {
			t.Fatalf("page %d total = %d, want 25 (draft re-gated out)", page, body.Meta.Total)
		}
		if got := slugsOf(body.Data); strings.Join(got, ",") != strings.Join(slice, ",") {
			t.Fatalf("page %d = %v, want %v", page, got, slice)
		}
	}
	for _, req := range stub.sent() {
		if strings.HasSuffix(req, "/documents/search") {
			continue
		}
		t.Fatalf("unexpected Typesense request %q", req)
	}

	// An explicit sort overrides relevance.
	body := getList(t, env, "/_journal/api/v1/posts?search=grzyb&per_page=3&sort=title+asc")
	if got := strings.Join(slugsOf(body.Data), ","); got != "grzyb-01,grzyb-02,grzyb-03" {
		t.Fatalf("sort=title asc = %s", got)
	}
}

// TestEngineSearchGateOff: with Typesense configured but the gate off, the
// list searches in SQL and sends nothing.
func TestEngineSearchGateOff(t *testing.T) {
	env, stub := newSearchEnv(t)
	seedSearchPosts(t, env, 3)
	stub.hits = []string{"999"}
	body := getList(t, env, "/_journal/api/v1/posts?search=grzyb")
	if body.Meta.Total != 3 || len(body.Data) != 3 {
		t.Fatalf("total=%d rows=%d, want the 3 SQL matches", body.Meta.Total, len(body.Data))
	}
	if got := strings.Join(slugsOf(body.Data), ","); got != "grzyb-01,grzyb-02,grzyb-03" {
		t.Fatalf("SQL order = %s, want the date order", got)
	}
	if n := len(stub.sent()); n != 0 {
		t.Fatalf("Typesense requests = %v, want none", stub.sent())
	}
}

// TestSuggestRows: ?fields=suggest serves only the suggestion keys, with a
// plain-text excerpt cut on a word boundary or taken from the content.
func TestSuggestRows(t *testing.T) {
	env := newPostsEnv(t)
	past := time.Now().UTC().Add(-time.Hour)
	long := strings.Repeat("borowik szlachetny ", 20)
	for _, p := range []models.Post{
		{Title: "Z zajawką", Slug: "z-zajawka", Excerpt: "<p>Krótka <b>zajawka</b></p>", Content: "treść", Published: true, PublishedAt: &past},
		{Title: "Długa", Slug: "dluga", Excerpt: long, Content: "treść", Published: true, PublishedAt: &past},
		{Title: "Bez zajawki", Slug: "bez-zajawki", Content: "x", ContentHTML: "<h2>Kurki</h2><p>rosną &amp; pachną</p>", Published: true, PublishedAt: &past},
	} {
		if err := env.gdb.Create(&p).Error; err != nil {
			t.Fatal(err)
		}
	}
	body := getList(t, env, "/_journal/api/v1/posts?fields=suggest&per_page=5&sort=title+asc")
	if len(body.Data) != 3 || body.Meta.Total != 3 {
		t.Fatalf("rows=%d total=%d", len(body.Data), body.Meta.Total)
	}
	wantKeys := "excerpt,featured_image_url,id,published_at,slug,title"
	excerpts := map[string]string{}
	for _, row := range body.Data {
		keys := make([]string, 0, len(row))
		for k := range row {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		if got := strings.Join(keys, ","); got != wantKeys {
			t.Fatalf("suggest keys = %s, want %s", got, wantKeys)
		}
		excerpts[row["slug"].(string)] = row["excerpt"].(string)
	}
	if got := excerpts["z-zajawka"]; got != "Krótka zajawka" {
		t.Fatalf("excerpt = %q", got)
	}
	if got := excerpts["bez-zajawki"]; got != "Kurki rosną & pachną" {
		t.Fatalf("content fallback = %q", got)
	}
	got := excerpts["dluga"]
	cut, ok := strings.CutSuffix(got, "…")
	if !ok || len([]rune(cut)) > 160 || !strings.HasPrefix(long, cut+" ") {
		t.Fatalf("long excerpt = %q, want under 160 runes cut on a word boundary", got)
	}

	// Without fields=suggest the rows are the full ones.
	full := getList(t, env, "/_journal/api/v1/posts?per_page=1")
	if _, ok := full.Data[0]["content"]; !ok {
		t.Fatal("full rows lost content")
	}
}

// TestReindexFresh: --fresh drops the collection, recreates it from the
// schema, upserts the published posts and removes the draft's document.
func TestReindexFresh(t *testing.T) {
	env, stub := newSearchEnv(t)
	published, draft := seedSearchPosts(t, env, 3)
	stub.exists = true
	setSearchGate(t, env, true)

	var buf strings.Builder
	out := bonfire.NewOutput(strings.NewReader(""), &buf, &buf)
	res, err := console.ReindexPosts(t.Context(), env.app, true, out)
	if err != nil {
		t.Fatalf("reindex: %v (%s)", err, buf.String())
	}
	if res.Synced != 3 || res.Removed != 1 || res.Failed != 0 {
		t.Fatalf("result = %+v, want 3 synced, 1 removed", res)
	}
	sent := stub.sent()
	if len(sent) < 3 || sent[0] != "DELETE /collections/"+searchIndex ||
		sent[1] != "GET /collections/"+searchIndex || sent[2] != "POST /collections" {
		t.Fatalf("requests = %v, want DELETE, GET, POST /collections first", sent)
	}
	fields, _ := stub.schema["fields"].([]any)
	if stub.schema["name"] != searchIndex || len(fields) != len(models.Post{}.SearchIndexSchema()["fields"].([]map[string]any)) {
		t.Fatalf("created schema = %v", stub.schema)
	}
	if len(stub.imported) != 3 {
		t.Fatalf("imported = %v, want the 3 published posts", stub.imported)
	}
	for i, id := range published {
		if stub.imported[i] != strconv.FormatUint(uint64(id), 10) {
			t.Fatalf("imported = %v, want %v", stub.imported, published)
		}
	}
	if len(stub.removed) != 1 || stub.removed[0] != strconv.FormatUint(uint64(draft), 10) {
		t.Fatalf("removed = %v, want the draft %d", stub.removed, draft)
	}
}

// TestReindexRefuses: with the gate off, or no engine configured, the
// command errors and sends nothing.
func TestReindexRefuses(t *testing.T) {
	env, stub := newSearchEnv(t)
	seedSearchPosts(t, env, 1)
	if _, err := console.ReindexPosts(t.Context(), env.app, true, nil); err == nil || !strings.Contains(err.Error(), "search_use_typesense") {
		t.Fatalf("gate off: err = %v", err)
	}
	if n := len(stub.sent()); n != 0 {
		t.Fatalf("gate off sent %v", stub.sent())
	}

	plain := newPostsEnv(t)
	setSearchGate(t, plain, true)
	if _, err := console.ReindexPosts(t.Context(), plain.app, false, nil); err == nil || !strings.Contains(err.Error(), "not configured") {
		t.Fatalf("null driver: err = %v", err)
	}
}