feat: relevance-ordered search, suggest rows and journal:reindex

- Engine search asks Typesense for the first 250 candidates, re-gates them
  in SQL, counts the gated rows and pages over them in rank order; pages
  after the first are no longer empty. An explicit sort overrides the rank.
- The settings row gates engine search, so it works straight after boot.
- ?fields=suggest serves id, title, slug, a plain-text excerpt,
  featured_image_url and published_at.
- Content is indexed as plain text; Post declares SearchIndexSchema.
- journal:reindex [--fresh] syncs every post into the index.

controllers/api/posts_search.go

@@ -3,15 +3,27 @@ package api
 import (
 	"net/http"
 	"strconv"
+	"strings"
 
 	"git.golem15.com/golem15/sm-journal-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
 	"gorm.io/gorm"
 )
 
-func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search string, publishedOnly bool, page, perPage int) ([]uint, bool) {
-	if app == nil || db == nil || !models.SearchGateEnabled() {
+// searchWindow is how many engine candidates one search re-gates in SQL.
+// The list pages over the gated rows, so a page never depends on how the
+// engine paged.
+const searchWindow = typesense.MaxPerPage
+
+// searchWithEngine returns the candidate post ids for search in the engine's
+// rank order, and whether the engine answered. It asks for the whole result
+// window (the first searchWindow hits), not one page: the caller re-gates
+// the ids in SQL, counts them and pages over them. When it reports false the
+// caller falls back to SQL search.
+func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search string, publishedOnly bool) ([]uint, bool) {
+	if app == nil || db == nil {
 		return nil, false
 	}
 	svc, err := beachcomber.From(app)
@@ -20,17 +32,27 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 	}
 	var settings models.Settings
 	if err := db.First(&settings, 1).Error; err != nil {
+		if !models.SearchGateEnabled() {
+			return nil, false
+		}
 		settings.SearchTitleWeight = 5
 		settings.SearchExcerptWeight = 3
 		settings.SearchContentWeight = 1
+	} else {
+		// The row is the gate: the in-memory one is set only once a sync
+		// has run since boot.
+		models.SetSearchGate(settings.SearchUseTypesense)
+		if !settings.SearchUseTypesense {
+			return nil, false
+		}
 	}
 	q := beachcomber.Query{
 		Q:              search,
 		QueryBy:        []string{"title", "excerpt", "content"},
 		QueryByWeights: []int{settings.SearchTitleWeight, settings.SearchExcerptWeight, settings.SearchContentWeight},
 		SortBy:         "_text_match:desc,published_at:desc",
-		Page:           page,
-		PerPage:        perPage,
+		Page:           1,
+		PerPage:        searchWindow,
 	}
 	if publishedOnly {
 		q.FilterBy = "published:=1"
@@ -40,6 +62,9 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 		pluginLog.Warn("journal: Typesense search failed, falling back to DB search", "err", err)
 		return nil, false
 	}
+	if res.Found > searchWindow {
+		pluginLog.Debug("journal: search matched more posts than one window; keeping the first", "found", res.Found, "kept", searchWindow)
+	}
 	ids := make([]uint, 0, len(res.IDs))
 	for _, raw := range res.IDs {
 		n, err := strconv.ParseUint(raw, 10, 64)
@@ -50,3 +75,18 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 	}
 	return ids, true
 }
+
+// rankOrder orders rows by their position in ids, the engine's rank, with
+// pinned posts first when pinnedFirst. The ids are parsed integers, so the
+// array literal carries no caller text.
+func rankOrder(ids []uint, pinnedFirst bool) string {
+	parts := make([]string, len(ids))
+	for i, id := range ids {
+		parts[i] = strconv.FormatUint(uint64(id), 10)
+	}
+	order := "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], id), id DESC"
+	if pinnedFirst {
+		return "is_pinned DESC, " + order
+	}
+	return order
+}