feat: relevance-ordered search, suggest rows and journal:reindex

- Engine search asks Typesense for the first 250 candidates, re-gates them in SQL, counts the gated rows and pages over them in rank order; pages after the first are no longer empty. An explicit sort overrides the rank. - The settings row gates engine search, so it works straight after boot. - ?fields=suggest serves id, title, slug, a plain-text excerpt, featured_image_url and published_at. - Content is indexed as plain text; Post declares SearchIndexSchema. - journal:reindex [--fresh] syncs every post into the index.
2026-10-30 17:59:57 +01:00
parent 5b38a3b20d
commit 08e3d82ecd
9 changed files with 768 additions and 18 deletions
--- a/controllers/api/posts.go
+++ b/controllers/api/posts.go
@@ -26,10 +26,17 @@ import (
 const (
 	defaultPerPage = 9
 	maxPerPage     = 30
+
+	// fieldsSuggest is the ?fields value of the suggestion rows.
+	fieldsSuggest = "suggest"
+	// suggestExcerptRunes is about how long a suggestion's excerpt runs.
+	suggestExcerptRunes = 160
 )

 // Index is GET /_journal/api/v1/posts. Anonymous callers see published posts
 // only; an editor with access_posts may include drafts unless ?published=true.
+// With ?fields=suggest each row is the short suggestion shape
+// (serializeSuggestion).
 func Index(app *backpack.App) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		r = optionalBackendPrincipal(app, r)
@@ -44,8 +51,11 @@ func Index(app *backpack.App) http.HandlerFunc {
 		perPage := clampPerPage(r.URL.Query().Get("per_page"))
 		page := max(1, atoiDefault(r.URL.Query().Get("page"), 1))

-		q := db.WithContext(r.Context()).Model(&models.Post{}).
-			Preload("User").Preload("Categories").Preload("Tags")
+		suggest := r.URL.Query().Get("fields") == fieldsSuggest
+		q := db.WithContext(r.Context()).Model(&models.Post{})
+		if !suggest {
+			q = q.Preload("User").Preload("Categories").Preload("Tags")
+		}

 		publishedOnly := true
 		if editor {
@@ -79,8 +89,11 @@ func Index(app *backpack.App) http.HandlerFunc {
 		if len([]rune(search)) < 3 {
 			search = ""
 		}
+		// rank is the engine's order of the candidates, nil for a SQL search.
+		var rank []uint
 		if search != "" {
-			if ids, used := searchWithEngine(app, r, db.WithContext(r.Context()), search, publishedOnly, page, perPage); used {
+			if ids, used := searchWithEngine(app, r, db.WithContext(r.Context()), search, publishedOnly); used {
+				rank = ids
 				if len(ids) == 0 {
 					q = q.Where("1 = 0")
 				} else {
@@ -102,7 +115,11 @@ func Index(app *backpack.App) http.HandlerFunc {
 		}

 		sort := strings.TrimSpace(r.URL.Query().Get("sort"))
-		order := listOrder(sort, queryBool(r.URL.Query().Get("pinned_first")))
+		pinnedFirst := queryBool(r.URL.Query().Get("pinned_first"))
+		order := listOrder(sort, pinnedFirst)
+		if len(rank) > 0 && sort == "" {
+			order = rankOrder(rank, pinnedFirst)
+		}
 		var posts []models.Post
 		if err := q.Order(order).Offset((page - 1) * perPage).Limit(perPage).Find(&posts).Error; err != nil {
 			writeAPIError(w, http.StatusInternalServerError, "Internal server error")
@@ -110,7 +127,11 @@ func Index(app *backpack.App) http.HandlerFunc {
 		}
 		rows := make([]map[string]any, 0, len(posts))
 		for i := range posts {
-			rows = append(rows, serializePost(&posts[i], db.WithContext(r.Context())))
+			if suggest {
+				rows = append(rows, serializeSuggestion(&posts[i], db.WithContext(r.Context())))
+			} else {
+				rows = append(rows, serializePost(&posts[i], db.WithContext(r.Context())))
+			}
 		}
 		wire.WriteJSON(w, http.StatusOK, lagoon.Paginate(rows, page, perPage, total))
 	}
@@ -585,10 +606,7 @@ func serializePost(p *models.Post, db *gorm.DB) map[string]any {
 		tags = append(tags, map[string]any{"id": t.ID, "name": t.Name, "slug": t.Slug})
 	}
 	featured := featuredImages(db, p)
-	var featuredURL any
-	if len(featured) > 0 {
-		featuredURL = featured[0]["url"]
-	}
+	featuredURL := firstImageURL(featured)
 	row := map[string]any{
 		"id":                 p.ID,
 		"title":              p.Title,
@@ -614,6 +632,57 @@ func serializePost(p *models.Post, db *gorm.DB) map[string]any {
 	return row
 }

+// serializeSuggestion is the ?fields=suggest row: a subset of serializePost's
+// keys with the same value formats, except that excerpt is plain text cut to
+// about suggestExcerptRunes on a word boundary, taken from the content when
+// the post has no excerpt.
+func serializeSuggestion(p *models.Post, db *gorm.DB) map[string]any {
+	if p == nil {
+		return map[string]any{}
+	}
+	return map[string]any{
+		"id":                 p.ID,
+		"title":              p.Title,
+		"slug":               p.Slug,
+		"excerpt":            suggestionExcerpt(p),
+		"featured_image_url": firstImageURL(featuredImages(db, p)),
+		"published_at":       carbonTime(p.PublishedAt),
+	}
+}
+
+func suggestionExcerpt(p *models.Post) string {
+	text := models.PlainText(p.Excerpt)
+	if text == "" {
+		body := p.ContentHTML
+		if strings.TrimSpace(body) == "" {
+			body = p.Content
+		}
+		text = models.PlainText(body)
+	}
+	return truncateWords(text, suggestExcerptRunes)
+}
+
+// truncateWords cuts s to at most n runes at the last space before the cut,
+// with an ellipsis; s is returned whole when it fits.
+func truncateWords(s string, n int) string {
+	runes := []rune(s)
+	if len(runes) <= n {
+		return s
+	}
+	cut := runes[:n]
+	if i := strings.LastIndexFunc(string(cut), unicode.IsSpace); i > 0 {
+		return strings.TrimRightFunc(string(cut)[:i], func(r rune) bool { return unicode.IsSpace(r) || unicode.IsPunct(r) }) + "…"
+	}
+	return string(cut) + "…"
+}
+
+func firstImageURL(featured []map[string]any) any {
+	if len(featured) == 0 {
+		return nil
+	}
+	return featured[0]["url"]
+}
+
 func serializeAuthor(user *cabana.BackendUser) any {
 	if user == nil {
 		return nil
--- a/controllers/api/posts_search.go
+++ b/controllers/api/posts_search.go
@@ -3,15 +3,27 @@ package api
 import (
 	"net/http"
 	"strconv"
+	"strings"

 	"git.golem15.com/golem15/sm-journal-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/backpack"
 	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
 	"gorm.io/gorm"
 )

-func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search string, publishedOnly bool, page, perPage int) ([]uint, bool) {
-	if app == nil || db == nil || !models.SearchGateEnabled() {
+// searchWindow is how many engine candidates one search re-gates in SQL.
+// The list pages over the gated rows, so a page never depends on how the
+// engine paged.
+const searchWindow = typesense.MaxPerPage
+
+// searchWithEngine returns the candidate post ids for search in the engine's
+// rank order, and whether the engine answered. It asks for the whole result
+// window (the first searchWindow hits), not one page: the caller re-gates
+// the ids in SQL, counts them and pages over them. When it reports false the
+// caller falls back to SQL search.
+func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search string, publishedOnly bool) ([]uint, bool) {
+	if app == nil || db == nil {
 		return nil, false
 	}
 	svc, err := beachcomber.From(app)
@@ -20,17 +32,27 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 	}
 	var settings models.Settings
 	if err := db.First(&settings, 1).Error; err != nil {
+		if !models.SearchGateEnabled() {
+			return nil, false
+		}
 		settings.SearchTitleWeight = 5
 		settings.SearchExcerptWeight = 3
 		settings.SearchContentWeight = 1
+	} else {
+		// The row is the gate: the in-memory one is set only once a sync
+		// has run since boot.
+		models.SetSearchGate(settings.SearchUseTypesense)
+		if !settings.SearchUseTypesense {
+			return nil, false
+		}
 	}
 	q := beachcomber.Query{
 		Q:              search,
 		QueryBy:        []string{"title", "excerpt", "content"},
 		QueryByWeights: []int{settings.SearchTitleWeight, settings.SearchExcerptWeight, settings.SearchContentWeight},
 		SortBy:         "_text_match:desc,published_at:desc",
-		Page:           page,
-		PerPage:        perPage,
+		Page:           1,
+		PerPage:        searchWindow,
 	}
 	if publishedOnly {
 		q.FilterBy = "published:=1"
@@ -40,6 +62,9 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 		pluginLog.Warn("journal: Typesense search failed, falling back to DB search", "err", err)
 		return nil, false
 	}
+	if res.Found > searchWindow {
+		pluginLog.Debug("journal: search matched more posts than one window; keeping the first", "found", res.Found, "kept", searchWindow)
+	}
 	ids := make([]uint, 0, len(res.IDs))
 	for _, raw := range res.IDs {
 		n, err := strconv.ParseUint(raw, 10, 64)
@@ -50,3 +75,18 @@ func searchWithEngine(app *backpack.App, r *http.Request, db *gorm.DB, search st
 	}
 	return ids, true
 }
+
+// rankOrder orders rows by their position in ids, the engine's rank, with
+// pinned posts first when pinnedFirst. The ids are parsed integers, so the
+// array literal carries no caller text.
+func rankOrder(ids []uint, pinnedFirst bool) string {
+	parts := make([]string, len(ids))
+	for i, id := range ids {
+		parts[i] = strconv.FormatUint(uint64(id), 10)
+	}
+	order := "array_position(ARRAY[" + strings.Join(parts, ",") + "]::bigint[], id), id DESC"
+	if pinnedFirst {
+		return "is_pinned DESC, " + order
+	}
+	return order
+}