feat(11.1-04): add the Backend section, the remaining Services pages and the concept map links

- docs/backend: admin controllers, forms, lists and filters, relation
  manager, users and permissions, settings, partials and widgets, admin SPA
- docs/services: storage, outbound HTTP, realtime, Web Push, search, parity
  testing and the Frontend and AJAX (not provided) page
- Examples for cabana (with testdata/docs YAML), fetchguard, lighthouse and
  its centrifugo driver, flare, beachcomber and typesense, tide; lighthouse
  and beachcomber TestDocs* regions run on their Postgres harnesses
- concept map rows link their guide pages and the not-provided rows the
  Frontend and AJAX page; index lists Backend, Database and Services
- TestDocsRequiredPages asserts the D-08 section order

modules/beachcomber/example_test.go

@@ -0,0 +1,258 @@
+package beachcomber_test
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/compass"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"gorm.io/gorm"
+)
+
+// Post is the acme.blog post model. It implements beachcomber.Searchable
+// without importing beachcomber.
+type Post struct {
+	ID        uint   `gorm:"column:id;primaryKey"`
+	BlogID    uint   `gorm:"column:blog_id"`
+	Title     string `gorm:"column:title"`
+	Published bool   `gorm:"column:published"`
+}
+
+func (Post) TableName() string { return "acme_blog_posts" }
+
+// SearchableAs is the index name, before search.prefix.
+func (Post) SearchableAs() string { return "acme_blog_posts" }
+
+// ShouldBeSearchable keeps drafts out of the index.
+func (p *Post) ShouldBeSearchable() bool { return p.Published }
+
+// ToSearchableArray builds the document from the committed row.
+func (p *Post) ToSearchableArray(ctx context.Context, db *gorm.DB) (map[string]any, error) {
+	return map[string]any{
+		"id":      strconv.FormatUint(uint64(p.ID), 10),
+		"blog_id": int64(p.BlogID),
+		"title":   p.Title,
+	}, nil
+}
+
+// memoryEngine is a toy engine: it matches Q against the title and ignores
+// FilterBy, so its answers are loose candidates, as a stale index's are.
+type memoryEngine struct {
+	mu   sync.Mutex
+	docs map[string]map[string]map[string]any // index -> id -> document
+}
+
+func (e *memoryEngine) Name() string     { return "acme-memory" }
+func (e *memoryEngine) Configured() bool { return true }
+
+func (e *memoryEngine) Upsert(ctx context.Context, index string, schema map[string]any, docs []map[string]any) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.docs[index] == nil {
+		e.docs[index] = map[string]map[string]any{}
+	}
+	for _, d := range docs {
+		e.docs[index][fmt.Sprint(d["id"])] = d
+	}
+	return nil
+}
+
+func (e *memoryEngine) Delete(ctx context.Context, index string, ids []string) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	for _, id := range ids {
+		delete(e.docs[index], id)
+	}
+	return nil
+}
+
+func (e *memoryEngine) Flush(ctx context.Context, index string) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	delete(e.docs, index)
+	return nil
+}
+
+func (e *memoryEngine) SearchIDs(ctx context.Context, index string, q beachcomber.Query) ([]string, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	ids := []string{}
+	for id, d := range e.docs[index] {
+		if strings.Contains(strings.ToLower(fmt.Sprint(d["title"])), strings.ToLower(q.Q)) {
+			ids = append(ids, id)
+		}
+	}
+	slices.Sort(ids)
+	return ids, nil
+}
+
+var memory = &memoryEngine{docs: map[string]map[string]map[string]any{}}
+
+func ExampleFrom() {
+	cfg, err := compass.Open(compass.Options{Dir: "config", Env: "development", Environ: []string{}})
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	_ = cfg.Set("search.prefix", "staging_")
+	svc, err := beachcomber.From(backpack.New(cfg)) // search.driver defaults to null
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(svc.Engine().Name(), svc.Engine().Configured(), svc.IndexName(&Post{}))
+	ids, err := svc.Engine().SearchIDs(context.Background(), svc.IndexName(&Post{}), beachcomber.Query{Q: "go"})
+	fmt.Println(ids, err)
+
+	_ = cfg.Set("search.driver", "elastic")
+	_, err = beachcomber.From(backpack.New(cfg))
+	fmt.Println(err != nil)
+	// Output:
+	// null false staging_acme_blog_posts
+	// [] <nil>
+	// true
+}
+
+// installGate turns search sync on only while the blog's search setting is
+// on; a failed read counts as off.
+func installGate(svc *beachcomber.Service) {
+	// docs:start gate
+	svc.SetGate(beachcomber.GateFunc(func(ctx context.Context, db *gorm.DB) bool {
+		var enabled bool
+		err := db.WithContext(ctx).Raw(`SELECT search_enabled FROM acme_blog_settings WHERE id = 1`).Scan(&enabled).Error
+		return err == nil && enabled
+	}))
+	// docs:end gate
+}
+
+// searchPosts answers a search in one blog.
+func searchPosts(ctx context.Context, svc *beachcomber.Service, db *gorm.DB, blogID uint, term string) ([]Post, error) {
+	// docs:start search
+	ids, err := svc.Engine().SearchIDs(ctx, svc.IndexName(&Post{}), beachcomber.Query{
+		Q:        term,
+		QueryBy:  []string{"title"},
+		FilterBy: "blog_id:=" + strconv.FormatUint(uint64(blogID), 10),
+	})
+	if err != nil {
+		return nil, err
+	}
+	// The ids are candidates from an index that may be stale or loosely
+	// filtered: re-check every one in SQL before exposing a row.
+	var posts []Post
+	err = db.WithContext(ctx).
+		Where("id IN ? AND blog_id = ? AND published", ids, blogID).
+		Order("id").
+		Find(&posts).Error
+	return posts, err
+	// docs:end search
+}
+
+// TestDocsSearch runs the gate and search regions of the Search page on the
+// package's Postgres harness.
+func TestDocsSearch(t *testing.T) {
+	app, db := beachcomber.DocsApp(t, nil)
+	for _, stmt := range []string{
+		`CREATE TABLE acme_blog_posts (id SERIAL PRIMARY KEY, blog_id INTEGER NOT NULL, title TEXT NOT NULL, published BOOLEAN NOT NULL DEFAULT FALSE)`,
+		`CREATE TABLE acme_blog_settings (id INTEGER PRIMARY KEY, search_enabled BOOLEAN NOT NULL)`,
+		`INSERT INTO acme_blog_settings VALUES (1, TRUE)`,
+	} {
+		if err := db.Exec(stmt).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	svc, err := beachcomber.From(app)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// An application selects a registered engine with search.driver; the
+	// test installs the toy engine directly, so the global engine list the
+	// package's own tests check stays unchanged.
+	beachcomber.DocsUseEngine(svc, memory)
+	installGate(svc)
+	ctx := t.Context()
+	indexed := func() int {
+		memory.mu.Lock()
+		defer memory.mu.Unlock()
+		return len(memory.docs[svc.IndexName(&Post{})])
+	}
+
+	// A committed write is indexed after commit; a rolled-back one never is.
+	create := func(p *Post, fail bool) error {
+		return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
+			if err := tx.Create(p).Error; err != nil {
+				return err
+			}
+			if fail {
+				return fmt.Errorf("rolled back")
+			}
+			return nil
+		})
+	}
+	hello := &Post{BlogID: 7, Title: "Hello Go", Published: true}
+	if err := create(hello, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := create(&Post{BlogID: 7, Title: "Go rolled back", Published: true}, true); err == nil {
+		t.Fatal("rolled-back create succeeded")
+	}
+	if err := create(&Post{BlogID: 8, Title: "Go elsewhere", Published: true}, false); err != nil {
+		t.Fatal(err)
+	}
+	if n := indexed(); n != 2 {
+		t.Fatalf("indexed %d documents, want 2", n)
+	}
+
+	// The engine answers both blogs; SQL keeps blog 7's post only.
+	posts, err := searchPosts(ctx, svc, db, 7, "go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(posts) != 1 || posts[0].ID != hello.ID {
+		t.Fatalf("search = %+v", posts)
+	}
+
+	// A write in a plain GORM transaction is not synced, so the index keeps
+	// the stale document; the SQL re-check still hides the draft.
+	if err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		return tx.Model(hello).Update("published", false).Error
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if n := indexed(); n != 2 {
+		t.Fatalf("indexed %d documents after an unsynced write, want 2", n)
+	}
+	if posts, err = searchPosts(ctx, svc, db, 7, "go"); err != nil || len(posts) != 0 {
+		t.Fatalf("search after unpublish = %+v, %v", posts, err)
+	}
+
+	// With the gate off nothing is sent.
+	if err := db.Exec(`UPDATE acme_blog_settings SET search_enabled = FALSE`).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := create(&Post{BlogID: 7, Title: "Go quietly", Published: true}, false); err != nil {
+		t.Fatal(err)
+	}
+	if n := indexed(); n != 2 {
+		t.Fatalf("indexed %d documents with the gate off, want 2", n)
+	}
+}
+
+// TestDocsDeclarations checks the Searchable methods the Search page shows.
+func TestDocsDeclarations(t *testing.T) {
+	var _ beachcomber.Searchable = &Post{}
+	p := &Post{ID: 5, BlogID: 7, Title: "Hello", Published: true}
+	if p.SearchableAs() != "acme_blog_posts" || !p.ShouldBeSearchable() {
+		t.Fatalf("SearchableAs %q, ShouldBeSearchable %v", p.SearchableAs(), p.ShouldBeSearchable())
+	}
+	doc, err := p.ToSearchableArray(t.Context(), nil)
+	if err != nil || doc["id"] != "5" || doc["blog_id"] != int64(7) {
+		t.Fatalf("ToSearchableArray = %v, %v", doc, err)
+	}
+}

modules/beachcomber/export_docs_test.go

@@ -0,0 +1,47 @@
+package beachcomber
+
+import (
+	"testing"
+
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/compass"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"gorm.io/gorm"
+)
+
+// DocsApp returns an app with the given config settings on a fresh migrated
+// database of this package's Postgres harness, with the database published,
+// for the database-backed docs examples in example_test.go. It skips under
+// -short and fails when the harness has no database, like the package's
+// other database tests.
+func DocsApp(t *testing.T, settings map[string]any) (*backpack.App, *gorm.DB) {
+	t.Helper()
+	db, dsn := migratedDB(t)
+	cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Env: "testing", Environ: []string{}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cfg.Set("database.dsn", dsn); err != nil {
+		t.Fatal(err)
+	}
+	for k, v := range settings {
+		if err := cfg.Set(k, v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	app := backpack.New(cfg)
+	gdb, err := lagoon.Use(t.Context(), db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := lagoon.Publish(app, db, gdb); err != nil {
+		t.Fatal(err)
+	}
+	return app, gdb
+}
+
+// DocsUseEngine replaces the engine of svc, for docs examples whose toy
+// engine must not join the global engine registry.
+func DocsUseEngine(svc *Service, e Engine) {
+	svc.engine = e
+}

modules/beachcomber/typesense/example_test.go

@@ -0,0 +1,49 @@
+package typesense_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"time"
+
+	"git.golem15.com/golem15/summercms/modules/beachcomber"
+	"git.golem15.com/golem15/summercms/modules/beachcomber/typesense"
+)
+
+func ExampleEngine_SearchIDs() {
+	// A stand-in Typesense node.
+	node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		fmt.Println(r.Method, r.URL.Path, "key sent:", r.Header.Get("X-TYPESENSE-API-KEY") != "")
+		fmt.Println("q", q.Get("q"), "query_by", q.Get("query_by"), "filter_by", q.Get("filter_by"))
+		fmt.Fprint(w, `{"hits":[{"document":{"id":"3"}},{"document":{"id":"1"}}]}`)
+	}))
+	defer node.Close()
+	u, _ := url.Parse(node.URL)
+	port, _ := strconv.Atoi(u.Port())
+
+	engine := typesense.New(typesense.Config{
+		APIKey:            "test-only-key", // SUMMER_SEARCH__TYPESENSE__API_KEY
+		Host:              u.Hostname(),
+		Port:              port,
+		Protocol:          "http",
+		ConnectionTimeout: 2 * time.Second,
+	})
+	ids, err := engine.SearchIDs(context.Background(), "acme_blog_posts", beachcomber.Query{
+		Q:        "go",
+		QueryBy:  []string{"title"},
+		FilterBy: "blog_id:=7",
+	})
+	fmt.Println(ids, err)
+
+	// Without an API key the engine is not configured and nothing is sent.
+	fmt.Println(typesense.New(typesense.Config{}).Configured())
+	// Output:
+	// GET /collections/acme_blog_posts/documents/search key sent: true
+	// q go query_by title filter_by blog_id:=7
+	// [3 1] <nil>
+	// false
+}