feat: route report endpoint

- POST /_seo/api/v1/registry/routes (public, bucket seo-notfound): the
  frontend server reports its static routes with the bearer token
  golem15.seo.route_report_token, else notfound_token (constant-time;
  both empty answer 404); at most 2000 paths
- unknown paths become registry pages (origin reported, kind page, a name
  from the last segment), known ones get reported_at; paths another
  content type owns and invalid ones are skipped; nothing is deleted or
  overwritten, and the report's time marks missing pages

classes/registry.go

@@ -16,6 +16,7 @@ import (
 	"git.golem15.com/golem15/summercms/modules/lagoon"
 	"git.golem15.com/golem15/summercms/modules/pact"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 // The page registry (D-33): the site's own frontend pages (home, contact,
@@ -45,6 +46,22 @@ const (
 	MaxRouteBytes     = 2048
 )
 
+// RouteReportTokenConfigKey is the bearer token the frontend server sends
+// with its route report to POST /_seo/api/v1/registry/routes (P33-20).
+// Empty uses golem15.seo.notfound_token; both empty turn the report off.
+const RouteReportTokenConfigKey = "golem15.seo.route_report_token"
+
+// RouteReportToken is the route report's token: route_report_token, else
+// notfound_token, "" when neither is set.
+func (s *Service) RouteReportToken() string {
+	if cfg := s.Config(); cfg != nil {
+		if t := strings.TrimSpace(cfg.String(RouteReportTokenConfigKey)); t != "" {
+			return t
+		}
+	}
+	return s.NotFoundToken()
+}
+
 // PageOrigin is the origin of a page created through the management API:
 // mcp through MCP, else manual.
 func PageOrigin(ctx context.Context) string {
@@ -54,6 +71,10 @@ func PageOrigin(ctx context.Context) string {
 	return models.PageOriginManual
 }
 
+// ErrTooManyRoutes refuses a route report of more than MaxReportedRoutes
+// paths.
+var ErrTooManyRoutes = errors.New("golem15.seo: a route report holds at most 2000 paths")
+
 // The phrase keys of the registry refusals (UI-SPEC C21).
 const (
 	errPageURLRequired = "golem15.seo::lang.pages.url_required"
@@ -698,6 +719,117 @@ func (s *Service) DeletePage(ctx context.Context, tx *gorm.DB, p models.Page) er
 	return nil
 }
 
+// RouteReport is the outcome of a route report: the paths inserted as new
+// registry pages, the known pages whose reported_at moved, and the paths
+// skipped (invalid, or owned by another content type).
+type RouteReport struct {
+	Inserted   int       `json:"inserted"`
+	Updated    int       `json:"updated"`
+	Skipped    int       `json:"skipped"`
+	ReportedAt time.Time `json:"reported_at"`
+}
+
+// ReportRoutes takes the frontend server's static routes (D-33, P33-20)
+// inside tx: a path no registry page has is inserted with origin reported,
+// kind page and a name from its last segment; a known page gets
+// reported_at; a path another content type owns, or that is no valid
+// address, is skipped. Nothing is ever deleted, and no value an editor set
+// is overwritten. The settings' routes_reported_at becomes the report's
+// time, so a reported page missing from it shows as missing. More than
+// MaxReportedRoutes paths is ErrTooManyRoutes.
+func (s *Service) ReportRoutes(ctx context.Context, tx *gorm.DB, paths []string) (RouteReport, error) {
+	now := time.Now().UTC()
+	out := RouteReport{ReportedAt: now}
+	if len(paths) > MaxReportedRoutes {
+		return out, ErrTooManyRoutes
+	}
+	settings, err := SettingsRow(tx)
+	if err != nil {
+		return out, err
+	}
+	opts := NormalizeOptionsOf(settings)
+	ctx = WithVia(ctx, models.ViaAuto)
+	seen := map[string]bool{}
+	for _, raw := range paths {
+		path := strings.TrimSpace(raw)
+		if len(path) > MaxRouteBytes || !ValidPagePath(path) {
+			out.Skipped++
+			continue
+		}
+		path = normalPagePath(path, opts)
+		if seen[path] {
+			continue
+		}
+		seen[path] = true
+		page, _, err := s.PageAt(ctx, tx, path)
+		if err != nil {
+			return out, err
+		}
+		if page != nil {
+			if err := tx.Model(&models.Page{}).Where("id = ?", page.ID).UpdateColumn("reported_at", now).Error; err != nil {
+				return out, err
+			}
+			out.Updated++
+			continue
+		}
+		owned, err := s.ownerOutsideRegistry(ctx, path)
+		if err != nil {
+			return out, err
+		}
+		if owned != nil {
+			out.Skipped++
+			continue
+		}
+		p := models.Page{
+			URL:        path,
+			URLs:       lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true},
+			Title:      s.routeTitle(path),
+			Titles:     lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true},
+			Kind:       models.PageKindPage,
+			Origin:     models.PageOriginReported,
+			ReportedAt: &now,
+			CreatedAt:  &now,
+			UpdatedAt:  &now,
+		}
+		res := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&p)
+		if res.Error != nil {
+			return out, res.Error
+		}
+		if res.RowsAffected == 0 || p.ID == 0 {
+			// A concurrent report inserted it first.
+			continue
+		}
+		out.Inserted++
+		if err := s.RecordPage(ctx, tx, nil, p, false); err != nil {
+			return out, err
+		}
+		cabana.EmitRecordSaved(ctx, s.app, tx, pact.RecordSaved{Controller: PagesController, Model: PageModel, ID: p.IDText(), Created: true})
+	}
+	if err := tx.Model(&models.Settings{}).Where("id = ?", 1).UpdateColumn("routes_reported_at", now).Error; err != nil {
+		return out, err
+	}
+	return out, nil
+}
+
+// routeTitle is a reported page's name: its last path segment with
+// hyphens and underscores as spaces and the first letter capitalised; the
+// home page is named pages.home_title.
+func (s *Service) routeTitle(path string) string {
+	seg := strings.Trim(path, "/")
+	if i := strings.LastIndex(seg, "/"); i >= 0 {
+		seg = seg[i+1:]
+	}
+	if decoded, err := url.PathUnescape(seg); err == nil {
+		seg = decoded
+	}
+	seg = strings.TrimSpace(strings.NewReplacer("-", " ", "_", " ").Replace(seg))
+	if seg == "" {
+		return s.Phrase("golem15.seo::lang.pages.home_title", nil)
+	}
+	r, size := utf8.DecodeRuneInString(seg)
+	return string(unicode.ToUpper(r)) + seg[size:]
+}
+
 // PageMissing reports whether p is a reported page that the newest route
 // report (at reported) did not name.
 func PageMissing(p models.Page, reported *time.Time) bool {

config/config.yaml

@@ -23,3 +23,9 @@ notfound_max_rows: 10000
 # robots_sitemaps lists the sitemap addresses robots.txt names, one
 # "Sitemap:" line each, made absolute on the site URL.
 robots_sitemaps: ["/sitemap.xml"]
+#
+# route_report_token is the shared secret the frontend server sends as
+# "Authorization: Bearer <token>" with its route report to
+# POST /_seo/api/v1/registry/routes (the page registry). Empty uses
+# notfound_token; both empty turn the report off (it answers 404).
+route_report_token: ""

controllers/api/registry.go

@@ -3,16 +3,82 @@ package api
 import (
 	"bytes"
 	"context"
+	"crypto/subtle"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"git.golem15.com/golem15/sm-seo-plugin/classes"
 	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/wire"
 	"gorm.io/gorm"
 )
 
+// maxRouteReportBody caps the route report body: 2000 paths of 2048 bytes
+// with their JSON quoting.
+const maxRouteReportBody = classes.MaxReportedRoutes*(classes.MaxRouteBytes+8) + 1024
+
+// routeReport is the route report body.
+type routeReport struct {
+	Routes []string `json:"routes"`
+}
+
+// RouteReport serves POST /_seo/api/v1/registry/routes (D-33, P33-20): the
+// frontend server reports its static routes with "Authorization: Bearer
+// <token>", golem15.seo.route_report_token (else notfound_token), compared
+// in constant time. Unknown paths become registry pages (origin reported),
+// known ones get reported_at; nothing is deleted or overwritten. It answers
+// 200 with the counts, 401 for a missing or wrong token, 404 when no token
+// is configured (the report is off), and 422 for a malformed body or more
+// than 2000 paths.
+func RouteReport(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"})
+			return
+		}
+		want := svc.RouteReportToken()
+		if want == "" {
+			wire.WriteJSON(w, http.StatusNotFound, map[string]string{"error": errNotFound})
+			return
+		}
+		got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(want)) != 1 {
+			wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": "Unauthorized"})
+			return
+		}
+		var in routeReport
+		dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRouteReportBody))
+		if err := dec.Decode(&in); err != nil {
+			wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": errInvalidBody})
+			return
+		}
+		if len(in.Routes) > classes.MaxReportedRoutes {
+			wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]any{"error": "too_many_routes", "max": classes.MaxReportedRoutes})
+			return
+		}
+		var out classes.RouteReport
+		err := write(r.Context(), svc, func(ctx context.Context, tx *gorm.DB) error {
+			var err error
+			out, err = svc.ReportRoutes(ctx, tx, in.Routes)
+			return err
+		})
+		if err != nil {
+			if errors.Is(err, classes.ErrTooManyRoutes) {
+				wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]any{"error": "too_many_routes", "max": classes.MaxReportedRoutes})
+				return
+			}
+			log.Error("golem15.seo: route report", "error", err)
+			wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": errInternal})
+			return
+		}
+		wire.WriteJSON(w, http.StatusOK, out)
+	}
+}
+
 // ListPagesInput is the query of GET /_seo/api/v1/pages.
 type ListPagesInput struct {
 	Page    int    `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"`

lang/en/lang.yaml

@@ -392,3 +392,4 @@ pages:
     delete_selected_confirm:
         one: "Delete :count page from the registry? Its SEO tab is deleted with it."
         other: "Delete :count pages from the registry? Their SEO tabs are deleted with them."
+    home_title: Home

lang/pl/lang.yaml

@@ -406,3 +406,4 @@ pages:
         few: "Usunąć :count strony z rejestru? Ich zakładki SEO zostaną usunięte."
         many: "Usunąć :count stron z rejestru? Ich zakładki SEO zostaną usunięte."
         other: "Usunąć :count strony z rejestru? Ich zakładki SEO zostaną usunięte."
+    home_title: Strona główna

registry_test.go

@@ -182,3 +182,151 @@ func jsonID(id uint) string {
 	return string(raw)
 }
 
+// pageRow reads a registry page's stored values by its address.
+type pageRow struct {
+	ID     uint   `gorm:"column:id"`
+	URL    string `gorm:"column:url"`
+	Title  string `gorm:"column:title"`
+	Kind   string `gorm:"column:kind"`
+	Origin string `gorm:"column:origin"`
+	Note   string `gorm:"column:note"`
+	Stamp  string `gorm:"column:stamp"`
+}
+
+func (e *testEnv) pageRow(t *testing.T, url string) pageRow {
+	t.Helper()
+	var rows []pageRow
+	if err := e.db.Raw(`SELECT id, url, title, kind, origin, note, COALESCE(reported_at::text, '') AS stamp FROM golem15_seo_pages WHERE url = ?`, url).Scan(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 1 {
+		t.Fatalf("pages at %s = %d, want 1", url, len(rows))
+	}
+	return rows[0]
+}
+
+// reportRoutes posts a route report with the bearer token.
+func (e *testEnv) reportRoutes(t *testing.T, token string, routes []string) map[string]any {
+	t.Helper()
+	rec := e.expect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/registry/routes", token, map[string]any{"routes": routes})
+	if rec.Header().Get("X-Robots-Tag") != "noindex" {
+		t.Fatalf("route report headers = %v, want X-Robots-Tag noindex", rec.Header())
+	}
+	var out map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+// TestRouteReport covers the route report (D-33, P33-20, T-33-18): the
+// token, the size cap, insert-only behaviour, reported_at, the missing
+// badge and P9 warning, owned paths skipped, and that a later report never
+// deletes a page or overwrites an editor's values.
+func TestRouteReport(t *testing.T) {
+	off := bootEnv(t)
+	off.expect(t, http.StatusNotFound, http.MethodPost, "/_seo/api/v1/registry/routes", "anything", map[string]any{"routes": []string{"/"}})
+
+	// notfound_token is the fallback token.
+	fallback := bootEnv(t, "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN=nf-token")
+	fallback.reportRoutes(t, "nf-token", []string{"/x"})
+
+	env := bootEnv(t, "SUMMER_GOLEM15__SEO__ROUTE_REPORT_TOKEN=report-token", "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN=other")
+	admin := env.admin(t, true)
+	env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "wrong", map[string]any{"routes": []string{"/"}})
+	env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "other", map[string]any{"routes": []string{"/"}})
+	env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "", map[string]any{"routes": []string{"/"}})
+	many := make([]string, 2001)
+	for i := range many {
+		many[i] = "/p"
+	}
+	env.expect(t, http.StatusUnprocessableEntity, http.MethodPost, "/_seo/api/v1/registry/routes", "report-token", map[string]any{"routes": many})
+
+	env.createRegistryPage(t, admin, map[string]any{"url": map[string]any{"en": "/kontakt"}, "title": map[string]any{"en": "Kontakt"}})
+	env.createPage(t, admin, map[string]any{"title": "Blog post", "slug": "post", "published": true})
+
+	out := env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas", "/pages/post", "nope", "/o-nas"})
+	if out["inserted"] != float64(2) || out["updated"] != float64(1) || out["skipped"] != float64(2) {
+		t.Fatalf("first report = %v, want 2 inserted (/, /o-nas), 1 updated, 2 skipped", out)
+	}
+	about := env.pageRow(t, "/o-nas")
+	if about.Origin != "reported" || about.Kind != "page" || about.Title != "O nas" || about.Stamp == "" {
+		t.Fatalf("/o-nas = %+v, want origin reported, kind page, title O nas, reported", about)
+	}
+	if home := env.pageRow(t, "/"); home.Title != "Home" {
+		t.Fatalf("home = %+v, want the title Home", home)
+	}
+	contact := env.pageRow(t, "/kontakt")
+	if contact.Origin != "manual" || contact.Title != "Kontakt" || contact.Stamp == "" {
+		t.Fatalf("/kontakt = %+v, want the editor's page with reported_at", contact)
+	}
+	var owned int64
+	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_pages WHERE url = '/pages/post'`).Scan(&owned)
+	if owned != 0 {
+		t.Fatal("a path owned by a content type was inserted")
+	}
+	if got := env.resolve(t, "/o-nas"); got["action"] != "ok" {
+		t.Fatalf("resolve /o-nas = %v, want ok", got)
+	}
+
+	// The editor changes the reported page; the same report again changes
+	// only reported_at.
+	id := jsonID(about.ID)
+	env.expect(t, http.StatusOK, http.MethodPut, pagesAPI+"/"+id, admin, map[string]any{"title": map[string]any{"en": "About us"}, "kind": "offer", "note": "kept"})
+	var revisions int64
+	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions`).Scan(&revisions)
+	before := env.pageRow(t, "/o-nas")
+	out = env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas"})
+	if out["inserted"] != float64(0) || out["updated"] != float64(3) {
+		t.Fatalf("second report = %v, want 0 inserted, 3 updated", out)
+	}
+	after := env.pageRow(t, "/o-nas")
+	if after.Title != "About us" || after.Kind != "offer" || after.Note != "kept" || after.Origin != "reported" || after.Stamp == before.Stamp {
+		t.Fatalf("after the second report /o-nas = %+v (before %+v), want only reported_at changed", after, before)
+	}
+	var revisionsAfter int64
+	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions`).Scan(&revisionsAfter)
+	if revisionsAfter != revisions {
+		t.Fatalf("revisions %d -> %d: a report that inserts nothing writes no history", revisions, revisionsAfter)
+	}
+
+	// A report without /o-nas marks it missing and deletes nothing.
+	env.reportRoutes(t, "report-token", []string{"/", "/kontakt"})
+	missing := env.pageRow(t, "/o-nas")
+	if missing.Title != "About us" {
+		t.Fatalf("missing page = %+v, want it kept with its values", missing)
+	}
+	var list struct {
+		Meta struct {
+			RowStates map[string][]string `json:"row_states"`
+		} `json:"meta"`
+	}
+	if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, pagesAPI, admin, nil).Body.Bytes(), &list); err != nil {
+		t.Fatal(err)
+	}
+	if states := list.Meta.RowStates[id]; len(states) != 1 || states[0] != "negative" {
+		t.Fatalf("row states = %v, want /o-nas (%s) negative", list.Meta.RowStates, id)
+	}
+	if states := list.Meta.RowStates[jsonID(contact.ID)]; len(states) != 0 {
+		t.Fatalf("the manual page has row states %v", states)
+	}
+	status := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+id, admin, nil).Body.String()
+	if !strings.Contains(status, "summer-callout--warning") || !strings.Contains(status, "did not include this address") {
+		t.Fatalf("P9 of the missing page = %s", status)
+	}
+	if home := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+jsonID(env.pageRow(t, "/").ID), admin, nil).Body.String(); !strings.Contains(home, "Reported by the site on") || strings.Contains(home, "--warning") {
+		t.Fatalf("P9 of a reported page = %s", home)
+	}
+	if manual := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+jsonID(contact.ID), admin, nil).Body.String(); strings.Contains(manual, "summer-callout") {
+		t.Fatalf("P9 of a manual page = %s, want nothing", manual)
+	}
+	missingList := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/pages?missing=only", admin, nil).Body.String()
+	if !jsonContains(missingList, `"total":1`) || !jsonContains(missingList, `"missing":true`) {
+		t.Fatalf("missing=only list = %s", missingList)
+	}
+	// Reported again, it is back on the site.
+	env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas"})
+	if again := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/pages?missing=only", admin, nil).Body.String(); !jsonContains(again, `"total":0`) {
+		t.Fatalf("missing=only after the page is reported again = %s", again)
+	}
+}

routes.go

@@ -34,6 +34,9 @@ func (p *Plugin) Routes(r pact.Router) error {
 	r.Group("/_seo/api/v1", nil, func(g pact.Router) {
 		g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
 		g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
+		// The frontend server's route report (D-33), token-authenticated
+		// like the 404 report.
+		g.Post("/registry/routes", noindex(apicontrollers.RouteReport(p.Service)), "throttle:"+BucketNotFound)
 		// Verification files are served here only; the frontend maps them
 		// at the site root (D-103). A root route would capture every
 		// one-segment page of a site served at GET /.