diff --git a/classes/registry.go b/classes/registry.go index 16ddbf1..05e965d 100644 --- a/classes/registry.go +++ b/classes/registry.go @@ -16,6 +16,7 @@ import ( "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // The page registry (D-33): the site's own frontend pages (home, contact, @@ -45,6 +46,22 @@ const ( MaxRouteBytes = 2048 ) +// RouteReportTokenConfigKey is the bearer token the frontend server sends +// with its route report to POST /_seo/api/v1/registry/routes (P33-20). +// Empty uses golem15.seo.notfound_token; both empty turn the report off. +const RouteReportTokenConfigKey = "golem15.seo.route_report_token" + +// RouteReportToken is the route report's token: route_report_token, else +// notfound_token, "" when neither is set. +func (s *Service) RouteReportToken() string { + if cfg := s.Config(); cfg != nil { + if t := strings.TrimSpace(cfg.String(RouteReportTokenConfigKey)); t != "" { + return t + } + } + return s.NotFoundToken() +} + // PageOrigin is the origin of a page created through the management API: // mcp through MCP, else manual. func PageOrigin(ctx context.Context) string { @@ -54,6 +71,10 @@ func PageOrigin(ctx context.Context) string { return models.PageOriginManual } +// ErrTooManyRoutes refuses a route report of more than MaxReportedRoutes +// paths. +var ErrTooManyRoutes = errors.New("golem15.seo: a route report holds at most 2000 paths") + // The phrase keys of the registry refusals (UI-SPEC C21). const ( errPageURLRequired = "golem15.seo::lang.pages.url_required" @@ -698,6 +719,117 @@ func (s *Service) DeletePage(ctx context.Context, tx *gorm.DB, p models.Page) er return nil } +// RouteReport is the outcome of a route report: the paths inserted as new +// registry pages, the known pages whose reported_at moved, and the paths +// skipped (invalid, or owned by another content type). +type RouteReport struct { + Inserted int `json:"inserted"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + ReportedAt time.Time `json:"reported_at"` +} + +// ReportRoutes takes the frontend server's static routes (D-33, P33-20) +// inside tx: a path no registry page has is inserted with origin reported, +// kind page and a name from its last segment; a known page gets +// reported_at; a path another content type owns, or that is no valid +// address, is skipped. Nothing is ever deleted, and no value an editor set +// is overwritten. The settings' routes_reported_at becomes the report's +// time, so a reported page missing from it shows as missing. More than +// MaxReportedRoutes paths is ErrTooManyRoutes. +func (s *Service) ReportRoutes(ctx context.Context, tx *gorm.DB, paths []string) (RouteReport, error) { + now := time.Now().UTC() + out := RouteReport{ReportedAt: now} + if len(paths) > MaxReportedRoutes { + return out, ErrTooManyRoutes + } + settings, err := SettingsRow(tx) + if err != nil { + return out, err + } + opts := NormalizeOptionsOf(settings) + ctx = WithVia(ctx, models.ViaAuto) + seen := map[string]bool{} + for _, raw := range paths { + path := strings.TrimSpace(raw) + if len(path) > MaxRouteBytes || !ValidPagePath(path) { + out.Skipped++ + continue + } + path = normalPagePath(path, opts) + if seen[path] { + continue + } + seen[path] = true + page, _, err := s.PageAt(ctx, tx, path) + if err != nil { + return out, err + } + if page != nil { + if err := tx.Model(&models.Page{}).Where("id = ?", page.ID).UpdateColumn("reported_at", now).Error; err != nil { + return out, err + } + out.Updated++ + continue + } + owned, err := s.ownerOutsideRegistry(ctx, path) + if err != nil { + return out, err + } + if owned != nil { + out.Skipped++ + continue + } + p := models.Page{ + URL: path, + URLs: lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true}, + Title: s.routeTitle(path), + Titles: lagoon.Jsonable[map[string]string]{Data: map[string]string{}, Valid: true}, + Kind: models.PageKindPage, + Origin: models.PageOriginReported, + ReportedAt: &now, + CreatedAt: &now, + UpdatedAt: &now, + } + res := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&p) + if res.Error != nil { + return out, res.Error + } + if res.RowsAffected == 0 || p.ID == 0 { + // A concurrent report inserted it first. + continue + } + out.Inserted++ + if err := s.RecordPage(ctx, tx, nil, p, false); err != nil { + return out, err + } + cabana.EmitRecordSaved(ctx, s.app, tx, pact.RecordSaved{Controller: PagesController, Model: PageModel, ID: p.IDText(), Created: true}) + } + if err := tx.Model(&models.Settings{}).Where("id = ?", 1).UpdateColumn("routes_reported_at", now).Error; err != nil { + return out, err + } + return out, nil +} + +// routeTitle is a reported page's name: its last path segment with +// hyphens and underscores as spaces and the first letter capitalised; the +// home page is named pages.home_title. +func (s *Service) routeTitle(path string) string { + seg := strings.Trim(path, "/") + if i := strings.LastIndex(seg, "/"); i >= 0 { + seg = seg[i+1:] + } + if decoded, err := url.PathUnescape(seg); err == nil { + seg = decoded + } + seg = strings.TrimSpace(strings.NewReplacer("-", " ", "_", " ").Replace(seg)) + if seg == "" { + return s.Phrase("golem15.seo::lang.pages.home_title", nil) + } + r, size := utf8.DecodeRuneInString(seg) + return string(unicode.ToUpper(r)) + seg[size:] +} + // PageMissing reports whether p is a reported page that the newest route // report (at reported) did not name. func PageMissing(p models.Page, reported *time.Time) bool { diff --git a/config/config.yaml b/config/config.yaml index be46038..54ff838 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -23,3 +23,9 @@ notfound_max_rows: 10000 # robots_sitemaps lists the sitemap addresses robots.txt names, one # "Sitemap:" line each, made absolute on the site URL. robots_sitemaps: ["/sitemap.xml"] +# +# route_report_token is the shared secret the frontend server sends as +# "Authorization: Bearer " with its route report to +# POST /_seo/api/v1/registry/routes (the page registry). Empty uses +# notfound_token; both empty turn the report off (it answers 404). +route_report_token: "" diff --git a/controllers/api/registry.go b/controllers/api/registry.go index 38fcb02..67c254b 100644 --- a/controllers/api/registry.go +++ b/controllers/api/registry.go @@ -3,16 +3,82 @@ package api import ( "bytes" "context" + "crypto/subtle" "encoding/json" + "errors" "net/http" "strconv" "strings" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/wire" "gorm.io/gorm" ) +// maxRouteReportBody caps the route report body: 2000 paths of 2048 bytes +// with their JSON quoting. +const maxRouteReportBody = classes.MaxReportedRoutes*(classes.MaxRouteBytes+8) + 1024 + +// routeReport is the route report body. +type routeReport struct { + Routes []string `json:"routes"` +} + +// RouteReport serves POST /_seo/api/v1/registry/routes (D-33, P33-20): the +// frontend server reports its static routes with "Authorization: Bearer +// ", golem15.seo.route_report_token (else notfound_token), compared +// in constant time. Unknown paths become registry pages (origin reported), +// known ones get reported_at; nothing is deleted or overwritten. It answers +// 200 with the counts, 401 for a missing or wrong token, 404 when no token +// is configured (the report is off), and 422 for a malformed body or more +// than 2000 paths. +func RouteReport(service func() *classes.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + svc := service() + if svc == nil { + wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"}) + return + } + want := svc.RouteReportToken() + if want == "" { + wire.WriteJSON(w, http.StatusNotFound, map[string]string{"error": errNotFound}) + return + } + got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(want)) != 1 { + wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": "Unauthorized"}) + return + } + var in routeReport + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRouteReportBody)) + if err := dec.Decode(&in); err != nil { + wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": errInvalidBody}) + return + } + if len(in.Routes) > classes.MaxReportedRoutes { + wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]any{"error": "too_many_routes", "max": classes.MaxReportedRoutes}) + return + } + var out classes.RouteReport + err := write(r.Context(), svc, func(ctx context.Context, tx *gorm.DB) error { + var err error + out, err = svc.ReportRoutes(ctx, tx, in.Routes) + return err + }) + if err != nil { + if errors.Is(err, classes.ErrTooManyRoutes) { + wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]any{"error": "too_many_routes", "max": classes.MaxReportedRoutes}) + return + } + log.Error("golem15.seo: route report", "error", err) + wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": errInternal}) + return + } + wire.WriteJSON(w, http.StatusOK, out) + } +} + // ListPagesInput is the query of GET /_seo/api/v1/pages. type ListPagesInput struct { Page int `json:"page,omitempty" query:"page" jsonschema:"the page number, from 1"` diff --git a/lang/en/lang.yaml b/lang/en/lang.yaml index d795889..19f2128 100644 --- a/lang/en/lang.yaml +++ b/lang/en/lang.yaml @@ -392,3 +392,4 @@ pages: delete_selected_confirm: one: "Delete :count page from the registry? Its SEO tab is deleted with it." other: "Delete :count pages from the registry? Their SEO tabs are deleted with them." + home_title: Home diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index 65fd9ba..f5756bb 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -406,3 +406,4 @@ pages: few: "Usunąć :count strony z rejestru? Ich zakładki SEO zostaną usunięte." many: "Usunąć :count stron z rejestru? Ich zakładki SEO zostaną usunięte." other: "Usunąć :count strony z rejestru? Ich zakładki SEO zostaną usunięte." + home_title: Strona główna diff --git a/registry_test.go b/registry_test.go index 199515d..0e6e43b 100644 --- a/registry_test.go +++ b/registry_test.go @@ -182,3 +182,151 @@ func jsonID(id uint) string { return string(raw) } +// pageRow reads a registry page's stored values by its address. +type pageRow struct { + ID uint `gorm:"column:id"` + URL string `gorm:"column:url"` + Title string `gorm:"column:title"` + Kind string `gorm:"column:kind"` + Origin string `gorm:"column:origin"` + Note string `gorm:"column:note"` + Stamp string `gorm:"column:stamp"` +} + +func (e *testEnv) pageRow(t *testing.T, url string) pageRow { + t.Helper() + var rows []pageRow + if err := e.db.Raw(`SELECT id, url, title, kind, origin, note, COALESCE(reported_at::text, '') AS stamp FROM golem15_seo_pages WHERE url = ?`, url).Scan(&rows).Error; err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("pages at %s = %d, want 1", url, len(rows)) + } + return rows[0] +} + +// reportRoutes posts a route report with the bearer token. +func (e *testEnv) reportRoutes(t *testing.T, token string, routes []string) map[string]any { + t.Helper() + rec := e.expect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/registry/routes", token, map[string]any{"routes": routes}) + if rec.Header().Get("X-Robots-Tag") != "noindex" { + t.Fatalf("route report headers = %v, want X-Robots-Tag noindex", rec.Header()) + } + var out map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + return out +} + +// TestRouteReport covers the route report (D-33, P33-20, T-33-18): the +// token, the size cap, insert-only behaviour, reported_at, the missing +// badge and P9 warning, owned paths skipped, and that a later report never +// deletes a page or overwrites an editor's values. +func TestRouteReport(t *testing.T) { + off := bootEnv(t) + off.expect(t, http.StatusNotFound, http.MethodPost, "/_seo/api/v1/registry/routes", "anything", map[string]any{"routes": []string{"/"}}) + + // notfound_token is the fallback token. + fallback := bootEnv(t, "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN=nf-token") + fallback.reportRoutes(t, "nf-token", []string{"/x"}) + + env := bootEnv(t, "SUMMER_GOLEM15__SEO__ROUTE_REPORT_TOKEN=report-token", "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN=other") + admin := env.admin(t, true) + env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "wrong", map[string]any{"routes": []string{"/"}}) + env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "other", map[string]any{"routes": []string{"/"}}) + env.expect(t, http.StatusUnauthorized, http.MethodPost, "/_seo/api/v1/registry/routes", "", map[string]any{"routes": []string{"/"}}) + many := make([]string, 2001) + for i := range many { + many[i] = "/p" + } + env.expect(t, http.StatusUnprocessableEntity, http.MethodPost, "/_seo/api/v1/registry/routes", "report-token", map[string]any{"routes": many}) + + env.createRegistryPage(t, admin, map[string]any{"url": map[string]any{"en": "/kontakt"}, "title": map[string]any{"en": "Kontakt"}}) + env.createPage(t, admin, map[string]any{"title": "Blog post", "slug": "post", "published": true}) + + out := env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas", "/pages/post", "nope", "/o-nas"}) + if out["inserted"] != float64(2) || out["updated"] != float64(1) || out["skipped"] != float64(2) { + t.Fatalf("first report = %v, want 2 inserted (/, /o-nas), 1 updated, 2 skipped", out) + } + about := env.pageRow(t, "/o-nas") + if about.Origin != "reported" || about.Kind != "page" || about.Title != "O nas" || about.Stamp == "" { + t.Fatalf("/o-nas = %+v, want origin reported, kind page, title O nas, reported", about) + } + if home := env.pageRow(t, "/"); home.Title != "Home" { + t.Fatalf("home = %+v, want the title Home", home) + } + contact := env.pageRow(t, "/kontakt") + if contact.Origin != "manual" || contact.Title != "Kontakt" || contact.Stamp == "" { + t.Fatalf("/kontakt = %+v, want the editor's page with reported_at", contact) + } + var owned int64 + env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_pages WHERE url = '/pages/post'`).Scan(&owned) + if owned != 0 { + t.Fatal("a path owned by a content type was inserted") + } + if got := env.resolve(t, "/o-nas"); got["action"] != "ok" { + t.Fatalf("resolve /o-nas = %v, want ok", got) + } + + // The editor changes the reported page; the same report again changes + // only reported_at. + id := jsonID(about.ID) + env.expect(t, http.StatusOK, http.MethodPut, pagesAPI+"/"+id, admin, map[string]any{"title": map[string]any{"en": "About us"}, "kind": "offer", "note": "kept"}) + var revisions int64 + env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions`).Scan(&revisions) + before := env.pageRow(t, "/o-nas") + out = env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas"}) + if out["inserted"] != float64(0) || out["updated"] != float64(3) { + t.Fatalf("second report = %v, want 0 inserted, 3 updated", out) + } + after := env.pageRow(t, "/o-nas") + if after.Title != "About us" || after.Kind != "offer" || after.Note != "kept" || after.Origin != "reported" || after.Stamp == before.Stamp { + t.Fatalf("after the second report /o-nas = %+v (before %+v), want only reported_at changed", after, before) + } + var revisionsAfter int64 + env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions`).Scan(&revisionsAfter) + if revisionsAfter != revisions { + t.Fatalf("revisions %d -> %d: a report that inserts nothing writes no history", revisions, revisionsAfter) + } + + // A report without /o-nas marks it missing and deletes nothing. + env.reportRoutes(t, "report-token", []string{"/", "/kontakt"}) + missing := env.pageRow(t, "/o-nas") + if missing.Title != "About us" { + t.Fatalf("missing page = %+v, want it kept with its values", missing) + } + var list struct { + Meta struct { + RowStates map[string][]string `json:"row_states"` + } `json:"meta"` + } + if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, pagesAPI, admin, nil).Body.Bytes(), &list); err != nil { + t.Fatal(err) + } + if states := list.Meta.RowStates[id]; len(states) != 1 || states[0] != "negative" { + t.Fatalf("row states = %v, want /o-nas (%s) negative", list.Meta.RowStates, id) + } + if states := list.Meta.RowStates[jsonID(contact.ID)]; len(states) != 0 { + t.Fatalf("the manual page has row states %v", states) + } + status := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+id, admin, nil).Body.String() + if !strings.Contains(status, "summer-callout--warning") || !strings.Contains(status, "did not include this address") { + t.Fatalf("P9 of the missing page = %s", status) + } + if home := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+jsonID(env.pageRow(t, "/").ID), admin, nil).Body.String(); !strings.Contains(home, "Reported by the site on") || strings.Contains(home, "--warning") { + t.Fatalf("P9 of a reported page = %s", home) + } + if manual := env.expect(t, http.StatusOK, http.MethodGet, pagesAPI+"/partials/page_status?id="+jsonID(contact.ID), admin, nil).Body.String(); strings.Contains(manual, "summer-callout") { + t.Fatalf("P9 of a manual page = %s, want nothing", manual) + } + missingList := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/pages?missing=only", admin, nil).Body.String() + if !jsonContains(missingList, `"total":1`) || !jsonContains(missingList, `"missing":true`) { + t.Fatalf("missing=only list = %s", missingList) + } + // Reported again, it is back on the site. + env.reportRoutes(t, "report-token", []string{"/", "/kontakt", "/o-nas"}) + if again := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/pages?missing=only", admin, nil).Body.String(); !jsonContains(again, `"total":0`) { + t.Fatalf("missing=only after the page is reported again = %s", again) + } +} diff --git a/routes.go b/routes.go index 927b5bc..f2498b0 100644 --- a/routes.go +++ b/routes.go @@ -34,6 +34,9 @@ func (p *Plugin) Routes(r pact.Router) error { r.Group("/_seo/api/v1", nil, func(g pact.Router) { g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve) g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound) + // The frontend server's route report (D-33), token-authenticated + // like the 404 report. + g.Post("/registry/routes", noindex(apicontrollers.RouteReport(p.Service)), "throttle:"+BucketNotFound) // Verification files are served here only; the frontend maps them // at the site root (D-103). A root route would capture every // one-segment page of a site served at GET /.