feat: 404 log and report endpoint

- POST /_seo/api/v1/notfound with the golem15.seo.notfound_token bearer
  (constant-time, off when unset), URL and referer caps, a row cap with
  an hourly warning, throttle bucket seo-notfound
- golem15.seo.notfound list by hits with Resolved rows, a preview with the
  resolved hint, a prompted 301 and a 410 record action, a bulk 410 that
  refuses a selection with a covered address
- every /_seo/api/v1 answer carries X-Robots-Tag: noindex

controllers/admin_registry.go

@@ -30,6 +30,7 @@ func AdminControllers(app func() *backpack.App) []pact.AdminController {
 	return []pact.AdminController{
 		settingsAdminController{app: app},
 		redirectsAdminController{app: app},
+		notFoundAdminController{app: app},
 		rulesAdminController{app: app},
 		historyAdminController{app: app},
 	}

controllers/api/notfound.go

@@ -0,0 +1,64 @@
+package api
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/wire"
+)
+
+// maxReportBody caps the 404 report body: the URL and the referer fit
+// with room to spare.
+const maxReportBody = 8 << 10
+
+// notFoundReport is the 404 report body.
+type notFoundReport struct {
+	URL     string `json:"url"`
+	Referer string `json:"referer"`
+}
+
+// NotFound serves POST /_seo/api/v1/notfound (D-20, P-22): the frontend
+// server reports a 404 with "Authorization: Bearer <token>", the configured
+// golem15.seo.notfound_token, compared in constant time. It answers 204,
+// 401 for a missing or wrong token, 404 when no token is configured (the
+// endpoint is off), and 422 for a body without a path or http(s) URL of at
+// most 2048 bytes.
+func NotFound(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"})
+			return
+		}
+		want := svc.NotFoundToken()
+		if want == "" {
+			wire.WriteJSON(w, http.StatusNotFound, map[string]string{"error": "Not Found"})
+			return
+		}
+		got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(want)) != 1 {
+			wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": "Unauthorized"})
+			return
+		}
+		var in notFoundReport
+		dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReportBody))
+		if err := dec.Decode(&in); err != nil {
+			wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "The request body is invalid."})
+			return
+		}
+		if err := svc.ReportNotFound(r.Context(), in.URL, in.Referer); err != nil {
+			if errors.Is(err, classes.ErrNotFoundURL) {
+				wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": err.Error()})
+				return
+			}
+			log.Error("golem15.seo: 404 report", "error", err)
+			wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": "Internal Server Error"})
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}

controllers/notfound.go

@@ -0,0 +1,273 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/cabana"
+	"git.golem15.com/golem15/summercms/modules/pact"
+	"gorm.io/gorm"
+)
+
+// NotFoundControllerID is the 404 log (UI-SPEC C9).
+const NotFoundControllerID = "golem15.seo.notfound"
+
+// errNotFoundReadOnly refuses creating or editing 404 log entries: they
+// come from the report endpoint only.
+var errNotFoundReadOnly = errors.New("golem15.seo: the 404 log is read-only")
+
+// The 404 log's record and bulk action names.
+const (
+	NotFoundActionRedirect = "redirect"
+	NotFoundActionGone     = "gone"
+)
+
+// notFoundAdminController lists the 404 log by hits and previews an entry
+// with two record actions, a prompted "Create redirect…" (301) and "Mark
+// as gone" (410), and the bulk "Mark as gone" (D-20). An entry a redirect
+// or a 410 now answers is Resolved: badged, and its actions no longer
+// apply.
+type notFoundAdminController struct {
+	app func() *backpack.App
+}
+
+func (notFoundAdminController) ID() string        { return NotFoundControllerID }
+func (notFoundAdminController) ModelName() string { return `Golem15\SEO\Models\NotFound` }
+func (notFoundAdminController) ConfigDir() string { return "controllers/notfound" }
+func (notFoundAdminController) NewRecord() any    { return &models.NotFound{} }
+func (notFoundAdminController) RequiredPermissions() []string {
+	return []string{PermissionManage}
+}
+
+// FormBeforeCreate refuses: entries come from the report endpoint.
+func (notFoundAdminController) FormBeforeCreate(context.Context, any) error {
+	return errNotFoundReadOnly
+}
+
+// FormBeforeUpdate refuses: entries are never edited.
+func (notFoundAdminController) FormBeforeUpdate(context.Context, any) error {
+	return errNotFoundReadOnly
+}
+
+func (c notFoundAdminController) service() *classes.Service {
+	svc, err := classes.NewService(appOf(c.app))
+	if err != nil {
+		return nil
+	}
+	return svc
+}
+
+// cover returns the rule that answers the entry now, nil while it is
+// open.
+func (c notFoundAdminController) cover(ctx context.Context, entry *models.NotFound) (*models.Redirect, error) {
+	db, err := database(ctx, c.app)
+	if err != nil {
+		return nil, err
+	}
+	set, err := classes.LoadRedirectSet(db)
+	if err != nil {
+		return nil, err
+	}
+	return classes.NotFoundCover(set, entry.URL)
+}
+
+// open reports whether a record is an entry no rule answers yet.
+func (c notFoundAdminController) open(ctx context.Context, record any) (bool, error) {
+	entry, ok := record.(*models.NotFound)
+	if !ok || entry == nil {
+		return false, nil
+	}
+	rule, err := c.cover(ctx, entry)
+	return rule == nil, err
+}
+
+// AdminRecordActions are the preview's two actions (UI-SPEC C9); both
+// apply only while the entry is open.
+func (c notFoundAdminController) AdminRecordActions() []pact.AdminRecordAction {
+	return []pact.AdminRecordAction{
+		{
+			Name:        NotFoundActionRedirect,
+			Label:       "golem15.seo::lang.notfound.redirect",
+			Confirm:     "golem15.seo::lang.notfound.redirect_confirm",
+			Permissions: []string{PermissionManage},
+			Prompt: &pact.AdminActionPrompt{
+				Label:       "golem15.seo::lang.notfound.redirect_prompt",
+				Placeholder: "golem15.seo::lang.notfound.redirect_placeholder",
+				Comment:     "golem15.seo::lang.notfound.redirect_comment",
+				MaxLength:   models.MaxRedirectURL,
+			},
+			Applies: c.open,
+			Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) {
+				return c.resolve(ctx, in.Record, in.Value)
+			},
+		},
+		{
+			Name:        NotFoundActionGone,
+			Label:       "golem15.seo::lang.notfound.gone",
+			Confirm:     "golem15.seo::lang.notfound.gone_confirm",
+			Permissions: []string{PermissionManage},
+			Danger:      true,
+			Applies:     c.open,
+			Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) {
+				return c.resolve(ctx, in.Record, "")
+			},
+		},
+	}
+}
+
+// resolve creates the entry's 301 to target, or its 410 when target is
+// empty, inside the action's transaction.
+func (c notFoundAdminController) resolve(ctx context.Context, record any, target string) (pact.AdminRecordActionResult, error) {
+	entry, ok := record.(*models.NotFound)
+	svc := c.service()
+	if !ok || entry == nil || svc == nil {
+		return pact.AdminRecordActionResult{}, classes.ErrNotBooted
+	}
+	db, err := database(ctx, c.app)
+	if err != nil {
+		return pact.AdminRecordActionResult{}, err
+	}
+	rule, err := svc.ResolveNotFound(ctx, db, *entry, target)
+	if err != nil {
+		return pact.AdminRecordActionResult{}, c.refusal(ctx, svc, err)
+	}
+	tr := translator(c.app)
+	if rule.Status == models.Status410 {
+		return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.gone_done", map[string]string{"url": entry.URL})}, nil
+	}
+	return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.redirected", map[string]string{"url": entry.URL, "target": rule.Target})}, nil
+}
+
+// refusal turns a 404 action refusal into the admin's answer: a rule that
+// already covers the address and a loop are a 403 whose message the admin
+// shows as a toast; a bad target is a 422 on the prompt's payload.
+func (c notFoundAdminController) refusal(ctx context.Context, svc *classes.Service, err error) error {
+	tr := translator(c.app)
+	var covered *classes.NotFoundHasRuleError
+	var loop *classes.LoopError
+	var problems *classes.ProblemsError
+	switch {
+	case errors.As(err, &covered):
+		return &cabana.ForbiddenError{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.has_rule", map[string]string{"url": covered.URL})}
+	case errors.As(err, &loop):
+		return &cabana.ForbiddenError{Message: svc.LoopMessage(ctx, loop.Check)}
+	case errors.As(err, &problems):
+		var msgs []string
+		for _, list := range problems.Problems {
+			for _, p := range list {
+				msgs = append(msgs, Phrase(ctx, tr, p.Key, p.Params))
+			}
+		}
+		return &cabana.ValidationError{Details: map[string]any{"payload": msgs}}
+	}
+	return err
+}
+
+// AdminBulkActions is the list's bulk "Mark as gone" (UI-SPEC C9): one
+// entry a rule already answers refuses the whole selection.
+func (c notFoundAdminController) AdminBulkActions() []pact.AdminBulkAction {
+	return []pact.AdminBulkAction{{
+		Name:        NotFoundActionGone,
+		Label:       "golem15.seo::lang.notfound.gone",
+		Confirm:     "golem15.seo::lang.notfound.gone_selected_confirm",
+		Permissions: []string{PermissionManage},
+		Run: func(ctx context.Context, in pact.AdminBulkActionInput) (pact.AdminBulkActionResult, error) {
+			svc := c.service()
+			if svc == nil {
+				return pact.AdminBulkActionResult{}, classes.ErrNotBooted
+			}
+			db, err := database(ctx, c.app)
+			if err != nil {
+				return pact.AdminBulkActionResult{}, err
+			}
+			n := 0
+			for _, record := range in.Records {
+				entry, ok := record.(*models.NotFound)
+				if !ok || entry == nil {
+					continue
+				}
+				if _, err := svc.ResolveNotFound(ctx, db, *entry, ""); err != nil {
+					return pact.AdminBulkActionResult{}, c.refusal(ctx, svc, err)
+				}
+				n++
+			}
+			return pact.AdminBulkActionResult{Message: "golem15.seo::lang.notfound.gone_selected_done", Affected: n}, nil
+		},
+	}}
+}
+
+// ListRowStates marks resolved entries disabled, labelled
+// notfound.state_resolved (UI-SPEC C9).
+func (notFoundAdminController) ListRowStates(ctx context.Context, db *gorm.DB, records []any) ([][]pact.RowState, error) {
+	set, err := classes.LoadRedirectSet(db.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	out := make([][]pact.RowState, len(records))
+	for i, record := range records {
+		entry, ok := record.(*models.NotFound)
+		if !ok || entry == nil {
+			continue
+		}
+		rule, err := classes.NotFoundCover(set, entry.URL)
+		if err != nil {
+			return nil, err
+		}
+		if rule != nil {
+			out[i] = []pact.RowState{pact.RowStateDisabled}
+		}
+	}
+	return out, nil
+}
+
+// HintView is the preview's header partial: the resolved hint, empty
+// while the entry is open.
+type HintView struct {
+	Text string
+}
+
+// PartialData serves the preview's hint (notfound.resolved_hint).
+func (c notFoundAdminController) PartialData(ctx context.Context, name string, record any) (any, error) {
+	entry, ok := record.(*models.NotFound)
+	if name != "hint" || !ok || entry == nil {
+		return HintView{}, nil
+	}
+	rule, err := c.cover(ctx, entry)
+	if err != nil || rule == nil {
+		return HintView{}, err
+	}
+	label := rule.Source
+	if rule.Target != "" {
+		label += " → " + rule.Target
+	} else {
+		label += " → 410"
+	}
+	return HintView{Text: Phrase(ctx, translator(c.app), "golem15.seo::lang.notfound.resolved_hint", map[string]string{"rule": label})}, nil
+}
+
+// FilterOptions serves the status filter's open and resolved choices.
+func (notFoundAdminController) FilterOptions(scope string) []pact.Option {
+	if scope != models.FilterNotFoundStatus {
+		return nil
+	}
+	return []pact.Option{
+		{Value: models.NotFoundOpen, Label: "golem15.seo::lang.notfound.filter_status_open"},
+		{Value: models.NotFoundResolved, Label: "golem15.seo::lang.notfound.filter_status_resolved"},
+	}
+}
+
+var (
+	_ pact.AdminController       = notFoundAdminController{}
+	_ pact.AdminPermissioned     = notFoundAdminController{}
+	_ pact.AdminRecordSource     = notFoundAdminController{}
+	_ pact.FormBeforeCreate      = notFoundAdminController{}
+	_ pact.FormBeforeUpdate      = notFoundAdminController{}
+	_ pact.HasAdminRecordActions = notFoundAdminController{}
+	_ pact.HasAdminBulkActions   = notFoundAdminController{}
+	_ pact.ListRowStates         = notFoundAdminController{}
+	_ pact.AdminPartialData      = notFoundAdminController{}
+	_ pact.FilterOptions         = notFoundAdminController{}
+)

controllers/notfound/_hint.htm

@@ -0,0 +1,5 @@
+{{- with .Data.Text -}}
+<div class="summer-callout" role="status">
+<p class="summer-callout__text">{{ . }}</p>
+</div>
+{{- end -}}

controllers/notfound/config_filter.yaml

@@ -0,0 +1,11 @@
+# 404 log filters (UI-SPEC C9): open or resolved, and the last hit.
+scopes:
+    status:
+        label: golem15.seo::lang.notfound.filter_status
+        modelClass: Golem15\SEO\Models\NotFound
+        nameFrom: url
+        scope: FilterNotFoundStatus
+    last_hit:
+        label: golem15.seo::lang.notfound.last_hit
+        type: daterange
+        column: last_hit

controllers/notfound/config_form.yaml

@@ -0,0 +1,14 @@
+# 404 log preview (UI-SPEC C9): read-only, the resolved hint above the card
+# and the two actions that apply while the entry is open.
+name: golem15.seo::lang.notfound.name
+form: ~/plugins/golem15/seo/models/notfound/fields.yaml
+modelClass: Golem15\SEO\Models\NotFound
+defaultRedirect: golem15/seo/notfound
+
+preview:
+    headerPartial: hint
+
+recordActions: [redirect, gone]
+
+messages:
+    preview: golem15.seo::lang.notfound.preview

controllers/notfound/config_list.yaml

@@ -0,0 +1,25 @@
+# 404 log (UI-SPEC C9): most hits first, resolved entries badged, no create.
+list: ~/plugins/golem15/seo/models/notfound/columns.yaml
+modelClass: Golem15\SEO\Models\NotFound
+title: golem15.seo::lang.notfound.list_title
+recordUrl: golem15/seo/notfound/preview/:id
+noRecordsMessage: golem15.seo::lang.notfound.list_empty
+recordsPerPage: 50
+showCheckboxes: true
+showSetup: true
+filter: config_filter.yaml
+bulkActions: [gone]
+
+defaultSort:
+    column: hits
+    direction: desc
+
+toolbar:
+    buttons: [delete]
+    search:
+        prompt: backend::lang.list.search_prompt
+
+messages:
+    empty: golem15.seo::lang.notfound.list_empty
+    deleteConfirm: golem15.seo::lang.notfound.delete_selected_confirm
+    rowStateDisabled: golem15.seo::lang.notfound.state_resolved