feat: 404 log and report endpoint

- POST /_seo/api/v1/notfound with the golem15.seo.notfound_token bearer (constant-time, off when unset), URL and referer caps, a row cap with an hourly warning, throttle bucket seo-notfound - golem15.seo.notfound list by hits with Resolved rows, a preview with the resolved hint, a prompted 301 and a 410 record action, a bulk 410 that refuses a selection with a covered address - every /_seo/api/v1 answer carries X-Robots-Tag: noindex
2026-10-27 11:35:25 +01:00
parent 163b0aaf98
commit 6eb6cb7a4d
18 changed files with 997 additions and 15 deletions
--- a/admin_navigation.go
+++ b/admin_navigation.go
@@ -24,6 +24,13 @@ func (*Plugin) Navigation() []pact.NavigationItem {
 				Permissions: []string{controllers.PermissionManage},
 				Controller:  controllers.RedirectsControllerID,
 			},
+			{
+				Code:        "notfound",
+				Label:       "golem15.seo::lang.nav.notfound",
+				Icon:        "triangle-alert",
+				Permissions: []string{controllers.PermissionManage},
+				Controller:  controllers.NotFoundControllerID,
+			},
 			{
 				Code:        "rules",
 				Label:       "golem15.seo::lang.nav.rules",
--- a/classes/notfound.go
+++ b/classes/notfound.go
@@ -0,0 +1,181 @@
+package classes
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// Config keys of the 404 log (P-22).
+const (
+	// NotFoundTokenConfigKey is the shared secret the frontend server
+	// sends as "Authorization: Bearer <token>" with each 404 report. Empty
+	// turns the report endpoint off (it answers 404).
+	NotFoundTokenConfigKey = "golem15.seo.notfound_token"
+	// NotFoundMaxRowsConfigKey caps the 404 log; new addresses beyond it
+	// are dropped (default DefaultNotFoundMaxRows).
+	NotFoundMaxRowsConfigKey = "golem15.seo.notfound_max_rows"
+)
+
+// The 404 report limits (P-22).
+const (
+	DefaultNotFoundMaxRows = 10000
+	MaxNotFoundURLBytes    = 2048
+	MaxRefererBytes        = 1024
+)
+
+// ErrNotFoundURL refuses a reported address that is not a path or an
+// http(s) URL, or is longer than MaxNotFoundURLBytes.
+var ErrNotFoundURL = errors.New("golem15.seo: url must be a path or an absolute http(s) URL of at most 2048 bytes")
+
+// NotFoundToken is the configured report token, "" when unset.
+func (s *Service) NotFoundToken() string {
+	cfg := s.Config()
+	if cfg == nil {
+		return ""
+	}
+	return strings.TrimSpace(cfg.String(NotFoundTokenConfigKey))
+}
+
+// NotFoundMaxRows is the configured row cap of the 404 log.
+func (s *Service) NotFoundMaxRows() int64 {
+	if cfg := s.Config(); cfg != nil {
+		if n := cfg.Int(NotFoundMaxRowsConfigKey); n > 0 {
+			return int64(n)
+		}
+	}
+	return DefaultNotFoundMaxRows
+}
+
+// NotFoundPath turns a reported address into the logged one: the path
+// with its query; an absolute URL loses its scheme and host.
+func NotFoundPath(raw string) (string, error) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || len(raw) > MaxNotFoundURLBytes {
+		return "", ErrNotFoundURL
+	}
+	u, err := url.Parse(raw)
+	if err != nil {
+		return "", ErrNotFoundURL
+	}
+	if u.IsAbs() {
+		if u.Scheme != "http" && u.Scheme != "https" {
+			return "", ErrNotFoundURL
+		}
+	} else if !strings.HasPrefix(raw, "/") || strings.HasPrefix(raw, "//") {
+		return "", ErrNotFoundURL
+	}
+	p := u.Path
+	if p == "" {
+		p = "/"
+	}
+	return joinQuery(p, u.RawQuery), nil
+}
+
+// capWarned keeps when the row-cap warning was last logged.
+var capWarned struct {
+	sync.Mutex
+	at time.Time
+}
+
+// ReportNotFound records one 404 of rawURL (D-20, P-22): a known address
+// gets one more hit, its last hit time and, when given, the referer; a new
+// one is added unless the log holds NotFoundMaxRows rows, which is logged
+// at most once an hour.
+func (s *Service) ReportNotFound(ctx context.Context, rawURL, referer string) error {
+	path, err := NotFoundPath(rawURL)
+	if err != nil {
+		return err
+	}
+	referer = strings.TrimSpace(referer)
+	if len(referer) > MaxRefererBytes {
+		referer = referer[:MaxRefererBytes]
+		for len(referer) > 0 && !utf8ValidTail(referer) {
+			referer = referer[:len(referer)-1]
+		}
+	}
+	db, err := s.DB(ctx)
+	if err != nil {
+		return err
+	}
+	now := time.Now().UTC()
+	updates := map[string]any{"hits": gorm.Expr("hits + 1"), "last_hit": now}
+	if referer != "" {
+		updates["referer"] = referer
+	}
+	res := db.Model(&models.NotFound{}).Where("url = ?", path).Updates(updates)
+	if res.Error != nil || res.RowsAffected > 0 {
+		return res.Error
+	}
+	var count int64
+	if err := db.Model(&models.NotFound{}).Count(&count).Error; err != nil {
+		return err
+	}
+	if count >= s.NotFoundMaxRows() {
+		capWarned.Lock()
+		if time.Since(capWarned.at) >= time.Hour {
+			capWarned.at = time.Now()
+			log.Warn("golem15.seo: the 404 log is full, new addresses are dropped", "max_rows", s.NotFoundMaxRows(), "url", path)
+		}
+		capWarned.Unlock()
+		return nil
+	}
+	row := models.NotFound{URL: path, Hits: 1, FirstSeen: &now, LastHit: &now, Referer: referer}
+	return db.Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "url"}},
+		DoUpdates: clause.Assignments(map[string]any{"hits": gorm.Expr("golem15_seo_notfound.hits + 1"), "last_hit": now}),
+	}).Create(&row).Error
+}
+
+// utf8ValidTail reports whether s does not end inside a UTF-8 sequence.
+func utf8ValidTail(s string) bool {
+	return strings.ToValidUTF8(s, "<22>") == s
+}
+
+// NotFoundCover returns the rule that answers url now (a redirect or a
+// 410), nil when the address still answers 404: the Resolved state.
+func NotFoundCover(set *RedirectSet, url string) (*models.Redirect, error) {
+	return CoveringRedirect(set, url)
+}
+
+// NotFoundHasRuleError refuses a 404 action on an address a rule already
+// answers.
+type NotFoundHasRuleError struct{ URL string }
+
+func (e *NotFoundHasRuleError) Error() string {
+	return "golem15.seo: a redirect for " + e.URL + " already exists"
+}
+
+// ResolveNotFound creates the rule of a 404 log action inside tx (D-20): a
+// 301 to target when target is set, else a 410, with origin notfound. It
+// refuses an address a rule already answers (*NotFoundHasRuleError), and
+// the redirect's own refusals (*ProblemsError, *LoopError).
+func (s *Service) ResolveNotFound(ctx context.Context, tx *gorm.DB, entry models.NotFound, target string) (models.Redirect, error) {
+	set, err := LoadRedirectSet(tx)
+	if err != nil {
+		return models.Redirect{}, err
+	}
+	if rule, err := NotFoundCover(set, entry.URL); err != nil {
+		return models.Redirect{}, err
+	} else if rule != nil {
+		return models.Redirect{}, &NotFoundHasRuleError{URL: entry.URL}
+	}
+	r := models.NewRedirect()
+	r.Source, r.Origin = entry.URL, models.OriginNotFound
+	if strings.TrimSpace(target) == "" {
+		r.Status = models.Status410
+	} else {
+		r.Target = strings.TrimSpace(target)
+	}
+	if _, err := s.SaveRedirect(ctx, tx, &r); err != nil {
+		return r, err
+	}
+	return r, nil
+}
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -10,3 +10,12 @@ site_url: ""
 # prefix (for example pl on a site whose English pages start with /en).
 # Empty leaves the choice to each content type.
 unprefixed_locale: ""
+#
+# notfound_token is the shared secret the frontend server sends as
+# "Authorization: Bearer <token>" with each 404 report to
+# POST /_seo/api/v1/notfound. Empty turns the endpoint off (it answers 404).
+notfound_token: ""
+#
+# notfound_max_rows caps the 404 log; new addresses beyond it are dropped,
+# with one warning an hour.
+notfound_max_rows: 10000
--- a/controllers/admin_registry.go
+++ b/controllers/admin_registry.go
@@ -30,6 +30,7 @@ func AdminControllers(app func() *backpack.App) []pact.AdminController {
 	return []pact.AdminController{
 		settingsAdminController{app: app},
 		redirectsAdminController{app: app},
+		notFoundAdminController{app: app},
 		rulesAdminController{app: app},
 		historyAdminController{app: app},
 	}
--- a/controllers/api/notfound.go
+++ b/controllers/api/notfound.go
@@ -0,0 +1,64 @@
+package api
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/summercms/modules/wire"
+)
+
+// maxReportBody caps the 404 report body: the URL and the referer fit
+// with room to spare.
+const maxReportBody = 8 << 10
+
+// notFoundReport is the 404 report body.
+type notFoundReport struct {
+	URL     string `json:"url"`
+	Referer string `json:"referer"`
+}
+
+// NotFound serves POST /_seo/api/v1/notfound (D-20, P-22): the frontend
+// server reports a 404 with "Authorization: Bearer <token>", the configured
+// golem15.seo.notfound_token, compared in constant time. It answers 204,
+// 401 for a missing or wrong token, 404 when no token is configured (the
+// endpoint is off), and 422 for a body without a path or http(s) URL of at
+// most 2048 bytes.
+func NotFound(service func() *classes.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		svc := service()
+		if svc == nil {
+			wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"})
+			return
+		}
+		want := svc.NotFoundToken()
+		if want == "" {
+			wire.WriteJSON(w, http.StatusNotFound, map[string]string{"error": "Not Found"})
+			return
+		}
+		got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if !ok || subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(want)) != 1 {
+			wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": "Unauthorized"})
+			return
+		}
+		var in notFoundReport
+		dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReportBody))
+		if err := dec.Decode(&in); err != nil {
+			wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "The request body is invalid."})
+			return
+		}
+		if err := svc.ReportNotFound(r.Context(), in.URL, in.Referer); err != nil {
+			if errors.Is(err, classes.ErrNotFoundURL) {
+				wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": err.Error()})
+				return
+			}
+			log.Error("golem15.seo: 404 report", "error", err)
+			wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": "Internal Server Error"})
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
--- a/controllers/notfound.go
+++ b/controllers/notfound.go
@@ -0,0 +1,273 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/backpack"
+	"git.golem15.com/golem15/summercms/modules/cabana"
+	"git.golem15.com/golem15/summercms/modules/pact"
+	"gorm.io/gorm"
+)
+
+// NotFoundControllerID is the 404 log (UI-SPEC C9).
+const NotFoundControllerID = "golem15.seo.notfound"
+
+// errNotFoundReadOnly refuses creating or editing 404 log entries: they
+// come from the report endpoint only.
+var errNotFoundReadOnly = errors.New("golem15.seo: the 404 log is read-only")
+
+// The 404 log's record and bulk action names.
+const (
+	NotFoundActionRedirect = "redirect"
+	NotFoundActionGone     = "gone"
+)
+
+// notFoundAdminController lists the 404 log by hits and previews an entry
+// with two record actions, a prompted "Create redirect…" (301) and "Mark
+// as gone" (410), and the bulk "Mark as gone" (D-20). An entry a redirect
+// or a 410 now answers is Resolved: badged, and its actions no longer
+// apply.
+type notFoundAdminController struct {
+	app func() *backpack.App
+}
+
+func (notFoundAdminController) ID() string        { return NotFoundControllerID }
+func (notFoundAdminController) ModelName() string { return `Golem15\SEO\Models\NotFound` }
+func (notFoundAdminController) ConfigDir() string { return "controllers/notfound" }
+func (notFoundAdminController) NewRecord() any    { return &models.NotFound{} }
+func (notFoundAdminController) RequiredPermissions() []string {
+	return []string{PermissionManage}
+}
+
+// FormBeforeCreate refuses: entries come from the report endpoint.
+func (notFoundAdminController) FormBeforeCreate(context.Context, any) error {
+	return errNotFoundReadOnly
+}
+
+// FormBeforeUpdate refuses: entries are never edited.
+func (notFoundAdminController) FormBeforeUpdate(context.Context, any) error {
+	return errNotFoundReadOnly
+}
+
+func (c notFoundAdminController) service() *classes.Service {
+	svc, err := classes.NewService(appOf(c.app))
+	if err != nil {
+		return nil
+	}
+	return svc
+}
+
+// cover returns the rule that answers the entry now, nil while it is
+// open.
+func (c notFoundAdminController) cover(ctx context.Context, entry *models.NotFound) (*models.Redirect, error) {
+	db, err := database(ctx, c.app)
+	if err != nil {
+		return nil, err
+	}
+	set, err := classes.LoadRedirectSet(db)
+	if err != nil {
+		return nil, err
+	}
+	return classes.NotFoundCover(set, entry.URL)
+}
+
+// open reports whether a record is an entry no rule answers yet.
+func (c notFoundAdminController) open(ctx context.Context, record any) (bool, error) {
+	entry, ok := record.(*models.NotFound)
+	if !ok || entry == nil {
+		return false, nil
+	}
+	rule, err := c.cover(ctx, entry)
+	return rule == nil, err
+}
+
+// AdminRecordActions are the preview's two actions (UI-SPEC C9); both
+// apply only while the entry is open.
+func (c notFoundAdminController) AdminRecordActions() []pact.AdminRecordAction {
+	return []pact.AdminRecordAction{
+		{
+			Name:        NotFoundActionRedirect,
+			Label:       "golem15.seo::lang.notfound.redirect",
+			Confirm:     "golem15.seo::lang.notfound.redirect_confirm",
+			Permissions: []string{PermissionManage},
+			Prompt: &pact.AdminActionPrompt{
+				Label:       "golem15.seo::lang.notfound.redirect_prompt",
+				Placeholder: "golem15.seo::lang.notfound.redirect_placeholder",
+				Comment:     "golem15.seo::lang.notfound.redirect_comment",
+				MaxLength:   models.MaxRedirectURL,
+			},
+			Applies: c.open,
+			Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) {
+				return c.resolve(ctx, in.Record, in.Value)
+			},
+		},
+		{
+			Name:        NotFoundActionGone,
+			Label:       "golem15.seo::lang.notfound.gone",
+			Confirm:     "golem15.seo::lang.notfound.gone_confirm",
+			Permissions: []string{PermissionManage},
+			Danger:      true,
+			Applies:     c.open,
+			Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) {
+				return c.resolve(ctx, in.Record, "")
+			},
+		},
+	}
+}
+
+// resolve creates the entry's 301 to target, or its 410 when target is
+// empty, inside the action's transaction.
+func (c notFoundAdminController) resolve(ctx context.Context, record any, target string) (pact.AdminRecordActionResult, error) {
+	entry, ok := record.(*models.NotFound)
+	svc := c.service()
+	if !ok || entry == nil || svc == nil {
+		return pact.AdminRecordActionResult{}, classes.ErrNotBooted
+	}
+	db, err := database(ctx, c.app)
+	if err != nil {
+		return pact.AdminRecordActionResult{}, err
+	}
+	rule, err := svc.ResolveNotFound(ctx, db, *entry, target)
+	if err != nil {
+		return pact.AdminRecordActionResult{}, c.refusal(ctx, svc, err)
+	}
+	tr := translator(c.app)
+	if rule.Status == models.Status410 {
+		return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.gone_done", map[string]string{"url": entry.URL})}, nil
+	}
+	return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.redirected", map[string]string{"url": entry.URL, "target": rule.Target})}, nil
+}
+
+// refusal turns a 404 action refusal into the admin's answer: a rule that
+// already covers the address and a loop are a 403 whose message the admin
+// shows as a toast; a bad target is a 422 on the prompt's payload.
+func (c notFoundAdminController) refusal(ctx context.Context, svc *classes.Service, err error) error {
+	tr := translator(c.app)
+	var covered *classes.NotFoundHasRuleError
+	var loop *classes.LoopError
+	var problems *classes.ProblemsError
+	switch {
+	case errors.As(err, &covered):
+		return &cabana.ForbiddenError{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.has_rule", map[string]string{"url": covered.URL})}
+	case errors.As(err, &loop):
+		return &cabana.ForbiddenError{Message: svc.LoopMessage(ctx, loop.Check)}
+	case errors.As(err, &problems):
+		var msgs []string
+		for _, list := range problems.Problems {
+			for _, p := range list {
+				msgs = append(msgs, Phrase(ctx, tr, p.Key, p.Params))
+			}
+		}
+		return &cabana.ValidationError{Details: map[string]any{"payload": msgs}}
+	}
+	return err
+}
+
+// AdminBulkActions is the list's bulk "Mark as gone" (UI-SPEC C9): one
+// entry a rule already answers refuses the whole selection.
+func (c notFoundAdminController) AdminBulkActions() []pact.AdminBulkAction {
+	return []pact.AdminBulkAction{{
+		Name:        NotFoundActionGone,
+		Label:       "golem15.seo::lang.notfound.gone",
+		Confirm:     "golem15.seo::lang.notfound.gone_selected_confirm",
+		Permissions: []string{PermissionManage},
+		Run: func(ctx context.Context, in pact.AdminBulkActionInput) (pact.AdminBulkActionResult, error) {
+			svc := c.service()
+			if svc == nil {
+				return pact.AdminBulkActionResult{}, classes.ErrNotBooted
+			}
+			db, err := database(ctx, c.app)
+			if err != nil {
+				return pact.AdminBulkActionResult{}, err
+			}
+			n := 0
+			for _, record := range in.Records {
+				entry, ok := record.(*models.NotFound)
+				if !ok || entry == nil {
+					continue
+				}
+				if _, err := svc.ResolveNotFound(ctx, db, *entry, ""); err != nil {
+					return pact.AdminBulkActionResult{}, c.refusal(ctx, svc, err)
+				}
+				n++
+			}
+			return pact.AdminBulkActionResult{Message: "golem15.seo::lang.notfound.gone_selected_done", Affected: n}, nil
+		},
+	}}
+}
+
+// ListRowStates marks resolved entries disabled, labelled
+// notfound.state_resolved (UI-SPEC C9).
+func (notFoundAdminController) ListRowStates(ctx context.Context, db *gorm.DB, records []any) ([][]pact.RowState, error) {
+	set, err := classes.LoadRedirectSet(db.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	out := make([][]pact.RowState, len(records))
+	for i, record := range records {
+		entry, ok := record.(*models.NotFound)
+		if !ok || entry == nil {
+			continue
+		}
+		rule, err := classes.NotFoundCover(set, entry.URL)
+		if err != nil {
+			return nil, err
+		}
+		if rule != nil {
+			out[i] = []pact.RowState{pact.RowStateDisabled}
+		}
+	}
+	return out, nil
+}
+
+// HintView is the preview's header partial: the resolved hint, empty
+// while the entry is open.
+type HintView struct {
+	Text string
+}
+
+// PartialData serves the preview's hint (notfound.resolved_hint).
+func (c notFoundAdminController) PartialData(ctx context.Context, name string, record any) (any, error) {
+	entry, ok := record.(*models.NotFound)
+	if name != "hint" || !ok || entry == nil {
+		return HintView{}, nil
+	}
+	rule, err := c.cover(ctx, entry)
+	if err != nil || rule == nil {
+		return HintView{}, err
+	}
+	label := rule.Source
+	if rule.Target != "" {
+		label += " → " + rule.Target
+	} else {
+		label += " → 410"
+	}
+	return HintView{Text: Phrase(ctx, translator(c.app), "golem15.seo::lang.notfound.resolved_hint", map[string]string{"rule": label})}, nil
+}
+
+// FilterOptions serves the status filter's open and resolved choices.
+func (notFoundAdminController) FilterOptions(scope string) []pact.Option {
+	if scope != models.FilterNotFoundStatus {
+		return nil
+	}
+	return []pact.Option{
+		{Value: models.NotFoundOpen, Label: "golem15.seo::lang.notfound.filter_status_open"},
+		{Value: models.NotFoundResolved, Label: "golem15.seo::lang.notfound.filter_status_resolved"},
+	}
+}
+
+var (
+	_ pact.AdminController       = notFoundAdminController{}
+	_ pact.AdminPermissioned     = notFoundAdminController{}
+	_ pact.AdminRecordSource     = notFoundAdminController{}
+	_ pact.FormBeforeCreate      = notFoundAdminController{}
+	_ pact.FormBeforeUpdate      = notFoundAdminController{}
+	_ pact.HasAdminRecordActions = notFoundAdminController{}
+	_ pact.HasAdminBulkActions   = notFoundAdminController{}
+	_ pact.ListRowStates         = notFoundAdminController{}
+	_ pact.AdminPartialData      = notFoundAdminController{}
+	_ pact.FilterOptions         = notFoundAdminController{}
+)
--- a/controllers/notfound/_hint.htm
+++ b/controllers/notfound/_hint.htm
@@ -0,0 +1,5 @@
+{{- with .Data.Text -}}
+<div class="summer-callout" role="status">
+<p class="summer-callout__text">{{ . }}</p>
+</div>
+{{- end -}}
--- a/controllers/notfound/config_filter.yaml
+++ b/controllers/notfound/config_filter.yaml
@@ -0,0 +1,11 @@
+# 404 log filters (UI-SPEC C9): open or resolved, and the last hit.
+scopes:
+    status:
+        label: golem15.seo::lang.notfound.filter_status
+        modelClass: Golem15\SEO\Models\NotFound
+        nameFrom: url
+        scope: FilterNotFoundStatus
+    last_hit:
+        label: golem15.seo::lang.notfound.last_hit
+        type: daterange
+        column: last_hit
--- a/controllers/notfound/config_form.yaml
+++ b/controllers/notfound/config_form.yaml
@@ -0,0 +1,14 @@
+# 404 log preview (UI-SPEC C9): read-only, the resolved hint above the card
+# and the two actions that apply while the entry is open.
+name: golem15.seo::lang.notfound.name
+form: ~/plugins/golem15/seo/models/notfound/fields.yaml
+modelClass: Golem15\SEO\Models\NotFound
+defaultRedirect: golem15/seo/notfound
+
+preview:
+    headerPartial: hint
+
+recordActions: [redirect, gone]
+
+messages:
+    preview: golem15.seo::lang.notfound.preview
--- a/controllers/notfound/config_list.yaml
+++ b/controllers/notfound/config_list.yaml
@@ -0,0 +1,25 @@
+# 404 log (UI-SPEC C9): most hits first, resolved entries badged, no create.
+list: ~/plugins/golem15/seo/models/notfound/columns.yaml
+modelClass: Golem15\SEO\Models\NotFound
+title: golem15.seo::lang.notfound.list_title
+recordUrl: golem15/seo/notfound/preview/:id
+noRecordsMessage: golem15.seo::lang.notfound.list_empty
+recordsPerPage: 50
+showCheckboxes: true
+showSetup: true
+filter: config_filter.yaml
+bulkActions: [gone]
+
+defaultSort:
+    column: hits
+    direction: desc
+
+toolbar:
+    buttons: [delete]
+    search:
+        prompt: backend::lang.list.search_prompt
+
+messages:
+    empty: golem15.seo::lang.notfound.list_empty
+    deleteConfirm: golem15.seo::lang.notfound.delete_selected_confirm
+    rowStateDisabled: golem15.seo::lang.notfound.state_resolved
--- a/lang/en/lang.yaml
+++ b/lang/en/lang.yaml
@@ -2,6 +2,7 @@ plugin:
    name: SEO
    description: Search engine metadata for every content type, URL rules and global SEO settings.
 nav:
+    notfound: 404 log
    redirects: Redirects
    seo: SEO
    history: History
@@ -237,3 +238,33 @@ redirects:
    delete_selected_confirm:
        one: "Delete :count redirect? Its source address will return 404 again. You can undo it in History."
        other: "Delete :count redirects? Their source addresses will return 404 again. You can undo it in History."
+notfound:
+    list_title: "404 log"
+    name: "404 address"
+    url: "Address"
+    hits: "Hits"
+    last_hit: "Last hit"
+    first_seen: "First seen"
+    referer: "Last referer"
+    state_resolved: "Resolved"
+    filter_status: "Status"
+    filter_status_open: "Open"
+    filter_status_resolved: "Resolved"
+    preview: "Address that returned 404"
+    resolved_hint: "Resolved by “:rule”. This address no longer returns 404."
+    redirect: "Create redirect…"
+    redirect_prompt: "Redirect to (301)"
+    redirect_placeholder: "/new-address"
+    redirect_comment: "A path or a full URL. To use 302, edit the redirect afterwards."
+    redirect_confirm: "Create a 301 redirect from this address?"
+    redirected: "Redirect “:url” → “:target” created."
+    gone: "Mark as gone (410)"
+    gone_confirm: "Answer this address with 410 Gone? Search engines drop it faster than a 404. You can undo it in History."
+    gone_selected_confirm: "Answer the selected addresses with 410 Gone? You can undo it in History."
+    gone_done: "410 Gone created for “:url”."
+    gone_selected_done: "410 Gone created for the selected addresses."
+    has_rule: "A redirect for “:url” already exists. Edit it in Redirects."
+    list_empty: "No 404 errors recorded. Addresses that return 404 appear here with their hit count."
+    delete_selected_confirm:
+        one: "Remove :count address from the log? It is added again on its next 404."
+        other: "Remove :count addresses from the log? They are added again on their next 404."
--- a/lang/pl/lang.yaml
+++ b/lang/pl/lang.yaml
@@ -2,6 +2,7 @@ plugin:
    name: SEO
    description: Metadane dla wyszukiwarek we wszystkich typach treści, reguły adresów i globalne ustawienia SEO.
 nav:
+    notfound: Dziennik 404
    redirects: Przekierowania
    seo: SEO
    history: Historia zmian
@@ -243,3 +244,35 @@ redirects:
        few: "Usunąć :count przekierowania? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian."
        many: "Usunąć :count przekierowań? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian."
        other: "Usunąć :count przekierowania? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian."
+notfound:
+    list_title: "Dziennik 404"
+    name: "Adres 404"
+    url: "Adres"
+    hits: "Wejścia"
+    last_hit: "Ostatnie wejście"
+    first_seen: "Pierwsze wejście"
+    referer: "Ostatnie źródło"
+    state_resolved: "Rozwiązany"
+    filter_status: "Stan"
+    filter_status_open: "Otwarte"
+    filter_status_resolved: "Rozwiązane"
+    preview: "Adres, który zwrócił 404"
+    resolved_hint: "Rozwiązane przez „:rule”. Ten adres nie zwraca już 404."
+    redirect: "Utwórz przekierowanie…"
+    redirect_prompt: "Przekieruj na (301)"
+    redirect_placeholder: "/nowy-adres"
+    redirect_comment: "Ścieżka lub pełny adres. Aby użyć 302, edytuj potem przekierowanie."
+    redirect_confirm: "Utworzyć przekierowanie 301 z tego adresu?"
+    redirected: "Utworzono przekierowanie „:url” → „:target”."
+    gone: "Oznacz jako usunięty (410)"
+    gone_confirm: "Odpowiadać na ten adres kodem 410 Gone? Wyszukiwarki usuną go szybciej niż przy 404. Możesz to cofnąć w Historii zmian."
+    gone_selected_confirm: "Odpowiadać kodem 410 Gone na zaznaczone adresy? Możesz to cofnąć w Historii zmian."
+    gone_done: "Utworzono 410 Gone dla „:url”."
+    gone_selected_done: "Utworzono 410 Gone dla zaznaczonych adresów."
+    has_rule: "Przekierowanie dla „:url” już istnieje. Edytuj je w Przekierowaniach."
+    list_empty: "Nie zarejestrowano błędów 404. Adresy zwracające 404 pojawią się tutaj z liczbą wejść."
+    delete_selected_confirm:
+        one: "Usunąć :count adres z dziennika? Wróci przy następnym błędzie 404."
+        few: "Usunąć :count adresy z dziennika? Wrócą przy następnym błędzie 404."
+        many: "Usunąć :count adresów z dziennika? Wrócą przy następnym błędzie 404."
+        other: "Usunąć :count adresu z dziennika? Wrócą przy następnym błędzie 404."
--- a/models/notfound.go
+++ b/models/notfound.go
@@ -0,0 +1,65 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+// NotFound is one address of the 404 log on golem15_seo_notfound (D-20):
+// the path (with its query) that answered 404, how often, when first and
+// last, and the referer of the last hit. The frontend reports each 404
+// through the authenticated report endpoint.
+type NotFound struct {
+	ID        uint       `gorm:"column:id;primaryKey"`
+	URL       string     `gorm:"column:url"`
+	Hits      int64      `gorm:"column:hits"`
+	FirstSeen *time.Time `gorm:"column:first_seen"`
+	LastHit   *time.Time `gorm:"column:last_hit"`
+	Referer   string     `gorm:"column:referer"`
+}
+
+func (NotFound) TableName() string { return "golem15_seo_notfound" }
+
+// MorphName names a 404 log entry in history.
+func (NotFound) MorphName() string { return `Golem15\SEO\Models\NotFound` }
+
+// Fillable is empty: entries come from the report endpoint only, and the
+// preview edits nothing.
+func (NotFound) Fillable() []string { return []string{} }
+
+// Rules is empty: nothing is edited.
+func (NotFound) Rules() map[string]string { return map[string]string{} }
+
+// The 404 log's status filter scope and its values.
+const (
+	FilterNotFoundStatus = "FilterNotFoundStatus"
+	NotFoundOpen         = "open"
+	NotFoundResolved     = "resolved"
+)
+
+// FilterScopes lists the 404 log's filter scopes.
+func (NotFound) FilterScopes() []string { return []string{FilterNotFoundStatus} }
+
+// coveredSQL is true for an entry an exact redirect or 410 answers now
+// (the list filter; the Resolved badge also counts patterns and regex).
+const coveredSQL = `EXISTS (SELECT 1 FROM golem15_seo_redirects AS r WHERE r.match = 'exact' AND (r.source = golem15_seo_notfound.url OR (r.ignore_case AND lower(r.source) = lower(golem15_seo_notfound.url))))`
+
+// FilterScope narrows the 404 log to open or resolved entries; an unknown
+// value matches nothing.
+func (NotFound) FilterScope(name string, db *gorm.DB, value any) *gorm.DB {
+	if db == nil || name != FilterNotFoundStatus {
+		return db
+	}
+	switch s, _ := value.(string); s {
+	case NotFoundOpen:
+		return db.Where("NOT " + coveredSQL)
+	case NotFoundResolved:
+		return db.Where(coveredSQL)
+	}
+	return db.Where("1 = 0")
+}
+
+func init() {
+	Register(NotFound{})
+}
--- a/models/notfound/columns.yaml
+++ b/models/notfound/columns.yaml
@@ -0,0 +1,23 @@
+# 404 log columns (UI-SPEC C9).
+columns:
+    url:
+        label: golem15.seo::lang.notfound.url
+        type: text
+        searchable: true
+        sortable: true
+    hits:
+        label: golem15.seo::lang.notfound.hits
+        type: text
+        sortable: true
+    last_hit:
+        label: golem15.seo::lang.notfound.last_hit
+        type: datetime
+        sortable: true
+    referer:
+        label: golem15.seo::lang.notfound.referer
+        type: text
+    first_seen:
+        label: golem15.seo::lang.notfound.first_seen
+        type: datetime
+        sortable: true
+        invisible: true
--- a/models/notfound/fields.yaml
+++ b/models/notfound/fields.yaml
@@ -0,0 +1,29 @@
+# 404 log preview (UI-SPEC C9): every field is read-only.
+fields:
+    url:
+        label: golem15.seo::lang.notfound.url
+        type: text
+        span: full
+        context: preview
+    hits:
+        label: golem15.seo::lang.notfound.hits
+        type: number
+        span: left
+        context: preview
+    last_hit:
+        label: golem15.seo::lang.notfound.last_hit
+        type: datepicker
+        mode: datetime
+        span: right
+        context: preview
+    first_seen:
+        label: golem15.seo::lang.notfound.first_seen
+        type: datepicker
+        mode: datetime
+        span: left
+        context: preview
+    referer:
+        label: golem15.seo::lang.notfound.referer
+        type: text
+        span: right
+        context: preview
--- a/redirects_test.go
+++ b/redirects_test.go
@@ -1,12 +1,16 @@
 package seo_test

 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"net/http"
+	"net/http/httptest"
 	"sort"
+	"strconv"
 	"strings"
 	"testing"
+	"time"

 	"git.golem15.com/golem15/sm-seo-plugin/classes"
 	"git.golem15.com/golem15/sm-seo-plugin/models"
@@ -416,3 +420,158 @@ func TestLoopRefused(t *testing.T) {
 		t.Fatalf("redirect history rows = %d, want every admin save recorded", len(rows))
 	}
 }
+
+// report posts a 404 report with token ("" sends none).
+func (e *testEnv) report(t *testing.T, token string, body any) *httptest.ResponseRecorder {
+	t.Helper()
+	raw, err := json.Marshal(body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := httptest.NewRequest(http.MethodPost, "/_seo/api/v1/notfound", bytes.NewReader(raw))
+	req.Header.Set("Content-Type", "application/json")
+	req.RemoteAddr = "198.51.100.7:4000"
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	rec := httptest.NewRecorder()
+	e.h.ServeHTTP(rec, req)
+	return rec
+}
+
+// notFoundRow is a 404 log row as the tests read it.
+type notFoundRow struct {
+	ID        uint
+	URL       string
+	Hits      int64
+	FirstSeen time.Time
+	LastHit   time.Time
+	Referer   string
+}
+
+func (e *testEnv) notFoundRow(t *testing.T, url string) notFoundRow {
+	t.Helper()
+	var row notFoundRow
+	if err := e.db.Raw(`SELECT id, url, hits, first_seen, last_hit, referer FROM golem15_seo_notfound WHERE url = ?`, url).Scan(&row).Error; err != nil || row.ID == 0 {
+		t.Fatalf("404 log row %s: %+v err=%v", url, row, err)
+	}
+	return row
+}
+
+// TestNotFoundLog checks the 404 log (D-20, P-22, UI-SPEC C9): the
+// authenticated report endpoint, hits and referers, the prompted redirect
+// and the 410 record actions, the Resolved state, and the bulk 410 that
+// refuses a selection containing a covered address.
+func TestNotFoundLog(t *testing.T) {
+	off := bootEnv(t)
+	if rec := off.report(t, "anything", map[string]string{"url": "/x"}); rec.Code != http.StatusNotFound {
+		t.Fatalf("report without a configured token = %d, want 404", rec.Code)
+	}
+
+	const secret = "notfound-test-token"
+	env := bootEnv(t, "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN="+secret)
+	if rec := env.report(t, "", map[string]string{"url": "/missing"}); rec.Code != http.StatusUnauthorized {
+		t.Fatalf("report without a token = %d, want 401", rec.Code)
+	}
+	if rec := env.report(t, "wrong", map[string]string{"url": "/missing"}); rec.Code != http.StatusUnauthorized {
+		t.Fatalf("report with a wrong token = %d, want 401", rec.Code)
+	}
+	if rec := env.report(t, secret, map[string]string{"url": "/" + strings.Repeat("a", 2100)}); rec.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("report of a long URL = %d, want 422", rec.Code)
+	}
+	if rec := env.report(t, secret, map[string]string{"url": "/missing", "referer": "https://ref.example/a"}); rec.Code != http.StatusNoContent || rec.Header().Get("X-Robots-Tag") != "noindex" {
+		t.Fatalf("report = %d %v, want 204 with X-Robots-Tag", rec.Code, rec.Header())
+	}
+	first := env.notFoundRow(t, "/missing")
+	time.Sleep(1100 * time.Millisecond)
+	env.report(t, secret, map[string]string{"url": "https://www.example.test/missing", "referer": "https://ref.example/b"})
+	again := env.notFoundRow(t, "/missing")
+	if again.Hits != 2 || !again.FirstSeen.Equal(first.FirstSeen) || !again.LastHit.After(first.LastHit) || again.Referer != "https://ref.example/b" {
+		t.Fatalf("second report = %+v (first %+v), want hits 2, first seen kept, last hit and referer updated", again, first)
+	}
+	for _, u := range []string{"/old-a", "/old-b", "/old-c", "/old-c"} {
+		env.report(t, secret, map[string]string{"url": u})
+	}
+
+	token := env.admin(t, true)
+	api := adminAPI("/golem15/seo/notfound")
+	var list struct {
+		Data []struct {
+			ID  json.Number `json:"id"`
+			URL string      `json:"url"`
+		} `json:"data"`
+		Meta struct {
+			RowStates map[string][]string `json:"row_states"`
+		} `json:"meta"`
+	}
+	if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api, token, nil).Body.Bytes(), &list); err != nil {
+		t.Fatal(err)
+	}
+	if len(list.Data) != 4 || (list.Data[0].URL != "/missing" && list.Data[0].URL != "/old-c") {
+		t.Fatalf("404 list = %+v, want four entries, most hits first", list.Data)
+	}
+
+	missing := strconv.FormatUint(uint64(again.ID), 10)
+	rec := env.expect(t, http.StatusOK, http.MethodPost, api+"/"+missing+"/actions/redirect", token, map[string]any{"payload": "/found"})
+	if !strings.Contains(rec.Body.String(), "/missing") || !strings.Contains(rec.Body.String(), "/found") {
+		t.Fatalf("redirect action = %s, want the redirected message", rec.Body.String())
+	}
+	rows := env.redirects(t)
+	if target := redirectTo(rows, "/missing"); target != "/found" {
+		t.Fatalf("redirect from the 404 log = %+v, want /missing -> /found", rows)
+	}
+	for _, r := range rows {
+		if r.Source == "/missing" && (r.Origin != "notfound" || r.Status != 301) {
+			t.Fatalf("404 redirect = %+v, want origin notfound and 301", r)
+		}
+	}
+	env.expectRedirect(t, "/missing", 301, siteURL+"/found")
+	show := env.expect(t, http.StatusOK, http.MethodGet, api+"/"+missing, token, nil).Body.String()
+	if strings.Contains(show, `"actions"`) {
+		t.Fatalf("a resolved entry still offers actions: %s", show)
+	}
+	if rec := env.call(t, http.MethodPost, api+"/"+missing+"/actions/gone", token, map[string]any{}); rec.Code != http.StatusConflict {
+		t.Fatalf("gone on a resolved entry = %d %s, want 409", rec.Code, rec.Body.String())
+	}
+	hint := env.expect(t, http.StatusOK, http.MethodGet, adminAPI("/golem15/seo/notfound/partials/hint?id="+missing), token, nil).Body.String()
+	if !strings.Contains(hint, "Resolved by") || !strings.Contains(hint, "/missing → /found") {
+		t.Fatalf("hint = %s, want the resolved hint", hint)
+	}
+
+	oldA := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-a").ID), 10)
+	oldB := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-b").ID), 10)
+	oldC := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-c").ID), 10)
+	env.expect(t, http.StatusOK, http.MethodPost, api+"/"+oldA+"/actions/gone", token, map[string]any{})
+	if got := env.resolve(t, "/old-a"); got["action"] != "gone" {
+		t.Fatalf("/old-a after gone = %v, want gone", got)
+	}
+	before := env.redirectCount(t)
+	rec = env.call(t, http.MethodPost, api+"/bulk/gone", token, map[string]any{"ids": []string{oldA, oldB, oldC}})
+	if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A redirect for “/old-a” already exists") {
+		t.Fatalf("bulk gone over a covered entry = %d %s, want 403 notfound.has_rule", rec.Code, rec.Body.String())
+	}
+	if after := env.redirectCount(t); after != before {
+		t.Fatalf("redirects after a refused bulk = %d, want %d", after, before)
+	}
+	rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/bulk/gone", token, map[string]any{"ids": []string{oldB, oldC}})
+	if !strings.Contains(rec.Body.String(), `"affected":2`) {
+		t.Fatalf("bulk gone = %s, want 2 affected", rec.Body.String())
+	}
+	if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api, token, nil).Body.Bytes(), &list); err != nil {
+		t.Fatal(err)
+	}
+	for _, id := range []string{missing, oldA, oldB, oldC} {
+		if states := list.Meta.RowStates[id]; len(states) != 1 || states[0] != "disabled" {
+			t.Fatalf("row states = %v, want %s resolved", list.Meta.RowStates, id)
+		}
+	}
+	var open []string
+	if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api+"?filter[status]=open", token, nil).Body.Bytes(), &list); err == nil {
+		for _, d := range list.Data {
+			open = append(open, d.URL)
+		}
+	}
+	if len(open) != 0 {
+		t.Fatalf("open filter = %v, want none", open)
+	}
+}
--- a/routes.go
+++ b/routes.go
@@ -9,34 +9,54 @@ import (
 	"git.golem15.com/golem15/summercms/modules/surf"
 )

-// BucketResolve is the throttle bucket of the public resolve endpoint.
-const BucketResolve = "seo-resolve"
+// The throttle buckets of the public endpoints.
+const (
+	// BucketResolve is the resolve endpoint's bucket.
+	BucketResolve = "seo-resolve"
+	// BucketNotFound is the 404 report endpoint's bucket.
+	BucketNotFound = "seo-notfound"
+)

-// Routes mounts the public /_seo/api/v1 endpoints. They carry no backend
-// middleware: the frontend server calls them anonymously.
+// Routes mounts the /_seo/api/v1 endpoints. They carry no backend
+// middleware: the frontend server calls them anonymously (the 404 report
+// with its own token). Every answer carries X-Robots-Tag: noindex.
 func (p *Plugin) Routes(r pact.Router) error {
-	r.Group("/_seo/api/v1", surf.Use("throttle:"+BucketResolve), func(g pact.Router) {
-		g.Get("/resolve", apicontrollers.Resolve(p.Service))
+	r.Group("/_seo/api/v1", nil, func(g pact.Router) {
+		g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve)
+		g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound)
 	})
 	return nil
 }

-// Buckets registers seo-resolve: 600 requests a minute per client IP behind
-// the configured trusted proxies (the frontend server resolves every page
-// it renders).
+// noindex adds X-Robots-Tag: noindex to every answer of h.
+func noindex(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Robots-Tag", "noindex")
+		h(w, r)
+	}
+}
+
+// Buckets registers the per-client-IP buckets behind the configured
+// trusted proxies: seo-resolve 600 a minute (the frontend server resolves
+// every page it renders) and seo-notfound 600 a minute (one report per
+// 404 page).
 func (p *Plugin) Buckets() map[string]surf.Bucket {
 	trusted := surf.TrustedProxies(nil)
 	if p.app != nil {
 		trusted = surf.TrustedProxies(p.app.Config)
 	}
-	return map[string]surf.Bucket{
-		BucketResolve: {
-			Name:  BucketResolve,
-			Max:   600,
+	bucket := func(name string, max int) surf.Bucket {
+		return surf.Bucket{
+			Name:  name,
+			Max:   max,
 			Decay: time.Minute,
 			Key: func(r *http.Request) string {
-				return BucketResolve + "|" + surf.ClientIP(r, trusted)
+				return name + "|" + surf.ClientIP(r, trusted)
 			},
-		},
+		}
+	}
+	return map[string]surf.Bucket{
+		BucketResolve:  bucket(BucketResolve, 600),
+		BucketNotFound: bucket(BucketNotFound, 600),
 	}
 }
--- a/updates/202610270007_create_golem15_seo_notfound.go
+++ b/updates/202610270007_create_golem15_seo_notfound.go
@@ -0,0 +1,32 @@
+package updates
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// createNotFound creates golem15_seo_notfound: the 404 log (D-20), one row
+// per address.
+var createNotFound = []*gormigrate.Migration{
+	{
+		ID: "202610270007_create_golem15_seo_notfound",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.Exec(`CREATE TABLE golem15_seo_notfound (
+	id BIGSERIAL PRIMARY KEY,
+	url TEXT NOT NULL,
+	hits BIGINT NOT NULL DEFAULT 1,
+	first_seen TIMESTAMP(0) NULL,
+	last_hit TIMESTAMP(0) NULL,
+	referer TEXT NOT NULL DEFAULT '',
+	CONSTRAINT golem15_seo_notfound_url UNIQUE (url)
+)`).Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_notfound`).Error
+		},
+	},
+}
+
+func init() {
+	Register(createNotFound...)
+}