From 6eb6cb7a4dff1c861f2f72367222d70fb812c85b Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Tue, 27 Oct 2026 11:35:25 +0100 Subject: [PATCH] feat: 404 log and report endpoint - POST /_seo/api/v1/notfound with the golem15.seo.notfound_token bearer (constant-time, off when unset), URL and referer caps, a row cap with an hourly warning, throttle bucket seo-notfound - golem15.seo.notfound list by hits with Resolved rows, a preview with the resolved hint, a prompted 301 and a 410 record action, a bulk 410 that refuses a selection with a covered address - every /_seo/api/v1 answer carries X-Robots-Tag: noindex --- admin_navigation.go | 7 + classes/notfound.go | 181 ++++++++++++ config/config.yaml | 9 + controllers/admin_registry.go | 1 + controllers/api/notfound.go | 64 ++++ controllers/notfound.go | 273 ++++++++++++++++++ controllers/notfound/_hint.htm | 5 + controllers/notfound/config_filter.yaml | 11 + controllers/notfound/config_form.yaml | 14 + controllers/notfound/config_list.yaml | 25 ++ lang/en/lang.yaml | 31 ++ lang/pl/lang.yaml | 33 +++ models/notfound.go | 65 +++++ models/notfound/columns.yaml | 23 ++ models/notfound/fields.yaml | 29 ++ redirects_test.go | 159 ++++++++++ routes.go | 50 +++- ...02610270007_create_golem15_seo_notfound.go | 32 ++ 18 files changed, 997 insertions(+), 15 deletions(-) create mode 100644 classes/notfound.go create mode 100644 controllers/api/notfound.go create mode 100644 controllers/notfound.go create mode 100644 controllers/notfound/_hint.htm create mode 100644 controllers/notfound/config_filter.yaml create mode 100644 controllers/notfound/config_form.yaml create mode 100644 controllers/notfound/config_list.yaml create mode 100644 models/notfound.go create mode 100644 models/notfound/columns.yaml create mode 100644 models/notfound/fields.yaml create mode 100644 updates/202610270007_create_golem15_seo_notfound.go diff --git a/admin_navigation.go b/admin_navigation.go index 101650f..f148f2e 100644 --- a/admin_navigation.go +++ b/admin_navigation.go @@ -24,6 +24,13 @@ func (*Plugin) Navigation() []pact.NavigationItem { Permissions: []string{controllers.PermissionManage}, Controller: controllers.RedirectsControllerID, }, + { + Code: "notfound", + Label: "golem15.seo::lang.nav.notfound", + Icon: "triangle-alert", + Permissions: []string{controllers.PermissionManage}, + Controller: controllers.NotFoundControllerID, + }, { Code: "rules", Label: "golem15.seo::lang.nav.rules", diff --git a/classes/notfound.go b/classes/notfound.go new file mode 100644 index 0000000..e2ed304 --- /dev/null +++ b/classes/notfound.go @@ -0,0 +1,181 @@ +package classes + +import ( + "context" + "errors" + "net/url" + "strings" + "sync" + "time" + + "git.golem15.com/golem15/sm-seo-plugin/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// Config keys of the 404 log (P-22). +const ( + // NotFoundTokenConfigKey is the shared secret the frontend server + // sends as "Authorization: Bearer " with each 404 report. Empty + // turns the report endpoint off (it answers 404). + NotFoundTokenConfigKey = "golem15.seo.notfound_token" + // NotFoundMaxRowsConfigKey caps the 404 log; new addresses beyond it + // are dropped (default DefaultNotFoundMaxRows). + NotFoundMaxRowsConfigKey = "golem15.seo.notfound_max_rows" +) + +// The 404 report limits (P-22). +const ( + DefaultNotFoundMaxRows = 10000 + MaxNotFoundURLBytes = 2048 + MaxRefererBytes = 1024 +) + +// ErrNotFoundURL refuses a reported address that is not a path or an +// http(s) URL, or is longer than MaxNotFoundURLBytes. +var ErrNotFoundURL = errors.New("golem15.seo: url must be a path or an absolute http(s) URL of at most 2048 bytes") + +// NotFoundToken is the configured report token, "" when unset. +func (s *Service) NotFoundToken() string { + cfg := s.Config() + if cfg == nil { + return "" + } + return strings.TrimSpace(cfg.String(NotFoundTokenConfigKey)) +} + +// NotFoundMaxRows is the configured row cap of the 404 log. +func (s *Service) NotFoundMaxRows() int64 { + if cfg := s.Config(); cfg != nil { + if n := cfg.Int(NotFoundMaxRowsConfigKey); n > 0 { + return int64(n) + } + } + return DefaultNotFoundMaxRows +} + +// NotFoundPath turns a reported address into the logged one: the path +// with its query; an absolute URL loses its scheme and host. +func NotFoundPath(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" || len(raw) > MaxNotFoundURLBytes { + return "", ErrNotFoundURL + } + u, err := url.Parse(raw) + if err != nil { + return "", ErrNotFoundURL + } + if u.IsAbs() { + if u.Scheme != "http" && u.Scheme != "https" { + return "", ErrNotFoundURL + } + } else if !strings.HasPrefix(raw, "/") || strings.HasPrefix(raw, "//") { + return "", ErrNotFoundURL + } + p := u.Path + if p == "" { + p = "/" + } + return joinQuery(p, u.RawQuery), nil +} + +// capWarned keeps when the row-cap warning was last logged. +var capWarned struct { + sync.Mutex + at time.Time +} + +// ReportNotFound records one 404 of rawURL (D-20, P-22): a known address +// gets one more hit, its last hit time and, when given, the referer; a new +// one is added unless the log holds NotFoundMaxRows rows, which is logged +// at most once an hour. +func (s *Service) ReportNotFound(ctx context.Context, rawURL, referer string) error { + path, err := NotFoundPath(rawURL) + if err != nil { + return err + } + referer = strings.TrimSpace(referer) + if len(referer) > MaxRefererBytes { + referer = referer[:MaxRefererBytes] + for len(referer) > 0 && !utf8ValidTail(referer) { + referer = referer[:len(referer)-1] + } + } + db, err := s.DB(ctx) + if err != nil { + return err + } + now := time.Now().UTC() + updates := map[string]any{"hits": gorm.Expr("hits + 1"), "last_hit": now} + if referer != "" { + updates["referer"] = referer + } + res := db.Model(&models.NotFound{}).Where("url = ?", path).Updates(updates) + if res.Error != nil || res.RowsAffected > 0 { + return res.Error + } + var count int64 + if err := db.Model(&models.NotFound{}).Count(&count).Error; err != nil { + return err + } + if count >= s.NotFoundMaxRows() { + capWarned.Lock() + if time.Since(capWarned.at) >= time.Hour { + capWarned.at = time.Now() + log.Warn("golem15.seo: the 404 log is full, new addresses are dropped", "max_rows", s.NotFoundMaxRows(), "url", path) + } + capWarned.Unlock() + return nil + } + row := models.NotFound{URL: path, Hits: 1, FirstSeen: &now, LastHit: &now, Referer: referer} + return db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "url"}}, + DoUpdates: clause.Assignments(map[string]any{"hits": gorm.Expr("golem15_seo_notfound.hits + 1"), "last_hit": now}), + }).Create(&row).Error +} + +// utf8ValidTail reports whether s does not end inside a UTF-8 sequence. +func utf8ValidTail(s string) bool { + return strings.ToValidUTF8(s, "�") == s +} + +// NotFoundCover returns the rule that answers url now (a redirect or a +// 410), nil when the address still answers 404: the Resolved state. +func NotFoundCover(set *RedirectSet, url string) (*models.Redirect, error) { + return CoveringRedirect(set, url) +} + +// NotFoundHasRuleError refuses a 404 action on an address a rule already +// answers. +type NotFoundHasRuleError struct{ URL string } + +func (e *NotFoundHasRuleError) Error() string { + return "golem15.seo: a redirect for " + e.URL + " already exists" +} + +// ResolveNotFound creates the rule of a 404 log action inside tx (D-20): a +// 301 to target when target is set, else a 410, with origin notfound. It +// refuses an address a rule already answers (*NotFoundHasRuleError), and +// the redirect's own refusals (*ProblemsError, *LoopError). +func (s *Service) ResolveNotFound(ctx context.Context, tx *gorm.DB, entry models.NotFound, target string) (models.Redirect, error) { + set, err := LoadRedirectSet(tx) + if err != nil { + return models.Redirect{}, err + } + if rule, err := NotFoundCover(set, entry.URL); err != nil { + return models.Redirect{}, err + } else if rule != nil { + return models.Redirect{}, &NotFoundHasRuleError{URL: entry.URL} + } + r := models.NewRedirect() + r.Source, r.Origin = entry.URL, models.OriginNotFound + if strings.TrimSpace(target) == "" { + r.Status = models.Status410 + } else { + r.Target = strings.TrimSpace(target) + } + if _, err := s.SaveRedirect(ctx, tx, &r); err != nil { + return r, err + } + return r, nil +} diff --git a/config/config.yaml b/config/config.yaml index 8b13b87..b0b703a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -10,3 +10,12 @@ site_url: "" # prefix (for example pl on a site whose English pages start with /en). # Empty leaves the choice to each content type. unprefixed_locale: "" +# +# notfound_token is the shared secret the frontend server sends as +# "Authorization: Bearer " with each 404 report to +# POST /_seo/api/v1/notfound. Empty turns the endpoint off (it answers 404). +notfound_token: "" +# +# notfound_max_rows caps the 404 log; new addresses beyond it are dropped, +# with one warning an hour. +notfound_max_rows: 10000 diff --git a/controllers/admin_registry.go b/controllers/admin_registry.go index fa12432..ef88832 100644 --- a/controllers/admin_registry.go +++ b/controllers/admin_registry.go @@ -30,6 +30,7 @@ func AdminControllers(app func() *backpack.App) []pact.AdminController { return []pact.AdminController{ settingsAdminController{app: app}, redirectsAdminController{app: app}, + notFoundAdminController{app: app}, rulesAdminController{app: app}, historyAdminController{app: app}, } diff --git a/controllers/api/notfound.go b/controllers/api/notfound.go new file mode 100644 index 0000000..17dfb20 --- /dev/null +++ b/controllers/api/notfound.go @@ -0,0 +1,64 @@ +package api + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "net/http" + "strings" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/summercms/modules/wire" +) + +// maxReportBody caps the 404 report body: the URL and the referer fit +// with room to spare. +const maxReportBody = 8 << 10 + +// notFoundReport is the 404 report body. +type notFoundReport struct { + URL string `json:"url"` + Referer string `json:"referer"` +} + +// NotFound serves POST /_seo/api/v1/notfound (D-20, P-22): the frontend +// server reports a 404 with "Authorization: Bearer ", the configured +// golem15.seo.notfound_token, compared in constant time. It answers 204, +// 401 for a missing or wrong token, 404 when no token is configured (the +// endpoint is off), and 422 for a body without a path or http(s) URL of at +// most 2048 bytes. +func NotFound(service func() *classes.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + svc := service() + if svc == nil { + wire.WriteJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "SEO is not available"}) + return + } + want := svc.NotFoundToken() + if want == "" { + wire.WriteJSON(w, http.StatusNotFound, map[string]string{"error": "Not Found"}) + return + } + got, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(want)) != 1 { + wire.WriteJSON(w, http.StatusUnauthorized, map[string]string{"error": "Unauthorized"}) + return + } + var in notFoundReport + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReportBody)) + if err := dec.Decode(&in); err != nil { + wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": "The request body is invalid."}) + return + } + if err := svc.ReportNotFound(r.Context(), in.URL, in.Referer); err != nil { + if errors.Is(err, classes.ErrNotFoundURL) { + wire.WriteJSON(w, http.StatusUnprocessableEntity, map[string]string{"error": err.Error()}) + return + } + log.Error("golem15.seo: 404 report", "error", err) + wire.WriteJSON(w, http.StatusInternalServerError, map[string]string{"error": "Internal Server Error"}) + return + } + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/controllers/notfound.go b/controllers/notfound.go new file mode 100644 index 0000000..d8a24f7 --- /dev/null +++ b/controllers/notfound.go @@ -0,0 +1,273 @@ +package controllers + +import ( + "context" + "errors" + + "git.golem15.com/golem15/sm-seo-plugin/classes" + "git.golem15.com/golem15/sm-seo-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/cabana" + "git.golem15.com/golem15/summercms/modules/pact" + "gorm.io/gorm" +) + +// NotFoundControllerID is the 404 log (UI-SPEC C9). +const NotFoundControllerID = "golem15.seo.notfound" + +// errNotFoundReadOnly refuses creating or editing 404 log entries: they +// come from the report endpoint only. +var errNotFoundReadOnly = errors.New("golem15.seo: the 404 log is read-only") + +// The 404 log's record and bulk action names. +const ( + NotFoundActionRedirect = "redirect" + NotFoundActionGone = "gone" +) + +// notFoundAdminController lists the 404 log by hits and previews an entry +// with two record actions, a prompted "Create redirect…" (301) and "Mark +// as gone" (410), and the bulk "Mark as gone" (D-20). An entry a redirect +// or a 410 now answers is Resolved: badged, and its actions no longer +// apply. +type notFoundAdminController struct { + app func() *backpack.App +} + +func (notFoundAdminController) ID() string { return NotFoundControllerID } +func (notFoundAdminController) ModelName() string { return `Golem15\SEO\Models\NotFound` } +func (notFoundAdminController) ConfigDir() string { return "controllers/notfound" } +func (notFoundAdminController) NewRecord() any { return &models.NotFound{} } +func (notFoundAdminController) RequiredPermissions() []string { + return []string{PermissionManage} +} + +// FormBeforeCreate refuses: entries come from the report endpoint. +func (notFoundAdminController) FormBeforeCreate(context.Context, any) error { + return errNotFoundReadOnly +} + +// FormBeforeUpdate refuses: entries are never edited. +func (notFoundAdminController) FormBeforeUpdate(context.Context, any) error { + return errNotFoundReadOnly +} + +func (c notFoundAdminController) service() *classes.Service { + svc, err := classes.NewService(appOf(c.app)) + if err != nil { + return nil + } + return svc +} + +// cover returns the rule that answers the entry now, nil while it is +// open. +func (c notFoundAdminController) cover(ctx context.Context, entry *models.NotFound) (*models.Redirect, error) { + db, err := database(ctx, c.app) + if err != nil { + return nil, err + } + set, err := classes.LoadRedirectSet(db) + if err != nil { + return nil, err + } + return classes.NotFoundCover(set, entry.URL) +} + +// open reports whether a record is an entry no rule answers yet. +func (c notFoundAdminController) open(ctx context.Context, record any) (bool, error) { + entry, ok := record.(*models.NotFound) + if !ok || entry == nil { + return false, nil + } + rule, err := c.cover(ctx, entry) + return rule == nil, err +} + +// AdminRecordActions are the preview's two actions (UI-SPEC C9); both +// apply only while the entry is open. +func (c notFoundAdminController) AdminRecordActions() []pact.AdminRecordAction { + return []pact.AdminRecordAction{ + { + Name: NotFoundActionRedirect, + Label: "golem15.seo::lang.notfound.redirect", + Confirm: "golem15.seo::lang.notfound.redirect_confirm", + Permissions: []string{PermissionManage}, + Prompt: &pact.AdminActionPrompt{ + Label: "golem15.seo::lang.notfound.redirect_prompt", + Placeholder: "golem15.seo::lang.notfound.redirect_placeholder", + Comment: "golem15.seo::lang.notfound.redirect_comment", + MaxLength: models.MaxRedirectURL, + }, + Applies: c.open, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + return c.resolve(ctx, in.Record, in.Value) + }, + }, + { + Name: NotFoundActionGone, + Label: "golem15.seo::lang.notfound.gone", + Confirm: "golem15.seo::lang.notfound.gone_confirm", + Permissions: []string{PermissionManage}, + Danger: true, + Applies: c.open, + Run: func(ctx context.Context, in pact.AdminRecordActionInput) (pact.AdminRecordActionResult, error) { + return c.resolve(ctx, in.Record, "") + }, + }, + } +} + +// resolve creates the entry's 301 to target, or its 410 when target is +// empty, inside the action's transaction. +func (c notFoundAdminController) resolve(ctx context.Context, record any, target string) (pact.AdminRecordActionResult, error) { + entry, ok := record.(*models.NotFound) + svc := c.service() + if !ok || entry == nil || svc == nil { + return pact.AdminRecordActionResult{}, classes.ErrNotBooted + } + db, err := database(ctx, c.app) + if err != nil { + return pact.AdminRecordActionResult{}, err + } + rule, err := svc.ResolveNotFound(ctx, db, *entry, target) + if err != nil { + return pact.AdminRecordActionResult{}, c.refusal(ctx, svc, err) + } + tr := translator(c.app) + if rule.Status == models.Status410 { + return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.gone_done", map[string]string{"url": entry.URL})}, nil + } + return pact.AdminRecordActionResult{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.redirected", map[string]string{"url": entry.URL, "target": rule.Target})}, nil +} + +// refusal turns a 404 action refusal into the admin's answer: a rule that +// already covers the address and a loop are a 403 whose message the admin +// shows as a toast; a bad target is a 422 on the prompt's payload. +func (c notFoundAdminController) refusal(ctx context.Context, svc *classes.Service, err error) error { + tr := translator(c.app) + var covered *classes.NotFoundHasRuleError + var loop *classes.LoopError + var problems *classes.ProblemsError + switch { + case errors.As(err, &covered): + return &cabana.ForbiddenError{Message: Phrase(ctx, tr, "golem15.seo::lang.notfound.has_rule", map[string]string{"url": covered.URL})} + case errors.As(err, &loop): + return &cabana.ForbiddenError{Message: svc.LoopMessage(ctx, loop.Check)} + case errors.As(err, &problems): + var msgs []string + for _, list := range problems.Problems { + for _, p := range list { + msgs = append(msgs, Phrase(ctx, tr, p.Key, p.Params)) + } + } + return &cabana.ValidationError{Details: map[string]any{"payload": msgs}} + } + return err +} + +// AdminBulkActions is the list's bulk "Mark as gone" (UI-SPEC C9): one +// entry a rule already answers refuses the whole selection. +func (c notFoundAdminController) AdminBulkActions() []pact.AdminBulkAction { + return []pact.AdminBulkAction{{ + Name: NotFoundActionGone, + Label: "golem15.seo::lang.notfound.gone", + Confirm: "golem15.seo::lang.notfound.gone_selected_confirm", + Permissions: []string{PermissionManage}, + Run: func(ctx context.Context, in pact.AdminBulkActionInput) (pact.AdminBulkActionResult, error) { + svc := c.service() + if svc == nil { + return pact.AdminBulkActionResult{}, classes.ErrNotBooted + } + db, err := database(ctx, c.app) + if err != nil { + return pact.AdminBulkActionResult{}, err + } + n := 0 + for _, record := range in.Records { + entry, ok := record.(*models.NotFound) + if !ok || entry == nil { + continue + } + if _, err := svc.ResolveNotFound(ctx, db, *entry, ""); err != nil { + return pact.AdminBulkActionResult{}, c.refusal(ctx, svc, err) + } + n++ + } + return pact.AdminBulkActionResult{Message: "golem15.seo::lang.notfound.gone_selected_done", Affected: n}, nil + }, + }} +} + +// ListRowStates marks resolved entries disabled, labelled +// notfound.state_resolved (UI-SPEC C9). +func (notFoundAdminController) ListRowStates(ctx context.Context, db *gorm.DB, records []any) ([][]pact.RowState, error) { + set, err := classes.LoadRedirectSet(db.WithContext(ctx)) + if err != nil { + return nil, err + } + out := make([][]pact.RowState, len(records)) + for i, record := range records { + entry, ok := record.(*models.NotFound) + if !ok || entry == nil { + continue + } + rule, err := classes.NotFoundCover(set, entry.URL) + if err != nil { + return nil, err + } + if rule != nil { + out[i] = []pact.RowState{pact.RowStateDisabled} + } + } + return out, nil +} + +// HintView is the preview's header partial: the resolved hint, empty +// while the entry is open. +type HintView struct { + Text string +} + +// PartialData serves the preview's hint (notfound.resolved_hint). +func (c notFoundAdminController) PartialData(ctx context.Context, name string, record any) (any, error) { + entry, ok := record.(*models.NotFound) + if name != "hint" || !ok || entry == nil { + return HintView{}, nil + } + rule, err := c.cover(ctx, entry) + if err != nil || rule == nil { + return HintView{}, err + } + label := rule.Source + if rule.Target != "" { + label += " → " + rule.Target + } else { + label += " → 410" + } + return HintView{Text: Phrase(ctx, translator(c.app), "golem15.seo::lang.notfound.resolved_hint", map[string]string{"rule": label})}, nil +} + +// FilterOptions serves the status filter's open and resolved choices. +func (notFoundAdminController) FilterOptions(scope string) []pact.Option { + if scope != models.FilterNotFoundStatus { + return nil + } + return []pact.Option{ + {Value: models.NotFoundOpen, Label: "golem15.seo::lang.notfound.filter_status_open"}, + {Value: models.NotFoundResolved, Label: "golem15.seo::lang.notfound.filter_status_resolved"}, + } +} + +var ( + _ pact.AdminController = notFoundAdminController{} + _ pact.AdminPermissioned = notFoundAdminController{} + _ pact.AdminRecordSource = notFoundAdminController{} + _ pact.FormBeforeCreate = notFoundAdminController{} + _ pact.FormBeforeUpdate = notFoundAdminController{} + _ pact.HasAdminRecordActions = notFoundAdminController{} + _ pact.HasAdminBulkActions = notFoundAdminController{} + _ pact.ListRowStates = notFoundAdminController{} + _ pact.AdminPartialData = notFoundAdminController{} + _ pact.FilterOptions = notFoundAdminController{} +) diff --git a/controllers/notfound/_hint.htm b/controllers/notfound/_hint.htm new file mode 100644 index 0000000..9441dd6 --- /dev/null +++ b/controllers/notfound/_hint.htm @@ -0,0 +1,5 @@ +{{- with .Data.Text -}} +
+

{{ . }}

+
+{{- end -}} diff --git a/controllers/notfound/config_filter.yaml b/controllers/notfound/config_filter.yaml new file mode 100644 index 0000000..1462c10 --- /dev/null +++ b/controllers/notfound/config_filter.yaml @@ -0,0 +1,11 @@ +# 404 log filters (UI-SPEC C9): open or resolved, and the last hit. +scopes: + status: + label: golem15.seo::lang.notfound.filter_status + modelClass: Golem15\SEO\Models\NotFound + nameFrom: url + scope: FilterNotFoundStatus + last_hit: + label: golem15.seo::lang.notfound.last_hit + type: daterange + column: last_hit diff --git a/controllers/notfound/config_form.yaml b/controllers/notfound/config_form.yaml new file mode 100644 index 0000000..c407ac9 --- /dev/null +++ b/controllers/notfound/config_form.yaml @@ -0,0 +1,14 @@ +# 404 log preview (UI-SPEC C9): read-only, the resolved hint above the card +# and the two actions that apply while the entry is open. +name: golem15.seo::lang.notfound.name +form: ~/plugins/golem15/seo/models/notfound/fields.yaml +modelClass: Golem15\SEO\Models\NotFound +defaultRedirect: golem15/seo/notfound + +preview: + headerPartial: hint + +recordActions: [redirect, gone] + +messages: + preview: golem15.seo::lang.notfound.preview diff --git a/controllers/notfound/config_list.yaml b/controllers/notfound/config_list.yaml new file mode 100644 index 0000000..a88c0b6 --- /dev/null +++ b/controllers/notfound/config_list.yaml @@ -0,0 +1,25 @@ +# 404 log (UI-SPEC C9): most hits first, resolved entries badged, no create. +list: ~/plugins/golem15/seo/models/notfound/columns.yaml +modelClass: Golem15\SEO\Models\NotFound +title: golem15.seo::lang.notfound.list_title +recordUrl: golem15/seo/notfound/preview/:id +noRecordsMessage: golem15.seo::lang.notfound.list_empty +recordsPerPage: 50 +showCheckboxes: true +showSetup: true +filter: config_filter.yaml +bulkActions: [gone] + +defaultSort: + column: hits + direction: desc + +toolbar: + buttons: [delete] + search: + prompt: backend::lang.list.search_prompt + +messages: + empty: golem15.seo::lang.notfound.list_empty + deleteConfirm: golem15.seo::lang.notfound.delete_selected_confirm + rowStateDisabled: golem15.seo::lang.notfound.state_resolved diff --git a/lang/en/lang.yaml b/lang/en/lang.yaml index 4a1c6c2..a22c6ef 100644 --- a/lang/en/lang.yaml +++ b/lang/en/lang.yaml @@ -2,6 +2,7 @@ plugin: name: SEO description: Search engine metadata for every content type, URL rules and global SEO settings. nav: + notfound: 404 log redirects: Redirects seo: SEO history: History @@ -237,3 +238,33 @@ redirects: delete_selected_confirm: one: "Delete :count redirect? Its source address will return 404 again. You can undo it in History." other: "Delete :count redirects? Their source addresses will return 404 again. You can undo it in History." +notfound: + list_title: "404 log" + name: "404 address" + url: "Address" + hits: "Hits" + last_hit: "Last hit" + first_seen: "First seen" + referer: "Last referer" + state_resolved: "Resolved" + filter_status: "Status" + filter_status_open: "Open" + filter_status_resolved: "Resolved" + preview: "Address that returned 404" + resolved_hint: "Resolved by “:rule”. This address no longer returns 404." + redirect: "Create redirect…" + redirect_prompt: "Redirect to (301)" + redirect_placeholder: "/new-address" + redirect_comment: "A path or a full URL. To use 302, edit the redirect afterwards." + redirect_confirm: "Create a 301 redirect from this address?" + redirected: "Redirect “:url” → “:target” created." + gone: "Mark as gone (410)" + gone_confirm: "Answer this address with 410 Gone? Search engines drop it faster than a 404. You can undo it in History." + gone_selected_confirm: "Answer the selected addresses with 410 Gone? You can undo it in History." + gone_done: "410 Gone created for “:url”." + gone_selected_done: "410 Gone created for the selected addresses." + has_rule: "A redirect for “:url” already exists. Edit it in Redirects." + list_empty: "No 404 errors recorded. Addresses that return 404 appear here with their hit count." + delete_selected_confirm: + one: "Remove :count address from the log? It is added again on its next 404." + other: "Remove :count addresses from the log? They are added again on their next 404." diff --git a/lang/pl/lang.yaml b/lang/pl/lang.yaml index 5bf72a7..b695886 100644 --- a/lang/pl/lang.yaml +++ b/lang/pl/lang.yaml @@ -2,6 +2,7 @@ plugin: name: SEO description: Metadane dla wyszukiwarek we wszystkich typach treści, reguły adresów i globalne ustawienia SEO. nav: + notfound: Dziennik 404 redirects: Przekierowania seo: SEO history: Historia zmian @@ -243,3 +244,35 @@ redirects: few: "Usunąć :count przekierowania? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian." many: "Usunąć :count przekierowań? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian." other: "Usunąć :count przekierowania? Ich adresy źródłowe znowu zwrócą 404. Możesz to cofnąć w Historii zmian." +notfound: + list_title: "Dziennik 404" + name: "Adres 404" + url: "Adres" + hits: "Wejścia" + last_hit: "Ostatnie wejście" + first_seen: "Pierwsze wejście" + referer: "Ostatnie źródło" + state_resolved: "Rozwiązany" + filter_status: "Stan" + filter_status_open: "Otwarte" + filter_status_resolved: "Rozwiązane" + preview: "Adres, który zwrócił 404" + resolved_hint: "Rozwiązane przez „:rule”. Ten adres nie zwraca już 404." + redirect: "Utwórz przekierowanie…" + redirect_prompt: "Przekieruj na (301)" + redirect_placeholder: "/nowy-adres" + redirect_comment: "Ścieżka lub pełny adres. Aby użyć 302, edytuj potem przekierowanie." + redirect_confirm: "Utworzyć przekierowanie 301 z tego adresu?" + redirected: "Utworzono przekierowanie „:url” → „:target”." + gone: "Oznacz jako usunięty (410)" + gone_confirm: "Odpowiadać na ten adres kodem 410 Gone? Wyszukiwarki usuną go szybciej niż przy 404. Możesz to cofnąć w Historii zmian." + gone_selected_confirm: "Odpowiadać kodem 410 Gone na zaznaczone adresy? Możesz to cofnąć w Historii zmian." + gone_done: "Utworzono 410 Gone dla „:url”." + gone_selected_done: "Utworzono 410 Gone dla zaznaczonych adresów." + has_rule: "Przekierowanie dla „:url” już istnieje. Edytuj je w Przekierowaniach." + list_empty: "Nie zarejestrowano błędów 404. Adresy zwracające 404 pojawią się tutaj z liczbą wejść." + delete_selected_confirm: + one: "Usunąć :count adres z dziennika? Wróci przy następnym błędzie 404." + few: "Usunąć :count adresy z dziennika? Wrócą przy następnym błędzie 404." + many: "Usunąć :count adresów z dziennika? Wrócą przy następnym błędzie 404." + other: "Usunąć :count adresu z dziennika? Wrócą przy następnym błędzie 404." diff --git a/models/notfound.go b/models/notfound.go new file mode 100644 index 0000000..3e931af --- /dev/null +++ b/models/notfound.go @@ -0,0 +1,65 @@ +package models + +import ( + "time" + + "gorm.io/gorm" +) + +// NotFound is one address of the 404 log on golem15_seo_notfound (D-20): +// the path (with its query) that answered 404, how often, when first and +// last, and the referer of the last hit. The frontend reports each 404 +// through the authenticated report endpoint. +type NotFound struct { + ID uint `gorm:"column:id;primaryKey"` + URL string `gorm:"column:url"` + Hits int64 `gorm:"column:hits"` + FirstSeen *time.Time `gorm:"column:first_seen"` + LastHit *time.Time `gorm:"column:last_hit"` + Referer string `gorm:"column:referer"` +} + +func (NotFound) TableName() string { return "golem15_seo_notfound" } + +// MorphName names a 404 log entry in history. +func (NotFound) MorphName() string { return `Golem15\SEO\Models\NotFound` } + +// Fillable is empty: entries come from the report endpoint only, and the +// preview edits nothing. +func (NotFound) Fillable() []string { return []string{} } + +// Rules is empty: nothing is edited. +func (NotFound) Rules() map[string]string { return map[string]string{} } + +// The 404 log's status filter scope and its values. +const ( + FilterNotFoundStatus = "FilterNotFoundStatus" + NotFoundOpen = "open" + NotFoundResolved = "resolved" +) + +// FilterScopes lists the 404 log's filter scopes. +func (NotFound) FilterScopes() []string { return []string{FilterNotFoundStatus} } + +// coveredSQL is true for an entry an exact redirect or 410 answers now +// (the list filter; the Resolved badge also counts patterns and regex). +const coveredSQL = `EXISTS (SELECT 1 FROM golem15_seo_redirects AS r WHERE r.match = 'exact' AND (r.source = golem15_seo_notfound.url OR (r.ignore_case AND lower(r.source) = lower(golem15_seo_notfound.url))))` + +// FilterScope narrows the 404 log to open or resolved entries; an unknown +// value matches nothing. +func (NotFound) FilterScope(name string, db *gorm.DB, value any) *gorm.DB { + if db == nil || name != FilterNotFoundStatus { + return db + } + switch s, _ := value.(string); s { + case NotFoundOpen: + return db.Where("NOT " + coveredSQL) + case NotFoundResolved: + return db.Where(coveredSQL) + } + return db.Where("1 = 0") +} + +func init() { + Register(NotFound{}) +} diff --git a/models/notfound/columns.yaml b/models/notfound/columns.yaml new file mode 100644 index 0000000..5f4fe80 --- /dev/null +++ b/models/notfound/columns.yaml @@ -0,0 +1,23 @@ +# 404 log columns (UI-SPEC C9). +columns: + url: + label: golem15.seo::lang.notfound.url + type: text + searchable: true + sortable: true + hits: + label: golem15.seo::lang.notfound.hits + type: text + sortable: true + last_hit: + label: golem15.seo::lang.notfound.last_hit + type: datetime + sortable: true + referer: + label: golem15.seo::lang.notfound.referer + type: text + first_seen: + label: golem15.seo::lang.notfound.first_seen + type: datetime + sortable: true + invisible: true diff --git a/models/notfound/fields.yaml b/models/notfound/fields.yaml new file mode 100644 index 0000000..67afbce --- /dev/null +++ b/models/notfound/fields.yaml @@ -0,0 +1,29 @@ +# 404 log preview (UI-SPEC C9): every field is read-only. +fields: + url: + label: golem15.seo::lang.notfound.url + type: text + span: full + context: preview + hits: + label: golem15.seo::lang.notfound.hits + type: number + span: left + context: preview + last_hit: + label: golem15.seo::lang.notfound.last_hit + type: datepicker + mode: datetime + span: right + context: preview + first_seen: + label: golem15.seo::lang.notfound.first_seen + type: datepicker + mode: datetime + span: left + context: preview + referer: + label: golem15.seo::lang.notfound.referer + type: text + span: right + context: preview diff --git a/redirects_test.go b/redirects_test.go index 43c4e44..c0e6287 100644 --- a/redirects_test.go +++ b/redirects_test.go @@ -1,12 +1,16 @@ package seo_test import ( + "bytes" "encoding/json" "errors" "net/http" + "net/http/httptest" "sort" + "strconv" "strings" "testing" + "time" "git.golem15.com/golem15/sm-seo-plugin/classes" "git.golem15.com/golem15/sm-seo-plugin/models" @@ -416,3 +420,158 @@ func TestLoopRefused(t *testing.T) { t.Fatalf("redirect history rows = %d, want every admin save recorded", len(rows)) } } + +// report posts a 404 report with token ("" sends none). +func (e *testEnv) report(t *testing.T, token string, body any) *httptest.ResponseRecorder { + t.Helper() + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/_seo/api/v1/notfound", bytes.NewReader(raw)) + req.Header.Set("Content-Type", "application/json") + req.RemoteAddr = "198.51.100.7:4000" + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + return rec +} + +// notFoundRow is a 404 log row as the tests read it. +type notFoundRow struct { + ID uint + URL string + Hits int64 + FirstSeen time.Time + LastHit time.Time + Referer string +} + +func (e *testEnv) notFoundRow(t *testing.T, url string) notFoundRow { + t.Helper() + var row notFoundRow + if err := e.db.Raw(`SELECT id, url, hits, first_seen, last_hit, referer FROM golem15_seo_notfound WHERE url = ?`, url).Scan(&row).Error; err != nil || row.ID == 0 { + t.Fatalf("404 log row %s: %+v err=%v", url, row, err) + } + return row +} + +// TestNotFoundLog checks the 404 log (D-20, P-22, UI-SPEC C9): the +// authenticated report endpoint, hits and referers, the prompted redirect +// and the 410 record actions, the Resolved state, and the bulk 410 that +// refuses a selection containing a covered address. +func TestNotFoundLog(t *testing.T) { + off := bootEnv(t) + if rec := off.report(t, "anything", map[string]string{"url": "/x"}); rec.Code != http.StatusNotFound { + t.Fatalf("report without a configured token = %d, want 404", rec.Code) + } + + const secret = "notfound-test-token" + env := bootEnv(t, "SUMMER_GOLEM15__SEO__NOTFOUND_TOKEN="+secret) + if rec := env.report(t, "", map[string]string{"url": "/missing"}); rec.Code != http.StatusUnauthorized { + t.Fatalf("report without a token = %d, want 401", rec.Code) + } + if rec := env.report(t, "wrong", map[string]string{"url": "/missing"}); rec.Code != http.StatusUnauthorized { + t.Fatalf("report with a wrong token = %d, want 401", rec.Code) + } + if rec := env.report(t, secret, map[string]string{"url": "/" + strings.Repeat("a", 2100)}); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("report of a long URL = %d, want 422", rec.Code) + } + if rec := env.report(t, secret, map[string]string{"url": "/missing", "referer": "https://ref.example/a"}); rec.Code != http.StatusNoContent || rec.Header().Get("X-Robots-Tag") != "noindex" { + t.Fatalf("report = %d %v, want 204 with X-Robots-Tag", rec.Code, rec.Header()) + } + first := env.notFoundRow(t, "/missing") + time.Sleep(1100 * time.Millisecond) + env.report(t, secret, map[string]string{"url": "https://www.example.test/missing", "referer": "https://ref.example/b"}) + again := env.notFoundRow(t, "/missing") + if again.Hits != 2 || !again.FirstSeen.Equal(first.FirstSeen) || !again.LastHit.After(first.LastHit) || again.Referer != "https://ref.example/b" { + t.Fatalf("second report = %+v (first %+v), want hits 2, first seen kept, last hit and referer updated", again, first) + } + for _, u := range []string{"/old-a", "/old-b", "/old-c", "/old-c"} { + env.report(t, secret, map[string]string{"url": u}) + } + + token := env.admin(t, true) + api := adminAPI("/golem15/seo/notfound") + var list struct { + Data []struct { + ID json.Number `json:"id"` + URL string `json:"url"` + } `json:"data"` + Meta struct { + RowStates map[string][]string `json:"row_states"` + } `json:"meta"` + } + if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api, token, nil).Body.Bytes(), &list); err != nil { + t.Fatal(err) + } + if len(list.Data) != 4 || (list.Data[0].URL != "/missing" && list.Data[0].URL != "/old-c") { + t.Fatalf("404 list = %+v, want four entries, most hits first", list.Data) + } + + missing := strconv.FormatUint(uint64(again.ID), 10) + rec := env.expect(t, http.StatusOK, http.MethodPost, api+"/"+missing+"/actions/redirect", token, map[string]any{"payload": "/found"}) + if !strings.Contains(rec.Body.String(), "/missing") || !strings.Contains(rec.Body.String(), "/found") { + t.Fatalf("redirect action = %s, want the redirected message", rec.Body.String()) + } + rows := env.redirects(t) + if target := redirectTo(rows, "/missing"); target != "/found" { + t.Fatalf("redirect from the 404 log = %+v, want /missing -> /found", rows) + } + for _, r := range rows { + if r.Source == "/missing" && (r.Origin != "notfound" || r.Status != 301) { + t.Fatalf("404 redirect = %+v, want origin notfound and 301", r) + } + } + env.expectRedirect(t, "/missing", 301, siteURL+"/found") + show := env.expect(t, http.StatusOK, http.MethodGet, api+"/"+missing, token, nil).Body.String() + if strings.Contains(show, `"actions"`) { + t.Fatalf("a resolved entry still offers actions: %s", show) + } + if rec := env.call(t, http.MethodPost, api+"/"+missing+"/actions/gone", token, map[string]any{}); rec.Code != http.StatusConflict { + t.Fatalf("gone on a resolved entry = %d %s, want 409", rec.Code, rec.Body.String()) + } + hint := env.expect(t, http.StatusOK, http.MethodGet, adminAPI("/golem15/seo/notfound/partials/hint?id="+missing), token, nil).Body.String() + if !strings.Contains(hint, "Resolved by") || !strings.Contains(hint, "/missing → /found") { + t.Fatalf("hint = %s, want the resolved hint", hint) + } + + oldA := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-a").ID), 10) + oldB := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-b").ID), 10) + oldC := strconv.FormatUint(uint64(env.notFoundRow(t, "/old-c").ID), 10) + env.expect(t, http.StatusOK, http.MethodPost, api+"/"+oldA+"/actions/gone", token, map[string]any{}) + if got := env.resolve(t, "/old-a"); got["action"] != "gone" { + t.Fatalf("/old-a after gone = %v, want gone", got) + } + before := env.redirectCount(t) + rec = env.call(t, http.MethodPost, api+"/bulk/gone", token, map[string]any{"ids": []string{oldA, oldB, oldC}}) + if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "A redirect for “/old-a” already exists") { + t.Fatalf("bulk gone over a covered entry = %d %s, want 403 notfound.has_rule", rec.Code, rec.Body.String()) + } + if after := env.redirectCount(t); after != before { + t.Fatalf("redirects after a refused bulk = %d, want %d", after, before) + } + rec = env.expect(t, http.StatusOK, http.MethodPost, api+"/bulk/gone", token, map[string]any{"ids": []string{oldB, oldC}}) + if !strings.Contains(rec.Body.String(), `"affected":2`) { + t.Fatalf("bulk gone = %s, want 2 affected", rec.Body.String()) + } + if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api, token, nil).Body.Bytes(), &list); err != nil { + t.Fatal(err) + } + for _, id := range []string{missing, oldA, oldB, oldC} { + if states := list.Meta.RowStates[id]; len(states) != 1 || states[0] != "disabled" { + t.Fatalf("row states = %v, want %s resolved", list.Meta.RowStates, id) + } + } + var open []string + if err := json.Unmarshal(env.expect(t, http.StatusOK, http.MethodGet, api+"?filter[status]=open", token, nil).Body.Bytes(), &list); err == nil { + for _, d := range list.Data { + open = append(open, d.URL) + } + } + if len(open) != 0 { + t.Fatalf("open filter = %v, want none", open) + } +} diff --git a/routes.go b/routes.go index 5f00357..66ba741 100644 --- a/routes.go +++ b/routes.go @@ -9,34 +9,54 @@ import ( "git.golem15.com/golem15/summercms/modules/surf" ) -// BucketResolve is the throttle bucket of the public resolve endpoint. -const BucketResolve = "seo-resolve" +// The throttle buckets of the public endpoints. +const ( + // BucketResolve is the resolve endpoint's bucket. + BucketResolve = "seo-resolve" + // BucketNotFound is the 404 report endpoint's bucket. + BucketNotFound = "seo-notfound" +) -// Routes mounts the public /_seo/api/v1 endpoints. They carry no backend -// middleware: the frontend server calls them anonymously. +// Routes mounts the /_seo/api/v1 endpoints. They carry no backend +// middleware: the frontend server calls them anonymously (the 404 report +// with its own token). Every answer carries X-Robots-Tag: noindex. func (p *Plugin) Routes(r pact.Router) error { - r.Group("/_seo/api/v1", surf.Use("throttle:"+BucketResolve), func(g pact.Router) { - g.Get("/resolve", apicontrollers.Resolve(p.Service)) + r.Group("/_seo/api/v1", nil, func(g pact.Router) { + g.Get("/resolve", noindex(apicontrollers.Resolve(p.Service)), "throttle:"+BucketResolve) + g.Post("/notfound", noindex(apicontrollers.NotFound(p.Service)), "throttle:"+BucketNotFound) }) return nil } -// Buckets registers seo-resolve: 600 requests a minute per client IP behind -// the configured trusted proxies (the frontend server resolves every page -// it renders). +// noindex adds X-Robots-Tag: noindex to every answer of h. +func noindex(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Robots-Tag", "noindex") + h(w, r) + } +} + +// Buckets registers the per-client-IP buckets behind the configured +// trusted proxies: seo-resolve 600 a minute (the frontend server resolves +// every page it renders) and seo-notfound 600 a minute (one report per +// 404 page). func (p *Plugin) Buckets() map[string]surf.Bucket { trusted := surf.TrustedProxies(nil) if p.app != nil { trusted = surf.TrustedProxies(p.app.Config) } - return map[string]surf.Bucket{ - BucketResolve: { - Name: BucketResolve, - Max: 600, + bucket := func(name string, max int) surf.Bucket { + return surf.Bucket{ + Name: name, + Max: max, Decay: time.Minute, Key: func(r *http.Request) string { - return BucketResolve + "|" + surf.ClientIP(r, trusted) + return name + "|" + surf.ClientIP(r, trusted) }, - }, + } + } + return map[string]surf.Bucket{ + BucketResolve: bucket(BucketResolve, 600), + BucketNotFound: bucket(BucketNotFound, 600), } } diff --git a/updates/202610270007_create_golem15_seo_notfound.go b/updates/202610270007_create_golem15_seo_notfound.go new file mode 100644 index 0000000..be62bb0 --- /dev/null +++ b/updates/202610270007_create_golem15_seo_notfound.go @@ -0,0 +1,32 @@ +package updates + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// createNotFound creates golem15_seo_notfound: the 404 log (D-20), one row +// per address. +var createNotFound = []*gormigrate.Migration{ + { + ID: "202610270007_create_golem15_seo_notfound", + Migrate: func(tx *gorm.DB) error { + return tx.Exec(`CREATE TABLE golem15_seo_notfound ( + id BIGSERIAL PRIMARY KEY, + url TEXT NOT NULL, + hits BIGINT NOT NULL DEFAULT 1, + first_seen TIMESTAMP(0) NULL, + last_hit TIMESTAMP(0) NULL, + referer TEXT NOT NULL DEFAULT '', + CONSTRAINT golem15_seo_notfound_url UNIQUE (url) +)`).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec(`DROP TABLE IF EXISTS golem15_seo_notfound`).Error + }, + }, +} + +func init() { + Register(createNotFound...) +}