feat: 404 log and report endpoint
- POST /_seo/api/v1/notfound with the golem15.seo.notfound_token bearer (constant-time, off when unset), URL and referer caps, a row cap with an hourly warning, throttle bucket seo-notfound - golem15.seo.notfound list by hits with Resolved rows, a preview with the resolved hint, a prompted 301 and a 410 record action, a bulk 410 that refuses a selection with a covered address - every /_seo/api/v1 answer carries X-Robots-Tag: noindex
This commit is contained in:
181
classes/notfound.go
Normal file
181
classes/notfound.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// Config keys of the 404 log (P-22).
|
||||
const (
|
||||
// NotFoundTokenConfigKey is the shared secret the frontend server
|
||||
// sends as "Authorization: Bearer <token>" with each 404 report. Empty
|
||||
// turns the report endpoint off (it answers 404).
|
||||
NotFoundTokenConfigKey = "golem15.seo.notfound_token"
|
||||
// NotFoundMaxRowsConfigKey caps the 404 log; new addresses beyond it
|
||||
// are dropped (default DefaultNotFoundMaxRows).
|
||||
NotFoundMaxRowsConfigKey = "golem15.seo.notfound_max_rows"
|
||||
)
|
||||
|
||||
// The 404 report limits (P-22).
|
||||
const (
|
||||
DefaultNotFoundMaxRows = 10000
|
||||
MaxNotFoundURLBytes = 2048
|
||||
MaxRefererBytes = 1024
|
||||
)
|
||||
|
||||
// ErrNotFoundURL refuses a reported address that is not a path or an
|
||||
// http(s) URL, or is longer than MaxNotFoundURLBytes.
|
||||
var ErrNotFoundURL = errors.New("golem15.seo: url must be a path or an absolute http(s) URL of at most 2048 bytes")
|
||||
|
||||
// NotFoundToken is the configured report token, "" when unset.
|
||||
func (s *Service) NotFoundToken() string {
|
||||
cfg := s.Config()
|
||||
if cfg == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(cfg.String(NotFoundTokenConfigKey))
|
||||
}
|
||||
|
||||
// NotFoundMaxRows is the configured row cap of the 404 log.
|
||||
func (s *Service) NotFoundMaxRows() int64 {
|
||||
if cfg := s.Config(); cfg != nil {
|
||||
if n := cfg.Int(NotFoundMaxRowsConfigKey); n > 0 {
|
||||
return int64(n)
|
||||
}
|
||||
}
|
||||
return DefaultNotFoundMaxRows
|
||||
}
|
||||
|
||||
// NotFoundPath turns a reported address into the logged one: the path
|
||||
// with its query; an absolute URL loses its scheme and host.
|
||||
func NotFoundPath(raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" || len(raw) > MaxNotFoundURLBytes {
|
||||
return "", ErrNotFoundURL
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", ErrNotFoundURL
|
||||
}
|
||||
if u.IsAbs() {
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", ErrNotFoundURL
|
||||
}
|
||||
} else if !strings.HasPrefix(raw, "/") || strings.HasPrefix(raw, "//") {
|
||||
return "", ErrNotFoundURL
|
||||
}
|
||||
p := u.Path
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
return joinQuery(p, u.RawQuery), nil
|
||||
}
|
||||
|
||||
// capWarned keeps when the row-cap warning was last logged.
|
||||
var capWarned struct {
|
||||
sync.Mutex
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// ReportNotFound records one 404 of rawURL (D-20, P-22): a known address
|
||||
// gets one more hit, its last hit time and, when given, the referer; a new
|
||||
// one is added unless the log holds NotFoundMaxRows rows, which is logged
|
||||
// at most once an hour.
|
||||
func (s *Service) ReportNotFound(ctx context.Context, rawURL, referer string) error {
|
||||
path, err := NotFoundPath(rawURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
referer = strings.TrimSpace(referer)
|
||||
if len(referer) > MaxRefererBytes {
|
||||
referer = referer[:MaxRefererBytes]
|
||||
for len(referer) > 0 && !utf8ValidTail(referer) {
|
||||
referer = referer[:len(referer)-1]
|
||||
}
|
||||
}
|
||||
db, err := s.DB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
updates := map[string]any{"hits": gorm.Expr("hits + 1"), "last_hit": now}
|
||||
if referer != "" {
|
||||
updates["referer"] = referer
|
||||
}
|
||||
res := db.Model(&models.NotFound{}).Where("url = ?", path).Updates(updates)
|
||||
if res.Error != nil || res.RowsAffected > 0 {
|
||||
return res.Error
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&models.NotFound{}).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count >= s.NotFoundMaxRows() {
|
||||
capWarned.Lock()
|
||||
if time.Since(capWarned.at) >= time.Hour {
|
||||
capWarned.at = time.Now()
|
||||
log.Warn("golem15.seo: the 404 log is full, new addresses are dropped", "max_rows", s.NotFoundMaxRows(), "url", path)
|
||||
}
|
||||
capWarned.Unlock()
|
||||
return nil
|
||||
}
|
||||
row := models.NotFound{URL: path, Hits: 1, FirstSeen: &now, LastHit: &now, Referer: referer}
|
||||
return db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "url"}},
|
||||
DoUpdates: clause.Assignments(map[string]any{"hits": gorm.Expr("golem15_seo_notfound.hits + 1"), "last_hit": now}),
|
||||
}).Create(&row).Error
|
||||
}
|
||||
|
||||
// utf8ValidTail reports whether s does not end inside a UTF-8 sequence.
|
||||
func utf8ValidTail(s string) bool {
|
||||
return strings.ToValidUTF8(s, "<22>") == s
|
||||
}
|
||||
|
||||
// NotFoundCover returns the rule that answers url now (a redirect or a
|
||||
// 410), nil when the address still answers 404: the Resolved state.
|
||||
func NotFoundCover(set *RedirectSet, url string) (*models.Redirect, error) {
|
||||
return CoveringRedirect(set, url)
|
||||
}
|
||||
|
||||
// NotFoundHasRuleError refuses a 404 action on an address a rule already
|
||||
// answers.
|
||||
type NotFoundHasRuleError struct{ URL string }
|
||||
|
||||
func (e *NotFoundHasRuleError) Error() string {
|
||||
return "golem15.seo: a redirect for " + e.URL + " already exists"
|
||||
}
|
||||
|
||||
// ResolveNotFound creates the rule of a 404 log action inside tx (D-20): a
|
||||
// 301 to target when target is set, else a 410, with origin notfound. It
|
||||
// refuses an address a rule already answers (*NotFoundHasRuleError), and
|
||||
// the redirect's own refusals (*ProblemsError, *LoopError).
|
||||
func (s *Service) ResolveNotFound(ctx context.Context, tx *gorm.DB, entry models.NotFound, target string) (models.Redirect, error) {
|
||||
set, err := LoadRedirectSet(tx)
|
||||
if err != nil {
|
||||
return models.Redirect{}, err
|
||||
}
|
||||
if rule, err := NotFoundCover(set, entry.URL); err != nil {
|
||||
return models.Redirect{}, err
|
||||
} else if rule != nil {
|
||||
return models.Redirect{}, &NotFoundHasRuleError{URL: entry.URL}
|
||||
}
|
||||
r := models.NewRedirect()
|
||||
r.Source, r.Origin = entry.URL, models.OriginNotFound
|
||||
if strings.TrimSpace(target) == "" {
|
||||
r.Status = models.Status410
|
||||
} else {
|
||||
r.Target = strings.TrimSpace(target)
|
||||
}
|
||||
if _, err := s.SaveRedirect(ctx, tx, &r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
Reference in New Issue
Block a user