feat: redirect engine with loop refusal and chain notices
- rule set: exact sources by key, patterns by literal prefix, regex by id (stdlib RE2, compiled once per source); ignore case, trailing slash, query ignore/pass, a source with ? matches its query exactly - targets: captures escaped, a relative target never leaves the site, an absolute one keeps its host - SaveRedirect validates, simulates the rule in place (exact, pattern and regex probes, 10 hops) and refuses a loop; chains answer the notice - resolve follows the chain to its final target, 410 answers gone, a runtime cycle answers not_found and is logged once
This commit is contained in:
337
classes/redirect_graph.go
Normal file
337
classes/redirect_graph.go
Normal file
@@ -0,0 +1,337 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp/syntax"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// candidateID stands in for the id of a redirect that is not stored yet.
|
||||
const candidateID = ^uint(0)
|
||||
|
||||
// RedirectCheck is what the save-time simulation found for one redirect
|
||||
// (D-19, P-19).
|
||||
type RedirectCheck struct {
|
||||
// Loop is true when following redirects from the rule's source comes
|
||||
// back round (or never ends); Path lists the URLs of the loop.
|
||||
Loop bool
|
||||
// Chain is true when the rule is one hop of a longer chain; Path lists
|
||||
// it from its first source to Final, Hops is its length and Source the
|
||||
// rule that should point straight to Final.
|
||||
Chain bool
|
||||
Path []string
|
||||
Hops int
|
||||
Source string
|
||||
Final string
|
||||
}
|
||||
|
||||
// PathText joins the check's path for the notices: "A → B → C".
|
||||
func (c RedirectCheck) PathText() string { return strings.Join(c.Path, " → ") }
|
||||
|
||||
// LoopError refuses a redirect that would create a loop (D-19).
|
||||
type LoopError struct{ Check RedirectCheck }
|
||||
|
||||
func (e *LoopError) Error() string {
|
||||
return "golem15.seo: redirect would create a loop: " + e.Check.PathText()
|
||||
}
|
||||
|
||||
// ProblemsError refuses a redirect with field problems.
|
||||
type ProblemsError struct{ Problems map[string][]models.Problem }
|
||||
|
||||
func (e *ProblemsError) Error() string {
|
||||
keys := make([]string, 0, len(e.Problems))
|
||||
for field, list := range e.Problems {
|
||||
for _, p := range list {
|
||||
keys = append(keys, field+": "+p.Key)
|
||||
}
|
||||
}
|
||||
return "golem15.seo: redirect refused: " + strings.Join(keys, "; ")
|
||||
}
|
||||
|
||||
// redirectProbe returns a URL the rule's source matches: the source of an
|
||||
// exact rule, a pattern with every * replaced by a probe token, or a string
|
||||
// built from a regex's literals when one can be built (RESEARCH Pitfall
|
||||
// 10). ok is false when no probe could be built.
|
||||
func redirectProbe(r models.Redirect) (path, rawQuery string, ok bool) {
|
||||
var probe string
|
||||
switch r.Match {
|
||||
case models.RedirectExact:
|
||||
probe = r.Source
|
||||
case models.RedirectPattern:
|
||||
probe = strings.ReplaceAll(r.Source, "*", probeToken)
|
||||
case models.RedirectRegex:
|
||||
re, err := syntax.Parse(r.Source, syntax.Perl)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
var b strings.Builder
|
||||
if !writeProbe(&b, re.Simplify(), 0) {
|
||||
return "", "", false
|
||||
}
|
||||
probe = b.String()
|
||||
if expr := sourceExpr(r); expr == nil || !expr.MatchString(probe) || strings.Contains(probe, "?") {
|
||||
return "", "", false
|
||||
}
|
||||
return probe, "", strings.HasPrefix(probe, "/")
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
p, q, _ := strings.Cut(probe, "?")
|
||||
return p, q, strings.HasPrefix(p, "/")
|
||||
}
|
||||
|
||||
// writeProbe writes a shortest string the regex node matches: literals,
|
||||
// the first alternative, the minimum of every repeat, the first rune of a
|
||||
// class. ok is false for a node it cannot build.
|
||||
func writeProbe(b *strings.Builder, re *syntax.Regexp, depth int) bool {
|
||||
if depth > 100 || b.Len() > 2048 {
|
||||
return false
|
||||
}
|
||||
switch re.Op {
|
||||
case syntax.OpEmptyMatch, syntax.OpBeginLine, syntax.OpEndLine, syntax.OpBeginText, syntax.OpEndText,
|
||||
syntax.OpWordBoundary, syntax.OpNoWordBoundary:
|
||||
return true
|
||||
case syntax.OpLiteral:
|
||||
for _, r := range re.Rune {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return true
|
||||
case syntax.OpCharClass:
|
||||
if len(re.Rune) < 2 {
|
||||
return false
|
||||
}
|
||||
// Prefer a letter or digit, so a probe stays a plain path segment.
|
||||
pick := re.Rune[0]
|
||||
for _, c := range []rune{'a', '0', 'A'} {
|
||||
if classHas(re.Rune, c) {
|
||||
pick = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if !utf8.ValidRune(pick) {
|
||||
return false
|
||||
}
|
||||
b.WriteRune(pick)
|
||||
return true
|
||||
case syntax.OpAnyChar, syntax.OpAnyCharNotNL:
|
||||
b.WriteString("a")
|
||||
return true
|
||||
case syntax.OpCapture:
|
||||
return writeProbe(b, re.Sub[0], depth+1)
|
||||
case syntax.OpConcat:
|
||||
for _, sub := range re.Sub {
|
||||
if !writeProbe(b, sub, depth+1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
case syntax.OpAlternate:
|
||||
return writeProbe(b, re.Sub[0], depth+1)
|
||||
case syntax.OpStar, syntax.OpQuest:
|
||||
return true
|
||||
case syntax.OpPlus:
|
||||
return writeProbe(b, re.Sub[0], depth+1)
|
||||
case syntax.OpRepeat:
|
||||
for i := 0; i < re.Min; i++ {
|
||||
if !writeProbe(b, re.Sub[0], depth+1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// classHas reports whether a character class's ranges hold c.
|
||||
func classHas(ranges []rune, c rune) bool {
|
||||
for i := 0; i+1 < len(ranges); i += 2 {
|
||||
if ranges[i] <= c && c <= ranges[i+1] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CheckRedirect simulates r against the stored rules in db with r in
|
||||
// place (P-19): a loop when following redirects from a URL r's source
|
||||
// matches comes back round or does not end within MaxRedirectHops, a
|
||||
// chain when it takes more than one hop or when another rule's target
|
||||
// leads into r. A 410 rule ends every chain and is never a loop.
|
||||
func (s *Service) CheckRedirect(ctx context.Context, db *gorm.DB, r models.Redirect) (RedirectCheck, error) {
|
||||
var check RedirectCheck
|
||||
if r.Status == models.Status410 {
|
||||
return check, nil
|
||||
}
|
||||
rules, err := LoadRedirects(db.WithContext(ctx))
|
||||
if err != nil {
|
||||
return check, err
|
||||
}
|
||||
if r.ID == 0 {
|
||||
r.ID = candidateID
|
||||
}
|
||||
placed := false
|
||||
for i := range rules {
|
||||
if rules[i].ID == r.ID {
|
||||
rules[i], placed = r, true
|
||||
}
|
||||
}
|
||||
if !placed {
|
||||
rules = append(rules, r)
|
||||
}
|
||||
set := NewRedirectSet(rules)
|
||||
site := s.SiteURL()
|
||||
path, query, ok := redirectProbe(r)
|
||||
if ok {
|
||||
ch, err := set.Follow(site, path, query, MaxRedirectHops)
|
||||
if err != nil {
|
||||
return check, err
|
||||
}
|
||||
if ch.Loop {
|
||||
check.Loop, check.Path = true, displayPath(r, ch)
|
||||
return check, nil
|
||||
}
|
||||
if len(ch.Hops) > 1 && ch.Hops[0].Rule.ID == r.ID {
|
||||
check.Chain, check.Path, check.Hops = true, displayPath(r, ch), len(ch.Hops)
|
||||
check.Source, check.Final = r.Source, displayURL(site, ch.Final)
|
||||
return check, nil
|
||||
}
|
||||
}
|
||||
// A rule whose target leads into r makes r the second hop of a chain.
|
||||
for _, other := range rules {
|
||||
if other.ID == r.ID || other.Status == models.Status410 {
|
||||
continue
|
||||
}
|
||||
next, nextQuery, internal := internalTarget(site, other.Target)
|
||||
if !internal || strings.ContainsAny(other.Target, "*$") {
|
||||
continue
|
||||
}
|
||||
hit, matched, err := set.Match(next, nextQuery)
|
||||
if err != nil {
|
||||
return check, err
|
||||
}
|
||||
if !matched || hit.Rule.ID != r.ID {
|
||||
continue
|
||||
}
|
||||
ch, err := set.Follow(site, next, nextQuery, MaxRedirectHops)
|
||||
if err != nil || ch.Loop {
|
||||
return check, err
|
||||
}
|
||||
check.Chain, check.Hops = true, len(ch.Hops)+1
|
||||
check.Path = append([]string{other.Source}, displayPath(r, ch)...)
|
||||
check.Source, check.Final = other.Source, displayURL(site, ch.Final)
|
||||
return check, nil
|
||||
}
|
||||
return check, nil
|
||||
}
|
||||
|
||||
// displayPath lists a chain's URLs for a notice, the source of r first.
|
||||
func displayPath(r models.Redirect, ch Chain) []string {
|
||||
out := make([]string, 0, len(ch.Visited))
|
||||
for i, v := range ch.Visited {
|
||||
if i == 0 && r.Match != models.RedirectExact && len(ch.Hops) > 0 && ch.Hops[0].Rule.ID == r.ID {
|
||||
v = r.Source
|
||||
}
|
||||
out = append(out, strings.ReplaceAll(v, probeToken, "*"))
|
||||
}
|
||||
if len(ch.Hops) > 0 && ch.Final != "" {
|
||||
if last := out[len(out)-1]; last != ch.Final {
|
||||
if _, _, internal := internalTarget("", ch.Final); !internal {
|
||||
out = append(out, ch.Final)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// displayURL shows an absolute URL on the site as its path.
|
||||
func displayURL(site, u string) string {
|
||||
if site != "" && strings.HasPrefix(u, site+"/") {
|
||||
return strings.TrimPrefix(u, site)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// SaveRedirect validates and stores r inside tx, the caller's transaction,
|
||||
// and records it in history (P-18). It refuses field problems with a
|
||||
// *ProblemsError and a loop with a *LoopError; the caller rolls back. The
|
||||
// check answers a chain, which is saved (D-19).
|
||||
func (s *Service) SaveRedirect(ctx context.Context, tx *gorm.DB, r *models.Redirect) (RedirectCheck, error) {
|
||||
if r == nil {
|
||||
return RedirectCheck{}, errors.New("golem15.seo: no redirect")
|
||||
}
|
||||
r.Normalize()
|
||||
if problems := r.Problems(tx); len(problems) > 0 {
|
||||
return RedirectCheck{}, &ProblemsError{Problems: problems}
|
||||
}
|
||||
check, err := s.CheckRedirect(ctx, tx, *r)
|
||||
if err != nil {
|
||||
return check, err
|
||||
}
|
||||
if check.Loop {
|
||||
return check, &LoopError{Check: check}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
var before map[string]any
|
||||
if r.ID == 0 {
|
||||
r.CreatedAt, r.UpdatedAt = &now, &now
|
||||
if err := tx.Create(r).Error; err != nil {
|
||||
return check, err
|
||||
}
|
||||
} else {
|
||||
var old models.Redirect
|
||||
if err := tx.Where("id = ?", r.ID).First(&old).Error; err != nil {
|
||||
return check, err
|
||||
}
|
||||
before = RedirectValues(old)
|
||||
r.UpdatedAt = &now
|
||||
if err := tx.Save(r).Error; err != nil {
|
||||
return check, err
|
||||
}
|
||||
}
|
||||
return check, s.recordRedirect(ctx, tx, *r, before, RedirectValues(*r))
|
||||
}
|
||||
|
||||
// DeleteRedirect deletes r inside tx and records it in history.
|
||||
func (s *Service) DeleteRedirect(ctx context.Context, tx *gorm.DB, r models.Redirect) error {
|
||||
return s.deleteRedirect(ctx, tx, r)
|
||||
}
|
||||
|
||||
// LoopMessage is the redirects.loop refusal in the request locale.
|
||||
func (s *Service) LoopMessage(ctx context.Context, check RedirectCheck) string {
|
||||
return s.PhraseCtx(ctx, "golem15.seo::lang.redirects.loop", map[string]string{"path": check.PathText()})
|
||||
}
|
||||
|
||||
// ChainNotice is the redirects.chain_notice of a chain check.
|
||||
func ChainNotice(check RedirectCheck) []pact.FormFieldNotice {
|
||||
if !check.Chain {
|
||||
return nil
|
||||
}
|
||||
return []pact.FormFieldNotice{{
|
||||
Message: "golem15.seo::lang.redirects.chain_notice",
|
||||
Params: map[string]string{
|
||||
"path": check.PathText(),
|
||||
"count": strconv.Itoa(check.Hops),
|
||||
"source": check.Source,
|
||||
"final": check.Final,
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// CoveringRedirect returns the rule that answers url now (a redirect or a
|
||||
// 410), nil when none does: the 404 log's Resolved state.
|
||||
func CoveringRedirect(set *RedirectSet, url string) (*models.Redirect, error) {
|
||||
path, query, _ := strings.Cut(url, "?")
|
||||
hit, ok, err := set.Match(path, query)
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
return &hit.Rule, nil
|
||||
}
|
||||
@@ -1,29 +1,539 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// log is the classes' logger (component=seo).
|
||||
var log = diary.For("seo")
|
||||
|
||||
// The resolve actions of the redirect stage.
|
||||
const (
|
||||
ActionRedirect = "redirect"
|
||||
ActionGone = "gone"
|
||||
)
|
||||
|
||||
// exactRedirect returns the exact redirect whose source is path, nil when
|
||||
// there is none.
|
||||
func exactRedirect(db *gorm.DB, path string) (*models.Redirect, error) {
|
||||
// MaxRedirectHops caps how many redirects the resolver follows for one
|
||||
// URL, and how far the save-time simulation looks (P-19, P-22).
|
||||
const MaxRedirectHops = 10
|
||||
|
||||
// MaxRegexSource caps a regex source (T-32-17).
|
||||
const MaxRegexSource = 512
|
||||
|
||||
// probeToken stands in for * when a pattern source is simulated.
|
||||
const probeToken = "seoprobe"
|
||||
|
||||
// compiledRedirect is a pattern or regex rule with its expression.
|
||||
type compiledRedirect struct {
|
||||
rule models.Redirect
|
||||
re *regexp.Regexp
|
||||
prefix int
|
||||
}
|
||||
|
||||
// regexCache keeps the compiled expressions of pattern and regex sources,
|
||||
// so each distinct source is compiled once (stdlib RE2: linear time on any
|
||||
// input).
|
||||
var regexCache sync.Map
|
||||
|
||||
// sourceExpr returns the expression of a pattern or regex source, nil
|
||||
// when it does not compile.
|
||||
func sourceExpr(r models.Redirect) *regexp.Regexp {
|
||||
key := r.Match + "\x00" + strconv.FormatBool(r.IgnoreCase) + "\x00" + strconv.FormatBool(r.TrailingSlash) + "\x00" + r.Source
|
||||
if re, ok := regexCache.Load(key); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
}
|
||||
var expr string
|
||||
switch r.Match {
|
||||
case models.RedirectPattern:
|
||||
expr = WildcardExpr(r.Source, r.TrailingSlash)
|
||||
case models.RedirectRegex:
|
||||
expr = r.Source
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
if r.IgnoreCase {
|
||||
expr = "(?i)" + expr
|
||||
}
|
||||
re, err := regexp.Compile(expr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
regexCache.Store(key, re)
|
||||
return re
|
||||
}
|
||||
|
||||
// WildcardExpr turns a pattern source into an anchored expression: every *
|
||||
// matches any text (a capture), everything else is literal. With
|
||||
// trailingSlash a source that does not end in * or / also matches with a
|
||||
// trailing slash, and one that ends in / also without it.
|
||||
func WildcardExpr(source string, trailingSlash bool) string {
|
||||
src := source
|
||||
optional := false
|
||||
if trailingSlash && !strings.Contains(src, "?") && !strings.HasSuffix(src, "*") && src != "/" {
|
||||
src = strings.TrimSuffix(src, "/")
|
||||
optional = true
|
||||
}
|
||||
expr := "^" + strings.ReplaceAll(regexp.QuoteMeta(src), `\*`, `(.*)`)
|
||||
if optional {
|
||||
expr += "/?"
|
||||
}
|
||||
return expr + "$"
|
||||
}
|
||||
|
||||
// literalPrefix is the length of a pattern source before its first *.
|
||||
func literalPrefix(source string) int {
|
||||
if i := strings.Index(source, "*"); i >= 0 {
|
||||
return i
|
||||
}
|
||||
return len(source)
|
||||
}
|
||||
|
||||
// trimSlash drops one trailing slash, except from the root.
|
||||
func trimSlash(p string) string {
|
||||
if len(p) > 1 && strings.HasSuffix(p, "/") {
|
||||
return strings.TrimSuffix(p, "/")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// toggleSlash adds a trailing slash, or drops it; the root stays.
|
||||
func toggleSlash(p string) string {
|
||||
if p == "/" {
|
||||
return p
|
||||
}
|
||||
if strings.HasSuffix(p, "/") {
|
||||
return strings.TrimSuffix(p, "/")
|
||||
}
|
||||
return p + "/"
|
||||
}
|
||||
|
||||
// exactKey is the lookup key of an exact source: lowercased, without a
|
||||
// trailing slash on its path, with its query as written.
|
||||
func exactKey(source string) string {
|
||||
p, q, hasQuery := strings.Cut(source, "?")
|
||||
key := strings.ToLower(trimSlash(p))
|
||||
if hasQuery {
|
||||
key += "?" + q
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// exactKeys are the lookup keys a request path and query may match.
|
||||
func exactKeys(path, rawQuery string) []string {
|
||||
keys := []string{strings.ToLower(trimSlash(path))}
|
||||
if rawQuery != "" {
|
||||
keys = append(keys, keys[0]+"?"+rawQuery)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// exactMatches reports whether an exact rule matches the request: the
|
||||
// source's path with the rule's case and trailing-slash options, and, when
|
||||
// the source has a query, the request's query exactly (Q-9).
|
||||
func exactMatches(r models.Redirect, path, rawQuery string) bool {
|
||||
srcPath, srcQuery, hasQuery := strings.Cut(r.Source, "?")
|
||||
if hasQuery && srcQuery != rawQuery {
|
||||
return false
|
||||
}
|
||||
eq := func(a, b string) bool {
|
||||
if r.IgnoreCase {
|
||||
return strings.EqualFold(a, b)
|
||||
}
|
||||
return a == b
|
||||
}
|
||||
if eq(srcPath, path) {
|
||||
return true
|
||||
}
|
||||
return r.TrailingSlash && eq(trimSlash(srcPath), trimSlash(path))
|
||||
}
|
||||
|
||||
// RedirectSet is a set of redirect rules ready to match request paths:
|
||||
// exact sources by key, then patterns by the length of their literal
|
||||
// prefix (longest first, then id), then regex sources by id (D-17).
|
||||
type RedirectSet struct {
|
||||
exact map[string][]models.Redirect
|
||||
patterns []*compiledRedirect
|
||||
regexes []*compiledRedirect
|
||||
// fetch loads the exact rules of more keys on demand (the resolver
|
||||
// reads only the rows a path can match); nil when every rule is in
|
||||
// memory.
|
||||
fetch func(keys []string) ([]models.Redirect, error)
|
||||
fetched map[string]bool
|
||||
}
|
||||
|
||||
// NewRedirectSet compiles rules into a set. A pattern or regex source that
|
||||
// does not compile is left out.
|
||||
func NewRedirectSet(rules []models.Redirect) *RedirectSet {
|
||||
s := &RedirectSet{exact: map[string][]models.Redirect{}, fetched: map[string]bool{}}
|
||||
s.add(rules)
|
||||
return s
|
||||
}
|
||||
|
||||
// add puts rules into the set, once per id.
|
||||
func (s *RedirectSet) add(rules []models.Redirect) {
|
||||
for _, r := range rules {
|
||||
switch r.Match {
|
||||
case models.RedirectPattern, models.RedirectRegex:
|
||||
re := sourceExpr(r)
|
||||
if re == nil {
|
||||
continue
|
||||
}
|
||||
c := &compiledRedirect{rule: r, re: re, prefix: literalPrefix(r.Source)}
|
||||
if r.Match == models.RedirectPattern {
|
||||
s.patterns = append(s.patterns, c)
|
||||
} else {
|
||||
s.regexes = append(s.regexes, c)
|
||||
}
|
||||
default:
|
||||
key := exactKey(r.Source)
|
||||
dup := false
|
||||
for _, have := range s.exact[key] {
|
||||
dup = dup || have.ID == r.ID
|
||||
}
|
||||
if !dup {
|
||||
s.exact[key] = append(s.exact[key], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(s.patterns, func(i, j int) bool {
|
||||
if s.patterns[i].prefix != s.patterns[j].prefix {
|
||||
return s.patterns[i].prefix > s.patterns[j].prefix
|
||||
}
|
||||
return s.patterns[i].rule.ID < s.patterns[j].rule.ID
|
||||
})
|
||||
sort.SliceStable(s.regexes, func(i, j int) bool { return s.regexes[i].rule.ID < s.regexes[j].rule.ID })
|
||||
for key := range s.exact {
|
||||
rows := s.exact[key]
|
||||
sort.SliceStable(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID })
|
||||
}
|
||||
}
|
||||
|
||||
// LoadRedirects returns every redirect rule.
|
||||
func LoadRedirects(db *gorm.DB) ([]models.Redirect, error) {
|
||||
var rows []models.Redirect
|
||||
if err := db.Where("match = ? AND source = ?", models.RedirectExact, path).Limit(1).Find(&rows).Error; err != nil {
|
||||
err := db.Order("id ASC").Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// LoadRedirectSet reads the rule set for the resolver: every pattern and
|
||||
// regex rule now, exact rules per path as the set needs them (indexed on
|
||||
// lower(source)).
|
||||
func LoadRedirectSet(db *gorm.DB) (*RedirectSet, error) {
|
||||
var rows []models.Redirect
|
||||
if err := db.Where("match <> ?", models.RedirectExact).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
s := NewRedirectSet(rows)
|
||||
s.fetch = func(keys []string) ([]models.Redirect, error) {
|
||||
var out []models.Redirect
|
||||
err := db.Where("match = ? AND lower(source) IN ?", models.RedirectExact, keys).Order("id ASC").Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
return &rows[0], nil
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ensure loads the exact rules of keys the set has not read yet.
|
||||
func (s *RedirectSet) ensure(keys []string) error {
|
||||
if s.fetch == nil {
|
||||
return nil
|
||||
}
|
||||
var missing []string
|
||||
for _, k := range keys {
|
||||
if !s.fetched[k] {
|
||||
missing = append(missing, k)
|
||||
s.fetched[k] = true
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.fetch(missing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.add(rows)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedirectHit is a rule that matched a request and the target it sends
|
||||
// the request to: a path or an absolute URL, "" for 410 Gone.
|
||||
type RedirectHit struct {
|
||||
Rule models.Redirect
|
||||
Target string
|
||||
}
|
||||
|
||||
// Match returns the first rule that matches path and rawQuery, in the
|
||||
// D-17 order, with its target. A rule whose substituted target would leave
|
||||
// the target's own host (an open redirect) does not match.
|
||||
func (s *RedirectSet) Match(path, rawQuery string) (RedirectHit, bool, error) {
|
||||
if err := s.ensure(exactKeys(path, rawQuery)); err != nil {
|
||||
return RedirectHit{}, false, err
|
||||
}
|
||||
for _, key := range exactKeys(path, rawQuery) {
|
||||
for _, r := range s.exact[key] {
|
||||
if exactMatches(r, path, rawQuery) {
|
||||
if hit, ok := finishHit(r, r.Target, rawQuery, false); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
full := path
|
||||
if rawQuery != "" {
|
||||
full = path + "?" + rawQuery
|
||||
}
|
||||
for _, c := range s.patterns {
|
||||
subject := path
|
||||
if strings.Contains(c.rule.Source, "?") {
|
||||
subject = full
|
||||
}
|
||||
m := c.re.FindStringSubmatch(subject)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
target := expandStars(c.rule.Target, m[1:])
|
||||
if hit, ok := finishHit(c.rule, target, rawQuery, strings.Contains(c.rule.Source, "?")); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
for _, c := range s.regexes {
|
||||
m := c.re.FindStringSubmatch(path)
|
||||
if m == nil && c.rule.TrailingSlash {
|
||||
m = c.re.FindStringSubmatch(toggleSlash(path))
|
||||
}
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
target := expandGroups(c.re, c.rule.Target, m)
|
||||
if hit, ok := finishHit(c.rule, target, rawQuery, false); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
return RedirectHit{}, false, nil
|
||||
}
|
||||
|
||||
// finishHit checks a substituted target against its literal and passes
|
||||
// the request's query on when the rule asks for it. ok is false when the
|
||||
// target would point to another host than its literal names (T-32-16).
|
||||
func finishHit(r models.Redirect, target, rawQuery string, queryMatched bool) (RedirectHit, bool) {
|
||||
if r.Status == models.Status410 {
|
||||
return RedirectHit{Rule: r}, true
|
||||
}
|
||||
safe, ok := SafeTarget(r.Target, target)
|
||||
if !ok {
|
||||
return RedirectHit{}, false
|
||||
}
|
||||
if r.Query == models.QueryPass && rawQuery != "" && !queryMatched && !strings.Contains(r.Source, "?") {
|
||||
sep := "?"
|
||||
if strings.Contains(safe, "?") {
|
||||
sep = "&"
|
||||
}
|
||||
safe += sep + rawQuery
|
||||
}
|
||||
return RedirectHit{Rule: r, Target: safe}, true
|
||||
}
|
||||
|
||||
// SafeTarget checks a substituted target against the rule's literal
|
||||
// target (T-32-16): a relative literal must stay a path on this site
|
||||
// (repeated leading slashes collapse, so a capture cannot make it
|
||||
// protocol-relative), and an absolute literal must keep its scheme and
|
||||
// host. ok is false otherwise.
|
||||
func SafeTarget(literal, target string) (string, bool) {
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return "", false
|
||||
}
|
||||
if lu, err := url.Parse(literal); err == nil && lu.IsAbs() {
|
||||
u, err := url.Parse(target)
|
||||
base, berr := url.Parse(regexpGroup.ReplaceAllString(strings.ReplaceAll(literal, "*", ""), ""))
|
||||
if err != nil || berr != nil || !u.IsAbs() || u.Host == "" ||
|
||||
!strings.EqualFold(u.Scheme, base.Scheme) || !strings.EqualFold(u.Host, base.Host) {
|
||||
return "", false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
if !strings.HasPrefix(target, "/") && !strings.HasPrefix(target, `\`) {
|
||||
return "", false
|
||||
}
|
||||
target = "/" + strings.TrimLeft(target, `/\`)
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.IsAbs() || u.Host != "" {
|
||||
return "", false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
|
||||
// regexpGroup finds $1, ${1} and ${name} references in a target.
|
||||
var regexpGroup = regexp.MustCompile(`\$(\d+|\{[A-Za-z0-9_]+\})`)
|
||||
|
||||
// expandStars replaces each * of a pattern target with the matching
|
||||
// capture of the source, in order; a * without a capture becomes "".
|
||||
func expandStars(target string, captures []string) string {
|
||||
if !strings.Contains(target, "*") {
|
||||
return target
|
||||
}
|
||||
var b strings.Builder
|
||||
i := 0
|
||||
for _, part := range strings.SplitAfter(target, "*") {
|
||||
if !strings.HasSuffix(part, "*") {
|
||||
b.WriteString(part)
|
||||
continue
|
||||
}
|
||||
b.WriteString(strings.TrimSuffix(part, "*"))
|
||||
if i < len(captures) {
|
||||
b.WriteString(escapeCapture(captures[i]))
|
||||
}
|
||||
i++
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// expandGroups replaces $1, ${1} and ${name} in a regex target with the
|
||||
// escaped captures of m; an unknown group becomes "".
|
||||
func expandGroups(re *regexp.Regexp, target string, m []string) string {
|
||||
return regexpGroup.ReplaceAllStringFunc(target, func(ref string) string {
|
||||
name := strings.Trim(strings.TrimPrefix(ref, "$"), "{}")
|
||||
idx, err := strconv.Atoi(name)
|
||||
if err != nil {
|
||||
idx = re.SubexpIndex(name)
|
||||
}
|
||||
if idx <= 0 || idx >= len(m) {
|
||||
return ""
|
||||
}
|
||||
return escapeCapture(m[idx])
|
||||
})
|
||||
}
|
||||
|
||||
// escapeCapture percent-encodes a captured piece of a decoded path for a
|
||||
// target's path: unreserved characters, "/" and the sub-delimiters stay,
|
||||
// everything else (?, #, \, %, spaces, non-ASCII) is encoded, so a capture
|
||||
// can neither start a query or fragment nor carry a backslash.
|
||||
func escapeCapture(s string) string {
|
||||
const keep = "-._~/!$&'()*+,;=:@"
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.IndexByte(keep, c) >= 0 {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteString("%" + strings.ToUpper(strconv.FormatUint(uint64(c)>>4, 16)) + strings.ToUpper(strconv.FormatUint(uint64(c)&15, 16)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Chain is the result of following redirects from one URL.
|
||||
type Chain struct {
|
||||
// Hops are the rules that fired, in order.
|
||||
Hops []RedirectHit
|
||||
// Visited are the URLs the chain went through: the start, then each
|
||||
// internal target (path with query).
|
||||
Visited []string
|
||||
// Final is the last target: a path or an absolute URL.
|
||||
Final string
|
||||
// Loop is true when a URL or a rule came round again, or the chain
|
||||
// reached MaxRedirectHops.
|
||||
Loop bool
|
||||
// Gone is true when the chain ends on a 410 rule.
|
||||
Gone bool
|
||||
// Status is 302 when any hop is temporary, else 301.
|
||||
Status int
|
||||
}
|
||||
|
||||
// Follow follows the set from path and rawQuery, at most max hops. site is
|
||||
// the site URL: an absolute target on it is followed like a path.
|
||||
func (s *RedirectSet) Follow(site, path, rawQuery string, max int) (Chain, error) {
|
||||
ch := Chain{Status: models.Status301}
|
||||
cur, query := path, rawQuery
|
||||
seen := map[string]bool{joinQuery(cur, query): true}
|
||||
used := map[uint]bool{}
|
||||
ch.Visited = append(ch.Visited, joinQuery(cur, query))
|
||||
for {
|
||||
hit, ok, err := s.Match(cur, query)
|
||||
if err != nil {
|
||||
return ch, err
|
||||
}
|
||||
if !ok {
|
||||
return ch, nil
|
||||
}
|
||||
if used[hit.Rule.ID] {
|
||||
ch.Loop = true
|
||||
return ch, nil
|
||||
}
|
||||
used[hit.Rule.ID] = true
|
||||
ch.Hops = append(ch.Hops, hit)
|
||||
if hit.Rule.Status == models.Status302 {
|
||||
ch.Status = models.Status302
|
||||
}
|
||||
if hit.Rule.Status == models.Status410 {
|
||||
ch.Gone = true
|
||||
return ch, nil
|
||||
}
|
||||
ch.Final = hit.Target
|
||||
next, nextQuery, internal := internalTarget(site, hit.Target)
|
||||
if !internal {
|
||||
return ch, nil
|
||||
}
|
||||
key := joinQuery(next, nextQuery)
|
||||
ch.Visited = append(ch.Visited, key)
|
||||
if seen[key] || len(ch.Hops) >= max {
|
||||
ch.Loop = true
|
||||
return ch, nil
|
||||
}
|
||||
seen[key] = true
|
||||
cur, query = next, nextQuery
|
||||
}
|
||||
}
|
||||
|
||||
// joinQuery is path with its query.
|
||||
func joinQuery(path, rawQuery string) string {
|
||||
if rawQuery == "" {
|
||||
return path
|
||||
}
|
||||
return path + "?" + rawQuery
|
||||
}
|
||||
|
||||
// internalTarget returns the decoded path and the query of a target on
|
||||
// this site: a path, or an absolute URL whose origin is the site URL.
|
||||
func internalTarget(site, target string) (string, string, bool) {
|
||||
u, err := url.Parse(target)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
if u.IsAbs() {
|
||||
su, err := url.Parse(site)
|
||||
if site == "" || err != nil || !strings.EqualFold(su.Host, u.Host) || !strings.EqualFold(su.Scheme, u.Scheme) {
|
||||
return "", "", false
|
||||
}
|
||||
} else if !strings.HasPrefix(target, "/") {
|
||||
return "", "", false
|
||||
}
|
||||
p := u.Path
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
return p, u.RawQuery, true
|
||||
}
|
||||
|
||||
// redirectHopsLogged keeps the URLs whose runtime redirect cycle was
|
||||
// logged, so each is logged once.
|
||||
var redirectHopsLogged sync.Map
|
||||
|
||||
// logRedirectCycle logs a redirect cycle met at runtime, once per URL.
|
||||
func logRedirectCycle(url string, ch Chain) {
|
||||
if _, seen := redirectHopsLogged.LoadOrStore(url, true); seen {
|
||||
return
|
||||
}
|
||||
log.Warn("golem15.seo: redirect cycle, answering not_found", "url", url, "path", strings.Join(ch.Visited, " → "))
|
||||
}
|
||||
|
||||
// RedirectValues are a redirect's values for history.
|
||||
|
||||
@@ -360,12 +360,16 @@ func (r *Resolver) Resolve(ctx context.Context, rawURL string) (Result, error) {
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
redirect, err := exactRedirect(db, path)
|
||||
set, err := LoadRedirectSet(db)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if redirect != nil {
|
||||
return r.redirectAnswer(svc.SiteURL()+path, *redirect), nil
|
||||
chain, err := set.Follow(svc.SiteURL(), path, rawQueryOf(rawURL), MaxRedirectHops)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if len(chain.Hops) > 0 {
|
||||
return r.redirectAnswer(svc.SiteURL()+joinQuery(path, rawQueryOf(rawURL)), chain), nil
|
||||
}
|
||||
key, match, rec, err := r.lookup(ctx, path)
|
||||
if err != nil {
|
||||
@@ -567,16 +571,21 @@ func (r *Resolver) resolveAt(ctx context.Context, db *gorm.DB, settings models.S
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// redirectAnswer is the answer of a matched redirect: gone for a 410,
|
||||
// else redirect with the rule's status and its target absolute on the
|
||||
// site URL.
|
||||
func (r *Resolver) redirectAnswer(self string, rule models.Redirect) Result {
|
||||
// redirectAnswer is the answer of a matched redirect chain (P-21): gone
|
||||
// for a chain that ends on a 410, not_found (logged once per URL) for a
|
||||
// cycle, else redirect with the chain's status and its final target
|
||||
// absolute on the site URL, so the visitor makes one hop.
|
||||
func (r *Resolver) redirectAnswer(self string, ch Chain) Result {
|
||||
res := Result{URL: self, Staging: r.Service.Staging(), Sources: map[string]string{}}
|
||||
if rule.Status == models.Status410 {
|
||||
switch {
|
||||
case ch.Loop:
|
||||
logRedirectCycle(self, ch)
|
||||
res.Action, res.Status = ActionNotFound, 404
|
||||
case ch.Gone:
|
||||
res.Action, res.Status = ActionGone, models.Status410
|
||||
return res
|
||||
default:
|
||||
res.Action, res.Status, res.Location = ActionRedirect, ch.Status, AbsoluteURL(r.Service.SiteURL(), ch.Final)
|
||||
}
|
||||
res.Action, res.Status, res.Location = ActionRedirect, rule.Status, AbsoluteURL(r.Service.SiteURL(), rule.Target)
|
||||
return res
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user