feat: redirect engine with loop refusal and chain notices
- rule set: exact sources by key, patterns by literal prefix, regex by id (stdlib RE2, compiled once per source); ignore case, trailing slash, query ignore/pass, a source with ? matches its query exactly - targets: captures escaped, a relative target never leaves the site, an absolute one keeps its host - SaveRedirect validates, simulates the rule in place (exact, pattern and regex probes, 10 hops) and refuses a loop; chains answer the notice - resolve follows the chain to its final target, 410 answers gone, a runtime cycle answers not_found and is logged once
This commit is contained in:
@@ -1,29 +1,539 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// log is the classes' logger (component=seo).
|
||||
var log = diary.For("seo")
|
||||
|
||||
// The resolve actions of the redirect stage.
|
||||
const (
|
||||
ActionRedirect = "redirect"
|
||||
ActionGone = "gone"
|
||||
)
|
||||
|
||||
// exactRedirect returns the exact redirect whose source is path, nil when
|
||||
// there is none.
|
||||
func exactRedirect(db *gorm.DB, path string) (*models.Redirect, error) {
|
||||
// MaxRedirectHops caps how many redirects the resolver follows for one
|
||||
// URL, and how far the save-time simulation looks (P-19, P-22).
|
||||
const MaxRedirectHops = 10
|
||||
|
||||
// MaxRegexSource caps a regex source (T-32-17).
|
||||
const MaxRegexSource = 512
|
||||
|
||||
// probeToken stands in for * when a pattern source is simulated.
|
||||
const probeToken = "seoprobe"
|
||||
|
||||
// compiledRedirect is a pattern or regex rule with its expression.
|
||||
type compiledRedirect struct {
|
||||
rule models.Redirect
|
||||
re *regexp.Regexp
|
||||
prefix int
|
||||
}
|
||||
|
||||
// regexCache keeps the compiled expressions of pattern and regex sources,
|
||||
// so each distinct source is compiled once (stdlib RE2: linear time on any
|
||||
// input).
|
||||
var regexCache sync.Map
|
||||
|
||||
// sourceExpr returns the expression of a pattern or regex source, nil
|
||||
// when it does not compile.
|
||||
func sourceExpr(r models.Redirect) *regexp.Regexp {
|
||||
key := r.Match + "\x00" + strconv.FormatBool(r.IgnoreCase) + "\x00" + strconv.FormatBool(r.TrailingSlash) + "\x00" + r.Source
|
||||
if re, ok := regexCache.Load(key); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
}
|
||||
var expr string
|
||||
switch r.Match {
|
||||
case models.RedirectPattern:
|
||||
expr = WildcardExpr(r.Source, r.TrailingSlash)
|
||||
case models.RedirectRegex:
|
||||
expr = r.Source
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
if r.IgnoreCase {
|
||||
expr = "(?i)" + expr
|
||||
}
|
||||
re, err := regexp.Compile(expr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
regexCache.Store(key, re)
|
||||
return re
|
||||
}
|
||||
|
||||
// WildcardExpr turns a pattern source into an anchored expression: every *
|
||||
// matches any text (a capture), everything else is literal. With
|
||||
// trailingSlash a source that does not end in * or / also matches with a
|
||||
// trailing slash, and one that ends in / also without it.
|
||||
func WildcardExpr(source string, trailingSlash bool) string {
|
||||
src := source
|
||||
optional := false
|
||||
if trailingSlash && !strings.Contains(src, "?") && !strings.HasSuffix(src, "*") && src != "/" {
|
||||
src = strings.TrimSuffix(src, "/")
|
||||
optional = true
|
||||
}
|
||||
expr := "^" + strings.ReplaceAll(regexp.QuoteMeta(src), `\*`, `(.*)`)
|
||||
if optional {
|
||||
expr += "/?"
|
||||
}
|
||||
return expr + "$"
|
||||
}
|
||||
|
||||
// literalPrefix is the length of a pattern source before its first *.
|
||||
func literalPrefix(source string) int {
|
||||
if i := strings.Index(source, "*"); i >= 0 {
|
||||
return i
|
||||
}
|
||||
return len(source)
|
||||
}
|
||||
|
||||
// trimSlash drops one trailing slash, except from the root.
|
||||
func trimSlash(p string) string {
|
||||
if len(p) > 1 && strings.HasSuffix(p, "/") {
|
||||
return strings.TrimSuffix(p, "/")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// toggleSlash adds a trailing slash, or drops it; the root stays.
|
||||
func toggleSlash(p string) string {
|
||||
if p == "/" {
|
||||
return p
|
||||
}
|
||||
if strings.HasSuffix(p, "/") {
|
||||
return strings.TrimSuffix(p, "/")
|
||||
}
|
||||
return p + "/"
|
||||
}
|
||||
|
||||
// exactKey is the lookup key of an exact source: lowercased, without a
|
||||
// trailing slash on its path, with its query as written.
|
||||
func exactKey(source string) string {
|
||||
p, q, hasQuery := strings.Cut(source, "?")
|
||||
key := strings.ToLower(trimSlash(p))
|
||||
if hasQuery {
|
||||
key += "?" + q
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// exactKeys are the lookup keys a request path and query may match.
|
||||
func exactKeys(path, rawQuery string) []string {
|
||||
keys := []string{strings.ToLower(trimSlash(path))}
|
||||
if rawQuery != "" {
|
||||
keys = append(keys, keys[0]+"?"+rawQuery)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// exactMatches reports whether an exact rule matches the request: the
|
||||
// source's path with the rule's case and trailing-slash options, and, when
|
||||
// the source has a query, the request's query exactly (Q-9).
|
||||
func exactMatches(r models.Redirect, path, rawQuery string) bool {
|
||||
srcPath, srcQuery, hasQuery := strings.Cut(r.Source, "?")
|
||||
if hasQuery && srcQuery != rawQuery {
|
||||
return false
|
||||
}
|
||||
eq := func(a, b string) bool {
|
||||
if r.IgnoreCase {
|
||||
return strings.EqualFold(a, b)
|
||||
}
|
||||
return a == b
|
||||
}
|
||||
if eq(srcPath, path) {
|
||||
return true
|
||||
}
|
||||
return r.TrailingSlash && eq(trimSlash(srcPath), trimSlash(path))
|
||||
}
|
||||
|
||||
// RedirectSet is a set of redirect rules ready to match request paths:
|
||||
// exact sources by key, then patterns by the length of their literal
|
||||
// prefix (longest first, then id), then regex sources by id (D-17).
|
||||
type RedirectSet struct {
|
||||
exact map[string][]models.Redirect
|
||||
patterns []*compiledRedirect
|
||||
regexes []*compiledRedirect
|
||||
// fetch loads the exact rules of more keys on demand (the resolver
|
||||
// reads only the rows a path can match); nil when every rule is in
|
||||
// memory.
|
||||
fetch func(keys []string) ([]models.Redirect, error)
|
||||
fetched map[string]bool
|
||||
}
|
||||
|
||||
// NewRedirectSet compiles rules into a set. A pattern or regex source that
|
||||
// does not compile is left out.
|
||||
func NewRedirectSet(rules []models.Redirect) *RedirectSet {
|
||||
s := &RedirectSet{exact: map[string][]models.Redirect{}, fetched: map[string]bool{}}
|
||||
s.add(rules)
|
||||
return s
|
||||
}
|
||||
|
||||
// add puts rules into the set, once per id.
|
||||
func (s *RedirectSet) add(rules []models.Redirect) {
|
||||
for _, r := range rules {
|
||||
switch r.Match {
|
||||
case models.RedirectPattern, models.RedirectRegex:
|
||||
re := sourceExpr(r)
|
||||
if re == nil {
|
||||
continue
|
||||
}
|
||||
c := &compiledRedirect{rule: r, re: re, prefix: literalPrefix(r.Source)}
|
||||
if r.Match == models.RedirectPattern {
|
||||
s.patterns = append(s.patterns, c)
|
||||
} else {
|
||||
s.regexes = append(s.regexes, c)
|
||||
}
|
||||
default:
|
||||
key := exactKey(r.Source)
|
||||
dup := false
|
||||
for _, have := range s.exact[key] {
|
||||
dup = dup || have.ID == r.ID
|
||||
}
|
||||
if !dup {
|
||||
s.exact[key] = append(s.exact[key], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(s.patterns, func(i, j int) bool {
|
||||
if s.patterns[i].prefix != s.patterns[j].prefix {
|
||||
return s.patterns[i].prefix > s.patterns[j].prefix
|
||||
}
|
||||
return s.patterns[i].rule.ID < s.patterns[j].rule.ID
|
||||
})
|
||||
sort.SliceStable(s.regexes, func(i, j int) bool { return s.regexes[i].rule.ID < s.regexes[j].rule.ID })
|
||||
for key := range s.exact {
|
||||
rows := s.exact[key]
|
||||
sort.SliceStable(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID })
|
||||
}
|
||||
}
|
||||
|
||||
// LoadRedirects returns every redirect rule.
|
||||
func LoadRedirects(db *gorm.DB) ([]models.Redirect, error) {
|
||||
var rows []models.Redirect
|
||||
if err := db.Where("match = ? AND source = ?", models.RedirectExact, path).Limit(1).Find(&rows).Error; err != nil {
|
||||
err := db.Order("id ASC").Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// LoadRedirectSet reads the rule set for the resolver: every pattern and
|
||||
// regex rule now, exact rules per path as the set needs them (indexed on
|
||||
// lower(source)).
|
||||
func LoadRedirectSet(db *gorm.DB) (*RedirectSet, error) {
|
||||
var rows []models.Redirect
|
||||
if err := db.Where("match <> ?", models.RedirectExact).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
s := NewRedirectSet(rows)
|
||||
s.fetch = func(keys []string) ([]models.Redirect, error) {
|
||||
var out []models.Redirect
|
||||
err := db.Where("match = ? AND lower(source) IN ?", models.RedirectExact, keys).Order("id ASC").Find(&out).Error
|
||||
return out, err
|
||||
}
|
||||
return &rows[0], nil
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ensure loads the exact rules of keys the set has not read yet.
|
||||
func (s *RedirectSet) ensure(keys []string) error {
|
||||
if s.fetch == nil {
|
||||
return nil
|
||||
}
|
||||
var missing []string
|
||||
for _, k := range keys {
|
||||
if !s.fetched[k] {
|
||||
missing = append(missing, k)
|
||||
s.fetched[k] = true
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.fetch(missing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.add(rows)
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedirectHit is a rule that matched a request and the target it sends
|
||||
// the request to: a path or an absolute URL, "" for 410 Gone.
|
||||
type RedirectHit struct {
|
||||
Rule models.Redirect
|
||||
Target string
|
||||
}
|
||||
|
||||
// Match returns the first rule that matches path and rawQuery, in the
|
||||
// D-17 order, with its target. A rule whose substituted target would leave
|
||||
// the target's own host (an open redirect) does not match.
|
||||
func (s *RedirectSet) Match(path, rawQuery string) (RedirectHit, bool, error) {
|
||||
if err := s.ensure(exactKeys(path, rawQuery)); err != nil {
|
||||
return RedirectHit{}, false, err
|
||||
}
|
||||
for _, key := range exactKeys(path, rawQuery) {
|
||||
for _, r := range s.exact[key] {
|
||||
if exactMatches(r, path, rawQuery) {
|
||||
if hit, ok := finishHit(r, r.Target, rawQuery, false); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
full := path
|
||||
if rawQuery != "" {
|
||||
full = path + "?" + rawQuery
|
||||
}
|
||||
for _, c := range s.patterns {
|
||||
subject := path
|
||||
if strings.Contains(c.rule.Source, "?") {
|
||||
subject = full
|
||||
}
|
||||
m := c.re.FindStringSubmatch(subject)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
target := expandStars(c.rule.Target, m[1:])
|
||||
if hit, ok := finishHit(c.rule, target, rawQuery, strings.Contains(c.rule.Source, "?")); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
for _, c := range s.regexes {
|
||||
m := c.re.FindStringSubmatch(path)
|
||||
if m == nil && c.rule.TrailingSlash {
|
||||
m = c.re.FindStringSubmatch(toggleSlash(path))
|
||||
}
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
target := expandGroups(c.re, c.rule.Target, m)
|
||||
if hit, ok := finishHit(c.rule, target, rawQuery, false); ok {
|
||||
return hit, true, nil
|
||||
}
|
||||
}
|
||||
return RedirectHit{}, false, nil
|
||||
}
|
||||
|
||||
// finishHit checks a substituted target against its literal and passes
|
||||
// the request's query on when the rule asks for it. ok is false when the
|
||||
// target would point to another host than its literal names (T-32-16).
|
||||
func finishHit(r models.Redirect, target, rawQuery string, queryMatched bool) (RedirectHit, bool) {
|
||||
if r.Status == models.Status410 {
|
||||
return RedirectHit{Rule: r}, true
|
||||
}
|
||||
safe, ok := SafeTarget(r.Target, target)
|
||||
if !ok {
|
||||
return RedirectHit{}, false
|
||||
}
|
||||
if r.Query == models.QueryPass && rawQuery != "" && !queryMatched && !strings.Contains(r.Source, "?") {
|
||||
sep := "?"
|
||||
if strings.Contains(safe, "?") {
|
||||
sep = "&"
|
||||
}
|
||||
safe += sep + rawQuery
|
||||
}
|
||||
return RedirectHit{Rule: r, Target: safe}, true
|
||||
}
|
||||
|
||||
// SafeTarget checks a substituted target against the rule's literal
|
||||
// target (T-32-16): a relative literal must stay a path on this site
|
||||
// (repeated leading slashes collapse, so a capture cannot make it
|
||||
// protocol-relative), and an absolute literal must keep its scheme and
|
||||
// host. ok is false otherwise.
|
||||
func SafeTarget(literal, target string) (string, bool) {
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return "", false
|
||||
}
|
||||
if lu, err := url.Parse(literal); err == nil && lu.IsAbs() {
|
||||
u, err := url.Parse(target)
|
||||
base, berr := url.Parse(regexpGroup.ReplaceAllString(strings.ReplaceAll(literal, "*", ""), ""))
|
||||
if err != nil || berr != nil || !u.IsAbs() || u.Host == "" ||
|
||||
!strings.EqualFold(u.Scheme, base.Scheme) || !strings.EqualFold(u.Host, base.Host) {
|
||||
return "", false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
if !strings.HasPrefix(target, "/") && !strings.HasPrefix(target, `\`) {
|
||||
return "", false
|
||||
}
|
||||
target = "/" + strings.TrimLeft(target, `/\`)
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || u.IsAbs() || u.Host != "" {
|
||||
return "", false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
|
||||
// regexpGroup finds $1, ${1} and ${name} references in a target.
|
||||
var regexpGroup = regexp.MustCompile(`\$(\d+|\{[A-Za-z0-9_]+\})`)
|
||||
|
||||
// expandStars replaces each * of a pattern target with the matching
|
||||
// capture of the source, in order; a * without a capture becomes "".
|
||||
func expandStars(target string, captures []string) string {
|
||||
if !strings.Contains(target, "*") {
|
||||
return target
|
||||
}
|
||||
var b strings.Builder
|
||||
i := 0
|
||||
for _, part := range strings.SplitAfter(target, "*") {
|
||||
if !strings.HasSuffix(part, "*") {
|
||||
b.WriteString(part)
|
||||
continue
|
||||
}
|
||||
b.WriteString(strings.TrimSuffix(part, "*"))
|
||||
if i < len(captures) {
|
||||
b.WriteString(escapeCapture(captures[i]))
|
||||
}
|
||||
i++
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// expandGroups replaces $1, ${1} and ${name} in a regex target with the
|
||||
// escaped captures of m; an unknown group becomes "".
|
||||
func expandGroups(re *regexp.Regexp, target string, m []string) string {
|
||||
return regexpGroup.ReplaceAllStringFunc(target, func(ref string) string {
|
||||
name := strings.Trim(strings.TrimPrefix(ref, "$"), "{}")
|
||||
idx, err := strconv.Atoi(name)
|
||||
if err != nil {
|
||||
idx = re.SubexpIndex(name)
|
||||
}
|
||||
if idx <= 0 || idx >= len(m) {
|
||||
return ""
|
||||
}
|
||||
return escapeCapture(m[idx])
|
||||
})
|
||||
}
|
||||
|
||||
// escapeCapture percent-encodes a captured piece of a decoded path for a
|
||||
// target's path: unreserved characters, "/" and the sub-delimiters stay,
|
||||
// everything else (?, #, \, %, spaces, non-ASCII) is encoded, so a capture
|
||||
// can neither start a query or fragment nor carry a backslash.
|
||||
func escapeCapture(s string) string {
|
||||
const keep = "-._~/!$&'()*+,;=:@"
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.IndexByte(keep, c) >= 0 {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteString("%" + strings.ToUpper(strconv.FormatUint(uint64(c)>>4, 16)) + strings.ToUpper(strconv.FormatUint(uint64(c)&15, 16)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Chain is the result of following redirects from one URL.
|
||||
type Chain struct {
|
||||
// Hops are the rules that fired, in order.
|
||||
Hops []RedirectHit
|
||||
// Visited are the URLs the chain went through: the start, then each
|
||||
// internal target (path with query).
|
||||
Visited []string
|
||||
// Final is the last target: a path or an absolute URL.
|
||||
Final string
|
||||
// Loop is true when a URL or a rule came round again, or the chain
|
||||
// reached MaxRedirectHops.
|
||||
Loop bool
|
||||
// Gone is true when the chain ends on a 410 rule.
|
||||
Gone bool
|
||||
// Status is 302 when any hop is temporary, else 301.
|
||||
Status int
|
||||
}
|
||||
|
||||
// Follow follows the set from path and rawQuery, at most max hops. site is
|
||||
// the site URL: an absolute target on it is followed like a path.
|
||||
func (s *RedirectSet) Follow(site, path, rawQuery string, max int) (Chain, error) {
|
||||
ch := Chain{Status: models.Status301}
|
||||
cur, query := path, rawQuery
|
||||
seen := map[string]bool{joinQuery(cur, query): true}
|
||||
used := map[uint]bool{}
|
||||
ch.Visited = append(ch.Visited, joinQuery(cur, query))
|
||||
for {
|
||||
hit, ok, err := s.Match(cur, query)
|
||||
if err != nil {
|
||||
return ch, err
|
||||
}
|
||||
if !ok {
|
||||
return ch, nil
|
||||
}
|
||||
if used[hit.Rule.ID] {
|
||||
ch.Loop = true
|
||||
return ch, nil
|
||||
}
|
||||
used[hit.Rule.ID] = true
|
||||
ch.Hops = append(ch.Hops, hit)
|
||||
if hit.Rule.Status == models.Status302 {
|
||||
ch.Status = models.Status302
|
||||
}
|
||||
if hit.Rule.Status == models.Status410 {
|
||||
ch.Gone = true
|
||||
return ch, nil
|
||||
}
|
||||
ch.Final = hit.Target
|
||||
next, nextQuery, internal := internalTarget(site, hit.Target)
|
||||
if !internal {
|
||||
return ch, nil
|
||||
}
|
||||
key := joinQuery(next, nextQuery)
|
||||
ch.Visited = append(ch.Visited, key)
|
||||
if seen[key] || len(ch.Hops) >= max {
|
||||
ch.Loop = true
|
||||
return ch, nil
|
||||
}
|
||||
seen[key] = true
|
||||
cur, query = next, nextQuery
|
||||
}
|
||||
}
|
||||
|
||||
// joinQuery is path with its query.
|
||||
func joinQuery(path, rawQuery string) string {
|
||||
if rawQuery == "" {
|
||||
return path
|
||||
}
|
||||
return path + "?" + rawQuery
|
||||
}
|
||||
|
||||
// internalTarget returns the decoded path and the query of a target on
|
||||
// this site: a path, or an absolute URL whose origin is the site URL.
|
||||
func internalTarget(site, target string) (string, string, bool) {
|
||||
u, err := url.Parse(target)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
if u.IsAbs() {
|
||||
su, err := url.Parse(site)
|
||||
if site == "" || err != nil || !strings.EqualFold(su.Host, u.Host) || !strings.EqualFold(su.Scheme, u.Scheme) {
|
||||
return "", "", false
|
||||
}
|
||||
} else if !strings.HasPrefix(target, "/") {
|
||||
return "", "", false
|
||||
}
|
||||
p := u.Path
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
return p, u.RawQuery, true
|
||||
}
|
||||
|
||||
// redirectHopsLogged keeps the URLs whose runtime redirect cycle was
|
||||
// logged, so each is logged once.
|
||||
var redirectHopsLogged sync.Map
|
||||
|
||||
// logRedirectCycle logs a redirect cycle met at runtime, once per URL.
|
||||
func logRedirectCycle(url string, ch Chain) {
|
||||
if _, seen := redirectHopsLogged.LoadOrStore(url, true); seen {
|
||||
return
|
||||
}
|
||||
log.Warn("golem15.seo: redirect cycle, answering not_found", "url", url, "path", strings.Join(ch.Visited, " → "))
|
||||
}
|
||||
|
||||
// RedirectValues are a redirect's values for history.
|
||||
|
||||
Reference in New Issue
Block a user