package classes import ( "net/url" "regexp" "sort" "strconv" "strings" "sync" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/diary" "gorm.io/gorm" ) // log is the classes' logger (component=seo). var log = diary.For("seo") // The resolve actions of the redirect stage. const ( ActionRedirect = "redirect" ActionGone = "gone" ) // MaxRedirectHops caps how many redirects the resolver follows for one // URL, and how far the save-time simulation looks (P-19, P-22). const MaxRedirectHops = 10 // MaxRegexSource caps a regex source (T-32-17). const MaxRegexSource = 512 // probeToken stands in for * when a pattern source is simulated. const probeToken = "seoprobe" // compiledRedirect is a pattern or regex rule with its expression. type compiledRedirect struct { rule models.Redirect re *regexp.Regexp prefix int } // regexCache keeps the compiled expressions of pattern and regex sources, // so each distinct source is compiled once (stdlib RE2: linear time on any // input). var regexCache sync.Map // sourceExpr returns the expression of a pattern or regex source, nil // when it does not compile. func sourceExpr(r models.Redirect) *regexp.Regexp { key := r.Match + "\x00" + strconv.FormatBool(r.IgnoreCase) + "\x00" + strconv.FormatBool(r.TrailingSlash) + "\x00" + r.Source if re, ok := regexCache.Load(key); ok { return re.(*regexp.Regexp) } var expr string switch r.Match { case models.RedirectPattern: expr = WildcardExpr(r.Source, r.TrailingSlash) case models.RedirectRegex: expr = r.Source default: return nil } if r.IgnoreCase { expr = "(?i)" + expr } re, err := regexp.Compile(expr) if err != nil { return nil } regexCache.Store(key, re) return re } // WildcardExpr turns a pattern source into an anchored expression: every * // matches any text (a capture), everything else is literal. With // trailingSlash a source that does not end in * or / also matches with a // trailing slash, and one that ends in / also without it. func WildcardExpr(source string, trailingSlash bool) string { src := source optional := false if trailingSlash && !strings.Contains(src, "?") && !strings.HasSuffix(src, "*") && src != "/" { src = strings.TrimSuffix(src, "/") optional = true } expr := "^" + strings.ReplaceAll(regexp.QuoteMeta(src), `\*`, `(.*)`) if optional { expr += "/?" } return expr + "$" } // literalPrefix is the length of a pattern source before its first *. func literalPrefix(source string) int { if i := strings.Index(source, "*"); i >= 0 { return i } return len(source) } // trimSlash drops one trailing slash, except from the root. func trimSlash(p string) string { if len(p) > 1 && strings.HasSuffix(p, "/") { return strings.TrimSuffix(p, "/") } return p } // toggleSlash adds a trailing slash, or drops it; the root stays. func toggleSlash(p string) string { if p == "/" { return p } if strings.HasSuffix(p, "/") { return strings.TrimSuffix(p, "/") } return p + "/" } // exactKey is the lookup key of an exact source: lowercased, without a // trailing slash on its path, with its query as written. The column // golem15_seo_redirects.source_key computes the same key in SQL (migration // 202610280001); change both together. func exactKey(source string) string { p, q, hasQuery := strings.Cut(source, "?") key := strings.ToLower(trimSlash(p)) if hasQuery { key += "?" + q } return key } // exactKeys are the lookup keys a request path and query may match, the // most specific first: a source with the request's query wins over the // path-only source (WR-08). func exactKeys(path, rawQuery string) []string { key := strings.ToLower(trimSlash(path)) if rawQuery != "" { return []string{key + "?" + rawQuery, key} } return []string{key} } // exactMatches reports whether an exact rule matches the request: the // source's path with the rule's case and trailing-slash options, and, when // the source has a query, the request's query exactly (Q-9). func exactMatches(r models.Redirect, path, rawQuery string) bool { srcPath, srcQuery, hasQuery := strings.Cut(r.Source, "?") if hasQuery && srcQuery != rawQuery { return false } eq := func(a, b string) bool { if r.IgnoreCase { return strings.EqualFold(a, b) } return a == b } if eq(srcPath, path) { return true } return r.TrailingSlash && eq(trimSlash(srcPath), trimSlash(path)) } // RedirectSet is a set of redirect rules ready to match request paths: // exact sources by key, then patterns by the length of their literal // prefix (longest first, then id), then regex sources by id (D-17). type RedirectSet struct { exact map[string][]models.Redirect patterns []*compiledRedirect regexes []*compiledRedirect // fetch loads the exact rules of more keys on demand (the resolver // reads only the rows a path can match); nil when every rule is in // memory. fetch func(keys []string) ([]models.Redirect, error) fetched map[string]bool } // NewRedirectSet compiles rules into a set. A pattern or regex source that // does not compile is left out. func NewRedirectSet(rules []models.Redirect) *RedirectSet { s := &RedirectSet{exact: map[string][]models.Redirect{}, fetched: map[string]bool{}} s.add(rules) return s } // add puts rules into the set, once per id. func (s *RedirectSet) add(rules []models.Redirect) { for _, r := range rules { switch r.Match { case models.RedirectPattern, models.RedirectRegex: re := sourceExpr(r) if re == nil { continue } c := &compiledRedirect{rule: r, re: re, prefix: literalPrefix(r.Source)} if r.Match == models.RedirectPattern { s.patterns = append(s.patterns, c) } else { s.regexes = append(s.regexes, c) } default: key := exactKey(r.Source) dup := false for _, have := range s.exact[key] { dup = dup || have.ID == r.ID } if !dup { s.exact[key] = append(s.exact[key], r) } } } sort.SliceStable(s.patterns, func(i, j int) bool { if s.patterns[i].prefix != s.patterns[j].prefix { return s.patterns[i].prefix > s.patterns[j].prefix } return s.patterns[i].rule.ID < s.patterns[j].rule.ID }) sort.SliceStable(s.regexes, func(i, j int) bool { return s.regexes[i].rule.ID < s.regexes[j].rule.ID }) for key := range s.exact { rows := s.exact[key] sort.SliceStable(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID }) } } // LoadRedirects returns every redirect rule. func LoadRedirects(db *gorm.DB) ([]models.Redirect, error) { var rows []models.Redirect err := db.Order("id ASC").Find(&rows).Error return rows, err } // LoadRedirectSet reads the rule set for the resolver: every pattern and // regex rule now, exact rules per path as the set needs them, by the // generated source_key column, which holds exactKey(source) (so the fetch // sees the same rules as an in-memory set). func LoadRedirectSet(db *gorm.DB) (*RedirectSet, error) { var rows []models.Redirect if err := db.Where("match <> ?", models.RedirectExact).Order("id ASC").Find(&rows).Error; err != nil { return nil, err } s := NewRedirectSet(rows) s.fetch = func(keys []string) ([]models.Redirect, error) { var out []models.Redirect err := db.Where("match = ? AND source_key IN ?", models.RedirectExact, keys).Order("id ASC").Find(&out).Error return out, err } return s, nil } // ensure loads the exact rules of keys the set has not read yet. func (s *RedirectSet) ensure(keys []string) error { if s.fetch == nil { return nil } var missing []string for _, k := range keys { if !s.fetched[k] { missing = append(missing, k) s.fetched[k] = true } } if len(missing) == 0 { return nil } rows, err := s.fetch(missing) if err != nil { return err } s.add(rows) return nil } // RedirectHit is a rule that matched a request and the target it sends // the request to: a path or an absolute URL, "" for 410 Gone. type RedirectHit struct { Rule models.Redirect Target string } // Match returns the first rule that matches path and rawQuery, in the // D-17 order, with its target. A rule whose substituted target would leave // the target's own host (an open redirect) does not match. func (s *RedirectSet) Match(path, rawQuery string) (RedirectHit, bool, error) { if err := s.ensure(exactKeys(path, rawQuery)); err != nil { return RedirectHit{}, false, err } for _, key := range exactKeys(path, rawQuery) { for _, r := range s.exact[key] { if exactMatches(r, path, rawQuery) { if hit, ok := finishHit(r, r.Target, rawQuery, false); ok { return hit, true, nil } } } } full := path if rawQuery != "" { full = path + "?" + rawQuery } for _, c := range s.patterns { subject := path if strings.Contains(c.rule.Source, "?") { subject = full } m := c.re.FindStringSubmatch(subject) if m == nil { continue } target := expandStars(c.rule.Target, m[1:]) if hit, ok := finishHit(c.rule, target, rawQuery, strings.Contains(c.rule.Source, "?")); ok { return hit, true, nil } } for _, c := range s.regexes { m := c.re.FindStringSubmatch(path) if m == nil && c.rule.TrailingSlash { m = c.re.FindStringSubmatch(toggleSlash(path)) } if m == nil { continue } target := expandGroups(c.re, c.rule.Target, m) if hit, ok := finishHit(c.rule, target, rawQuery, false); ok { return hit, true, nil } } return RedirectHit{}, false, nil } // finishHit checks a substituted target against its literal and passes // the request's query on when the rule asks for it. ok is false when the // target would point to another host than its literal names (T-32-16). func finishHit(r models.Redirect, target, rawQuery string, queryMatched bool) (RedirectHit, bool) { if r.Status == models.Status410 { return RedirectHit{Rule: r}, true } safe, ok := SafeTarget(r.Target, target) if !ok { return RedirectHit{}, false } if r.Query == models.QueryPass && rawQuery != "" && !queryMatched && !strings.Contains(r.Source, "?") { sep := "?" if strings.Contains(safe, "?") { sep = "&" } safe += sep + rawQuery } return RedirectHit{Rule: r, Target: safe}, true } // SafeTarget checks a substituted target against the rule's literal // target (T-32-16): a relative literal must stay a path on this site // (repeated leading slashes collapse, so a capture cannot make it // protocol-relative), and an absolute literal must keep its scheme and // host. ok is false otherwise. func SafeTarget(literal, target string) (string, bool) { target = strings.TrimSpace(target) if target == "" { return "", false } if lu, err := url.Parse(literal); err == nil && lu.IsAbs() { u, err := url.Parse(target) base, berr := url.Parse(regexpGroup.ReplaceAllString(strings.ReplaceAll(literal, "*", ""), "")) if err != nil || berr != nil || !u.IsAbs() || u.Host == "" || !strings.EqualFold(u.Scheme, base.Scheme) || !strings.EqualFold(u.Host, base.Host) { return "", false } return target, true } if !strings.HasPrefix(target, "/") && !strings.HasPrefix(target, `\`) { return "", false } target = "/" + strings.TrimLeft(target, `/\`) u, err := url.Parse(target) if err != nil || u.IsAbs() || u.Host != "" { return "", false } return target, true } // regexpGroup finds $1, ${1} and ${name} references in a target. var regexpGroup = regexp.MustCompile(`\$(\d+|\{[A-Za-z0-9_]+\})`) // expandStars replaces each * of a pattern target with the matching // capture of the source, in order; a * without a capture becomes "". func expandStars(target string, captures []string) string { if !strings.Contains(target, "*") { return target } var b strings.Builder i := 0 for _, part := range strings.SplitAfter(target, "*") { if !strings.HasSuffix(part, "*") { b.WriteString(part) continue } b.WriteString(strings.TrimSuffix(part, "*")) if i < len(captures) { b.WriteString(escapeCapture(captures[i])) } i++ } return b.String() } // expandGroups replaces $1, ${1} and ${name} in a regex target with the // escaped captures of m; an unknown group becomes "". func expandGroups(re *regexp.Regexp, target string, m []string) string { return regexpGroup.ReplaceAllStringFunc(target, func(ref string) string { name := strings.Trim(strings.TrimPrefix(ref, "$"), "{}") idx, err := strconv.Atoi(name) if err != nil { idx = re.SubexpIndex(name) } if idx <= 0 || idx >= len(m) { return "" } return escapeCapture(m[idx]) }) } // escapeCapture percent-encodes a captured piece of a decoded path for a // target's path: unreserved characters, "/" and the sub-delimiters stay, // everything else (?, #, \, %, spaces, non-ASCII) is encoded, so a capture // can neither start a query or fragment nor carry a backslash. func escapeCapture(s string) string { const keep = "-._~/!$&'()*+,;=:@" var b strings.Builder for i := 0; i < len(s); i++ { c := s[i] if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.IndexByte(keep, c) >= 0 { b.WriteByte(c) continue } b.WriteString("%" + strings.ToUpper(strconv.FormatUint(uint64(c)>>4, 16)) + strings.ToUpper(strconv.FormatUint(uint64(c)&15, 16))) } return b.String() } // Chain is the result of following redirects from one URL. type Chain struct { // Hops are the rules that fired, in order. Hops []RedirectHit // Visited are the URLs the chain went through: the start, then each // internal target (path with query). Visited []string // Final is the last target: a path or an absolute URL. Final string // Loop is true when a URL or a rule came round again, or the chain // reached MaxRedirectHops. Loop bool // Gone is true when the chain ends on a 410 rule. Gone bool // Status is 302 when any hop is temporary, else 301. Status int } // Follow follows the set from path and rawQuery, at most max hops. site is // the site URL: an absolute target on it is followed like a path. func (s *RedirectSet) Follow(site, path, rawQuery string, max int) (Chain, error) { ch := Chain{Status: int(models.Status301)} cur, query := path, rawQuery seen := map[string]bool{joinQuery(cur, query): true} used := map[uint]bool{} ch.Visited = append(ch.Visited, joinQuery(cur, query)) for { hit, ok, err := s.Match(cur, query) if err != nil { return ch, err } if !ok { return ch, nil } if used[hit.Rule.ID] { ch.Loop = true return ch, nil } used[hit.Rule.ID] = true ch.Hops = append(ch.Hops, hit) if hit.Rule.Status == models.Status302 { ch.Status = int(models.Status302) } if hit.Rule.Status == models.Status410 { ch.Gone = true return ch, nil } ch.Final = hit.Target next, nextQuery, internal := internalTarget(site, hit.Target) if !internal { return ch, nil } key := joinQuery(next, nextQuery) ch.Visited = append(ch.Visited, key) if seen[key] || len(ch.Hops) >= max { ch.Loop = true return ch, nil } seen[key] = true cur, query = next, nextQuery } } // joinQuery is path with its query. func joinQuery(path, rawQuery string) string { if rawQuery == "" { return path } return path + "?" + rawQuery } // internalTarget returns the decoded path and the query of a target on // this site: a path, or an absolute URL whose origin is the site URL. func internalTarget(site, target string) (string, string, bool) { u, err := url.Parse(target) if err != nil { return "", "", false } if u.IsAbs() { su, err := url.Parse(site) if site == "" || err != nil || !strings.EqualFold(su.Host, u.Host) || !strings.EqualFold(su.Scheme, u.Scheme) { return "", "", false } } else if !strings.HasPrefix(target, "/") { return "", "", false } p := u.Path if p == "" { p = "/" } return p, u.RawQuery, true } // redirectHopsLogged keeps the URLs whose runtime redirect cycle was // logged, so each is logged once. var redirectHopsLogged sync.Map // logRedirectCycle logs a redirect cycle met at runtime, once per URL. func logRedirectCycle(url string, ch Chain) { if _, seen := redirectHopsLogged.LoadOrStore(url, true); seen { return } log.Warn("golem15.seo: redirect cycle, answering not_found", "url", url, "path", strings.Join(ch.Visited, " → ")) } // RedirectValues are a redirect's values for history. func RedirectValues(r models.Redirect) map[string]any { return map[string]any{ "source": r.Source, "match": r.Match, "target": r.Target, "status": r.Status, "query": r.Query, "ignore_case": r.IgnoreCase, "trailing_slash": r.TrailingSlash, "origin": r.Origin, "note": r.Note, } } // redirectWhat is the history description of a redirect. func (s *Service) redirectWhat(r models.Redirect) string { return s.Phrase("golem15.seo::lang.history.what_redirect", map[string]string{"url": r.Source}) } // samePath reports whether target names path on this site: the path // itself, or an absolute URL on the site URL with that path. func (s *Service) samePath(target, path string) bool { target = strings.TrimSpace(target) if target == path { return true } site := s.SiteURL() return site != "" && target == site+path }