feat: URL normalisation, strict URLs and the one-hop redirect decision
- Normalize: https, www or bare host, double slashes and index files, trailing slash add/remove, optional ASCII lowercasing (defaults change nothing but scheme, slashes and index files) - strict URLs through signpost MatchPath: a path that names a record without being its address 301s to the record's address - resolve runs normalisation, redirects, strict URLs, then SEO; the location is the final address, so a visitor makes one hop - P1 warns when a record's canonical URL redirects
This commit is contained in:
155
classes/normalize.go
Normal file
155
classes/normalize.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/sm-seo-plugin/models"
|
||||
)
|
||||
|
||||
// NormalizeOptions are a site's address rules (D-20b, settings tab
|
||||
// Addresses): Host is keep, www or bare; Slash is keep, add or remove;
|
||||
// Lowercase lowercases the ASCII letters of the path. The defaults (keep,
|
||||
// keep, off) change nothing but scheme, double slashes and index files
|
||||
// (UI-SPEC Q-7).
|
||||
type NormalizeOptions struct {
|
||||
Host string
|
||||
Slash string
|
||||
Lowercase bool
|
||||
}
|
||||
|
||||
// NormalizeOptionsOf reads the address rules of the settings.
|
||||
func NormalizeOptionsOf(s models.Settings) NormalizeOptions {
|
||||
return NormalizeOptions{Host: s.URLHost, Slash: s.URLSlash, Lowercase: s.URLLowercase}
|
||||
}
|
||||
|
||||
// indexFiles are the directory index names collapsed into their directory.
|
||||
var indexFiles = []string{"/index.html", "/index.htm", "/index.php"}
|
||||
|
||||
// Normalize returns u in the site's address form and whether anything
|
||||
// changed (D-20b). It applies every rule at once, so any other form reaches
|
||||
// the final one in one hop:
|
||||
// - an absolute URL gets https, and its host gains or loses "www." per
|
||||
// opts.Host (keep leaves it);
|
||||
// - repeated slashes collapse and a trailing /index.html (.htm, .php)
|
||||
// collapses into its directory;
|
||||
// - with opts.Lowercase the ASCII letters of the decoded path are
|
||||
// lowercased; other letters (Ł) and the percent-encoding are kept, so
|
||||
// %C5%81 and %c5%81 stay the same address (RESEARCH Pitfall 13);
|
||||
// - opts.Slash add appends a trailing slash to a path whose last segment
|
||||
// has no file extension, remove drops it; the root / never changes.
|
||||
//
|
||||
// The query is kept as it is and a fragment is dropped. u is not modified.
|
||||
func Normalize(u *url.URL, opts NormalizeOptions) (*url.URL, bool) {
|
||||
if u == nil {
|
||||
return nil, false
|
||||
}
|
||||
out := *u
|
||||
out.Fragment, out.RawFragment = "", ""
|
||||
changed := false
|
||||
|
||||
if out.Host != "" {
|
||||
if out.Scheme != "https" {
|
||||
out.Scheme, changed = "https", true
|
||||
}
|
||||
host := strings.ToLower(out.Host)
|
||||
switch opts.Host {
|
||||
case models.HostWWW:
|
||||
if !strings.HasPrefix(host, "www.") {
|
||||
host = "www." + host
|
||||
}
|
||||
case models.HostBare:
|
||||
host = strings.TrimPrefix(host, "www.")
|
||||
}
|
||||
if host != strings.ToLower(out.Host) {
|
||||
changed = true
|
||||
}
|
||||
out.Host = host
|
||||
}
|
||||
|
||||
p := out.Path
|
||||
if p == "" {
|
||||
p = "/"
|
||||
}
|
||||
clean := collapseSlashes(p)
|
||||
for _, index := range indexFiles {
|
||||
if strings.HasSuffix(strings.ToLower(clean), index) {
|
||||
clean = clean[:len(clean)-len(index)]
|
||||
if clean == "" {
|
||||
clean = "/"
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if opts.Lowercase {
|
||||
clean = lowerASCII(clean)
|
||||
}
|
||||
if clean != "/" {
|
||||
switch opts.Slash {
|
||||
case models.SlashAdd:
|
||||
if !strings.HasSuffix(clean, "/") && !strings.Contains(lastSegment(clean), ".") {
|
||||
clean += "/"
|
||||
}
|
||||
case models.SlashDrop:
|
||||
clean = strings.TrimRight(clean, "/")
|
||||
if clean == "" {
|
||||
clean = "/"
|
||||
}
|
||||
}
|
||||
}
|
||||
if clean != p {
|
||||
changed = true
|
||||
}
|
||||
out.Path, out.RawPath = clean, ""
|
||||
return &out, changed
|
||||
}
|
||||
|
||||
// collapseSlashes replaces every run of slashes with one.
|
||||
func collapseSlashes(p string) string {
|
||||
if !strings.Contains(p, "//") {
|
||||
return p
|
||||
}
|
||||
var b strings.Builder
|
||||
prev := false
|
||||
for i := 0; i < len(p); i++ {
|
||||
if p[i] == '/' {
|
||||
if prev {
|
||||
continue
|
||||
}
|
||||
prev = true
|
||||
} else {
|
||||
prev = false
|
||||
}
|
||||
b.WriteByte(p[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// lowerASCII lowercases the ASCII letters of s only.
|
||||
func lowerASCII(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + ('a' - 'A')
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// lastSegment is the part of p after its last slash.
|
||||
func lastSegment(p string) string {
|
||||
if i := strings.LastIndex(p, "/"); i >= 0 {
|
||||
return p[i+1:]
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// sameSite reports whether host is the site URL's host or its www twin.
|
||||
func sameSite(site, host string) bool {
|
||||
su, err := url.Parse(site)
|
||||
if err != nil || su.Host == "" || host == "" {
|
||||
return false
|
||||
}
|
||||
bare := func(h string) string { return strings.TrimPrefix(strings.ToLower(h), "www.") }
|
||||
return bare(su.Host) == bare(host)
|
||||
}
|
||||
@@ -220,24 +220,31 @@ func (s *Service) ruleCallout(ctx context.Context, rules []models.Rule) (Callout
|
||||
return Callout{}, false
|
||||
}
|
||||
|
||||
// canonicalCallout warns when a canonical on this site resolves to a page
|
||||
// served noindex. Outside production every page is noindex, so it says
|
||||
// nothing there.
|
||||
// canonicalCallout warns about a canonical on this site (D-06): one that
|
||||
// redirects (status.canonical_redirected, naming the final address), or
|
||||
// one that resolves to a page served noindex (status.canonical_noindex).
|
||||
// Outside production every page is noindex, so only the redirect warning
|
||||
// applies there.
|
||||
func (s *Service) canonicalCallout(ctx context.Context, canonical string) (Callout, bool) {
|
||||
canonical = strings.TrimSpace(canonical)
|
||||
if canonical == "" || s.Staging() {
|
||||
if canonical == "" {
|
||||
return Callout{}, false
|
||||
}
|
||||
target := canonical
|
||||
if u, err := url.Parse(canonical); err == nil && u.IsAbs() {
|
||||
site, err := url.Parse(s.SiteURL())
|
||||
if err != nil || site.Host == "" || !strings.EqualFold(site.Host, u.Host) {
|
||||
if !sameSite(s.SiteURL(), u.Host) {
|
||||
return Callout{}, false
|
||||
}
|
||||
target = u.RequestURI()
|
||||
target = u.String()
|
||||
}
|
||||
res, err := s.Resolver().Resolve(ctx, target)
|
||||
if err != nil || res.SEO == nil || !strings.Contains(res.SEO.Robots, "noindex") {
|
||||
if err != nil {
|
||||
return Callout{}, false
|
||||
}
|
||||
if res.Action == ActionRedirect {
|
||||
return Callout{Title: s.PhraseCtx(ctx, "golem15.seo::lang.status.canonical_redirected", map[string]string{"url": canonical, "target": res.Location})}, true
|
||||
}
|
||||
if s.Staging() || res.SEO == nil || !strings.Contains(res.SEO.Robots, "noindex") {
|
||||
return Callout{}, false
|
||||
}
|
||||
return Callout{Title: s.PhraseCtx(ctx, "golem15.seo::lang.status.canonical_noindex", map[string]string{"url": canonical})}, true
|
||||
|
||||
@@ -218,26 +218,6 @@ func parseTarget(raw string) (string, url.Values, error) {
|
||||
return path, u.Query(), nil
|
||||
}
|
||||
|
||||
// lookup finds the published record behind path; key is "" when none.
|
||||
func (r *Resolver) lookup(ctx context.Context, path string) (key string, match *pact.ContentMatchResult, rec *pact.ContentRecord, err error) {
|
||||
reg := r.Service.Registry()
|
||||
if reg == nil {
|
||||
return "", nil, nil, nil
|
||||
}
|
||||
key, match, err = reg.MatchPath(ctx, path, r.Service.UnprefixedLocale())
|
||||
if err != nil || match == nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
rec, err = reg.Describe(ctx, key, match.ID, r.Service.UnprefixedLocale())
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
if rec == nil || !rec.Published {
|
||||
return "", nil, nil, nil
|
||||
}
|
||||
return key, match, rec, nil
|
||||
}
|
||||
|
||||
// contentLocale returns the record's description in locale, else its first
|
||||
// locale.
|
||||
func contentLocale(rec *pact.ContentRecord, locale string) pact.ContentLocale {
|
||||
@@ -339,19 +319,33 @@ func resolveRobots(layers []robotsLayer) (Robots, string) {
|
||||
return out.Inherit(Robots{Index: "index", Follow: "follow"}), source
|
||||
}
|
||||
|
||||
// Resolve answers the effective SEO of rawURL, a path or an absolute URL of
|
||||
// the site, with the D-02 precedence per value (P-14): URL rule > record >
|
||||
// per-type template or global settings > content-derived values. A path
|
||||
// with neither a published record nor a URL rule answers not_found, except
|
||||
// the home page. Outside production every answer is noindex,nofollow with
|
||||
// staging true (D-22). Absolute URLs come from SiteURL, never from a
|
||||
// request.
|
||||
// Resolve answers rawURL, a path or an absolute URL of the site: the
|
||||
// redirect decision first, then the effective SEO (D-20a, D-103, P-21).
|
||||
// The pipeline is:
|
||||
// 1. normalisation (D-20b): every address rule at once;
|
||||
// 2. redirects on the normalised path, followed to the final target
|
||||
// (a 410 answers gone, a cycle not_found);
|
||||
// 3. strict URLs (D-20c): a path that names a record without being its
|
||||
// address 301s to the record's address;
|
||||
// 4. the effective SEO with the D-02 precedence per value (P-14): URL rule
|
||||
// > record > per-type template or global settings > content-derived
|
||||
// values.
|
||||
//
|
||||
// When normalisation changed the URL or a redirect matched, the answer's
|
||||
// location is the final address, so a visitor makes one hop. A path with
|
||||
// neither a published record nor a URL rule answers not_found, except the
|
||||
// home page. Outside production every answer is noindex,nofollow with
|
||||
// staging true (D-22). Absolute URLs come from SiteURL, or from the
|
||||
// request's own URL when that is the site's host or its www twin; never
|
||||
// from a request Host header.
|
||||
func (r *Resolver) Resolve(ctx context.Context, rawURL string) (Result, error) {
|
||||
path, query, err := parseTarget(rawURL)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
rawQuery := rawQueryOf(rawURL)
|
||||
svc := r.Service
|
||||
site := svc.SiteURL()
|
||||
db, err := svc.DB(ctx)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -360,22 +354,80 @@ func (r *Resolver) Resolve(ctx context.Context, rawURL string) (Result, error) {
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
opts := NormalizeOptionsOf(settings)
|
||||
self := site + joinQuery(path, rawQuery)
|
||||
|
||||
// 1. Normalisation. An absolute URL of the site (or its www twin)
|
||||
// keeps its scheme and host for the scheme and host rules; any other
|
||||
// host is ignored.
|
||||
in := &url.URL{Path: path, RawQuery: rawQuery}
|
||||
if u, err := url.Parse(strings.TrimSpace(rawURL)); err == nil && u.IsAbs() && sameSite(site, u.Host) {
|
||||
in.Scheme, in.Host = u.Scheme, u.Host
|
||||
}
|
||||
norm, changed := Normalize(in, opts)
|
||||
origin := site
|
||||
if norm.Host != "" {
|
||||
origin = norm.Scheme + "://" + norm.Host
|
||||
}
|
||||
npath, nquery := norm.Path, norm.RawQuery
|
||||
|
||||
// 2. Redirects on the normalised path.
|
||||
set, err := LoadRedirectSet(db)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
chain, err := set.Follow(svc.SiteURL(), path, rawQueryOf(rawURL), MaxRedirectHops)
|
||||
chain, err := set.Follow(site, npath, nquery, MaxRedirectHops)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if len(chain.Hops) > 0 {
|
||||
return r.redirectAnswer(svc.SiteURL()+joinQuery(path, rawQueryOf(rawURL)), chain), nil
|
||||
if chain.Loop || chain.Gone {
|
||||
return r.redirectAnswer(self, chain), nil
|
||||
}
|
||||
location := chain.Final
|
||||
if fpath, fquery, internal := internalTarget(site, chain.Final); internal {
|
||||
final, _ := Normalize(&url.URL{Path: fpath, RawQuery: fquery}, opts)
|
||||
fpath = final.Path
|
||||
st, err := r.Strict(ctx, fpath, opts)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if st.Key != "" && st.Mismatch && st.Canonical != "" {
|
||||
fpath = st.Canonical
|
||||
}
|
||||
location = origin + joinQuery(escapePath(fpath), final.RawQuery)
|
||||
}
|
||||
res := Result{Action: ActionRedirect, Status: chain.Status, Location: location, URL: self, Staging: svc.Staging(), Sources: map[string]string{}}
|
||||
return res, nil
|
||||
}
|
||||
key, match, rec, err := r.lookup(ctx, path)
|
||||
|
||||
// 3. Strict URLs.
|
||||
st, err := r.Strict(ctx, npath, opts)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return r.resolveAt(ctx, db, settings, path, query, rawQueryOf(rawURL), key, match, rec, false)
|
||||
if st.Key != "" && st.Mismatch {
|
||||
if st.Canonical == "" {
|
||||
return Result{Action: ActionNotFound, Status: 404, URL: self, Staging: svc.Staging(), Sources: map[string]string{}}, nil
|
||||
}
|
||||
return r.moved(self, origin+joinQuery(escapePath(st.Canonical), nquery)), nil
|
||||
}
|
||||
if changed {
|
||||
return r.moved(self, origin+joinQuery(escapePath(npath), nquery)), nil
|
||||
}
|
||||
|
||||
// 4. The effective SEO.
|
||||
return r.resolveAt(ctx, db, settings, npath, query, nquery, st.Key, st.Match, st.Record, false)
|
||||
}
|
||||
|
||||
// moved is a 301 answer to location.
|
||||
func (r *Resolver) moved(self, location string) Result {
|
||||
return Result{Action: ActionRedirect, Status: int(models.Status301), Location: location, URL: self, Staging: r.Service.Staging(), Sources: map[string]string{}}
|
||||
}
|
||||
|
||||
// escapePath percent-encodes a decoded path for a Location.
|
||||
func escapePath(p string) string {
|
||||
return (&url.URL{Path: p}).EscapedPath()
|
||||
}
|
||||
|
||||
// ResolveRecord answers the effective SEO of a record in one locale at its
|
||||
@@ -571,21 +623,17 @@ func (r *Resolver) resolveAt(ctx context.Context, db *gorm.DB, settings models.S
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// redirectAnswer is the answer of a matched redirect chain (P-21): gone
|
||||
// for a chain that ends on a 410, not_found (logged once per URL) for a
|
||||
// cycle, else redirect with the chain's status and its final target
|
||||
// absolute on the site URL, so the visitor makes one hop.
|
||||
// redirectAnswer is the answer of a redirect chain that leads nowhere:
|
||||
// gone for a chain that ends on a 410, not_found (logged once per URL) for
|
||||
// a cycle (P-22).
|
||||
func (r *Resolver) redirectAnswer(self string, ch Chain) Result {
|
||||
res := Result{URL: self, Staging: r.Service.Staging(), Sources: map[string]string{}}
|
||||
switch {
|
||||
case ch.Loop:
|
||||
logRedirectCycle(self, ch)
|
||||
res.Action, res.Status = ActionNotFound, 404
|
||||
case ch.Gone:
|
||||
if ch.Gone && !ch.Loop {
|
||||
res.Action, res.Status = ActionGone, int(models.Status410)
|
||||
default:
|
||||
res.Action, res.Status, res.Location = ActionRedirect, ch.Status, AbsoluteURL(r.Service.SiteURL(), ch.Final)
|
||||
return res
|
||||
}
|
||||
logRedirectCycle(self, ch)
|
||||
res.Action, res.Status = ActionNotFound, 404
|
||||
return res
|
||||
}
|
||||
|
||||
|
||||
68
classes/strict.go
Normal file
68
classes/strict.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/pact"
|
||||
)
|
||||
|
||||
// StrictMatch is the record behind a path and whether the path is its
|
||||
// address (D-20c). Key is "" when no published record answers the path.
|
||||
type StrictMatch struct {
|
||||
Key string
|
||||
Match *pact.ContentMatchResult
|
||||
Record *pact.ContentRecord
|
||||
// Canonical is the record's own path in the matched locale, in the
|
||||
// site's address form; "" when the descriptor answers none.
|
||||
Canonical string
|
||||
// Mismatch is true when the path names the record but is not its
|
||||
// address: a wrong category or parent, a locale prefix with another
|
||||
// locale's slug, a moved record. It then 301s to Canonical.
|
||||
Mismatch bool
|
||||
}
|
||||
|
||||
// Strict finds the published record behind path through the signpost
|
||||
// patterns (MatchPath) and checks every segment against the record's own
|
||||
// address (D-20c): a path that names a record without being its address
|
||||
// is a mismatch, so a wrong URL never answers ok. The comparison uses the
|
||||
// site's address form (opts), so a strict 301 and normalisation never
|
||||
// send a visitor back and forth.
|
||||
func (r *Resolver) Strict(ctx context.Context, path string, opts NormalizeOptions) (StrictMatch, error) {
|
||||
var out StrictMatch
|
||||
reg := r.Service.Registry()
|
||||
if reg == nil {
|
||||
return out, nil
|
||||
}
|
||||
unprefixed := r.Service.UnprefixedLocale()
|
||||
// Content patterns carry no trailing slash; the site's trailing-slash
|
||||
// rule is applied to the record's address below.
|
||||
key, match, err := reg.MatchPath(ctx, trimSlash(path), unprefixed)
|
||||
if err != nil || match == nil {
|
||||
return out, err
|
||||
}
|
||||
rec, err := reg.Describe(ctx, key, match.ID, unprefixed)
|
||||
if err != nil || rec == nil || !rec.Published {
|
||||
return out, err
|
||||
}
|
||||
out.Key, out.Match, out.Record = key, match, rec
|
||||
described := ""
|
||||
for _, l := range rec.Locales {
|
||||
if l.Locale == match.Locale {
|
||||
described = strings.TrimSpace(l.URL)
|
||||
}
|
||||
}
|
||||
if described == "" {
|
||||
out.Mismatch = !match.Exact
|
||||
return out, nil
|
||||
}
|
||||
du, err := url.Parse(described)
|
||||
if err != nil {
|
||||
return out, nil
|
||||
}
|
||||
canonical, _ := Normalize(&url.URL{Path: du.Path}, opts)
|
||||
out.Canonical = canonical.Path
|
||||
out.Mismatch = !match.Exact || canonical.Path != path
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user