package classes import ( "context" "errors" "net/url" "sort" "strings" "unicode/utf8" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/summercms/modules/cabana" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) // Config keys of the resolver. const ( // SiteURLConfigKey is the absolute origin of the public site (for // example https://example.com). Canonical and other absolute URLs are // built from it, falling back to app.url; never from a request's Host. SiteURLConfigKey = "golem15.seo.site_url" // UnprefixedLocaleConfigKey names the locale whose public URLs carry no // locale prefix; empty leaves the choice to each content type. UnprefixedLocaleConfigKey = "golem15.seo.unprefixed_locale" ) // MaxURLBytes caps the URL the resolver accepts. const MaxURLBytes = 2048 // ErrBadURL is returned for an input that is not a path or an http(s) URL. var ErrBadURL = errors.New("golem15.seo: url must be a path or an absolute http(s) URL") // The actions of a resolve answer. const ( ActionOK = "ok" ActionNotFound = "not_found" ) // The sources of a resolved value (Result.Sources). const ( SourceRule = "rule" SourceRecord = "record" SourceTemplate = "template" SourceGlobal = "global" SourceContent = "content" SourceDefault = "default" ) // Result is the effective SEO of one URL. Action is ok, not_found, // redirect or gone; Status is the HTTP status the frontend should answer // with. URL is the absolute URL of the page (path and kept query on the // site URL). Sources maps each value name (title, description, canonical, // robots, og_title, og_description, og_image) to where it came from. type Result struct { Action string `json:"action"` Status int `json:"status"` // Location is the absolute address a redirect answer sends the visitor // to; absent on every other action. Location string `json:"location,omitempty"` URL string `json:"url"` Locale string `json:"locale"` Staging bool `json:"staging"` SEO *SEO `json:"seo"` // Outbound is how the layer renders links to other sites (P33-16), // present on ok answers. Outbound *Outbound `json:"outbound,omitempty"` Sources map[string]string `json:"-"` // TypeKey and RecordID name the record behind the URL, "" when none. TypeKey string `json:"-"` RecordID string `json:"-"` } // SEO is the head metadata of a resolved page. type SEO struct { Title string `json:"title"` Description string `json:"description"` Canonical string `json:"canonical"` Robots string `json:"robots"` XRobotsTag string `json:"x_robots_tag"` OG OpenGraph `json:"og"` Twitter TwitterTag `json:"twitter"` Verification []MetaTag `json:"verification"` } // Outbound holds the settings' Links rules for links to other sites // (D-83): open them in a new tab, and the domains (subdomains included) // whose links always get nofollow or sponsored. type Outbound struct { NewTab bool `json:"new_tab"` NofollowDomains []string `json:"nofollow_domains"` SponsoredDomains []string `json:"sponsored_domains"` } // OutboundOf is the Links rules of the settings. func OutboundOf(s models.Settings) *Outbound { return &Outbound{ NewTab: s.LinksNewTab, NofollowDomains: validDomains(s.NofollowDomains), SponsoredDomains: validDomains(s.SponsoredDomains), } } // validDomains lists the valid domains of a stored list. func validDomains(list string) []string { out := []string{} for _, d := range models.DomainLines(list) { if models.ValidDomain(d) { out = append(out, d) } } return out } // OpenGraph holds the og: values. type OpenGraph struct { Title string `json:"title"` Description string `json:"description"` Image string `json:"image"` URL string `json:"url"` } // TwitterTag holds the twitter: values. type TwitterTag struct { Card string `json:"card"` Site string `json:"site"` } // MetaTag is one verification meta tag. type MetaTag struct { Name string `json:"name"` Content string `json:"content"` } // Resolver answers the effective SEO of a URL. type Resolver struct { Service *Service // production computes the values as production serves them: the // non-production lock (D-22) is not applied. The sitemap filter decides // on these, so staging lists the same addresses as production. production bool } // Resolver returns the service's resolver. func (s *Service) Resolver() *Resolver { return &Resolver{Service: s} } // SiteURL is the configured public origin without a trailing slash: // golem15.seo.site_url, else app.url, else "". func (s *Service) SiteURL() string { cfg := s.Config() if cfg == nil { return "" } for _, key := range []string{SiteURLConfigKey, "app.url"} { if u := strings.TrimRight(strings.TrimSpace(cfg.String(key)), "/"); u != "" { return u } } return "" } // UnprefixedLocale is the configured unprefixed locale, "" when unset. func (s *Service) UnprefixedLocale() string { cfg := s.Config() if cfg == nil { return "" } return strings.TrimSpace(cfg.String(UnprefixedLocaleConfigKey)) } // Staging reports whether the site runs outside production. It depends on // the environment only (D-22): no setting, admin field or MCP tool can turn // the lock off. func (s *Service) Staging() bool { cfg := s.Config() if cfg == nil { return true } return cfg.Environment() != "production" } // DefaultLocale is the site's default content locale: the translation // writer's default, else the app locale. func (s *Service) DefaultLocale(ctx context.Context, db *gorm.DB) string { if s != nil && s.app != nil { if w, ok := s.app.Lookup[cabana.TranslationWriter](); ok && w != nil { if code, err := w.DefaultLocale(ctx, db); err == nil && code != "" { return code } } } if cfg := s.Config(); cfg != nil { if code := strings.TrimSpace(cfg.String("app.locale")); code != "" { return code } } return "en" } // translatedSetting returns a translatable settings column in locale: the // stored translation when there is one, else the row's own value. func (s *Service) translatedSetting(ctx context.Context, db *gorm.DB, row *models.Settings, field, own, locale string) string { if s == nil || s.app == nil || locale == "" { return own } w, ok := s.app.Lookup[cabana.TranslationWriter]() if !ok || w == nil { return own } if def, err := w.DefaultLocale(ctx, db); err == nil && def == locale { return own } text, found, err := w.TranslatedExact(ctx, db, row, field, locale) if err != nil || !found || strings.TrimSpace(text) == "" { return own } return text } // LoadSettings reads settings row 1; without one the defaults apply. func LoadSettings(db *gorm.DB) (models.Settings, error) { var rows []models.Settings if err := db.Where("id = ?", 1).Limit(1).Find(&rows).Error; err != nil { return models.Settings{}, err } if len(rows) == 0 { return models.DefaultSettings(), nil } return rows[0], nil } // parseTarget turns the resolver input into a path and its query. A path // must start with "/"; an absolute URL must be http or https. Fragments are // dropped. func parseTarget(raw string) (string, url.Values, error) { raw = strings.TrimSpace(raw) if raw == "" || len(raw) > MaxURLBytes { return "", nil, ErrBadURL } u, err := url.Parse(raw) if err != nil { return "", nil, ErrBadURL } if u.IsAbs() { if u.Scheme != "http" && u.Scheme != "https" { return "", nil, ErrBadURL } } else if !strings.HasPrefix(raw, "/") || strings.HasPrefix(raw, "//") { return "", nil, ErrBadURL } path := u.Path if path == "" { path = "/" } query := u.Query() if !safeText(path) || !safeText(u.RawQuery) { return "", nil, ErrBadURL } for k, vs := range query { if !safeText(k) { return "", nil, ErrBadURL } for _, v := range vs { if !safeText(v) { return "", nil, ErrBadURL } } } return path, query, nil } // safeText reports whether s can be stored and compared as Postgres text: // valid UTF-8 without NUL bytes. A decoded path such as /%00 or /%FF is // not (WR-12). func safeText(s string) bool { return utf8.ValidString(s) && !strings.ContainsRune(s, 0) } // contentLocale returns the record's description in locale, else its first // locale. func contentLocale(rec *pact.ContentRecord, locale string) pact.ContentLocale { if rec == nil || len(rec.Locales) == 0 { return pact.ContentLocale{Locale: locale} } for _, l := range rec.Locales { if l.Locale == locale { return l } } return rec.Locales[0] } // matchingRules returns the URL rules that apply to path and query, the // most specific first: exact rules, then patterns by the length of their // address. A rule address without "?" is compared with the path only; one // with a query with the path and the raw query. func matchingRules(db *gorm.DB, path, rawQuery string) ([]models.Rule, error) { var rows []models.Rule if err := db.Where("match = ? OR url = ? OR url = ?", models.MatchPattern, path, path+"?"+rawQuery).Find(&rows).Error; err != nil { return nil, err } full := path if rawQuery != "" { full = path + "?" + rawQuery } var out []models.Rule for _, r := range rows { target := path if strings.Contains(r.URL, "?") { target = full } if r.Match == models.MatchPattern { if MatchGlob(r.URL, target) { out = append(out, r) } } else if r.URL == target { out = append(out, r) } } sort.SliceStable(out, func(i, j int) bool { if (out[i].Match == models.MatchExact) != (out[j].Match == models.MatchExact) { return out[i].Match == models.MatchExact } return len(out[i].URL) > len(out[j].URL) }) return out, nil } // ruleValue returns the first non-empty value of column among rules. func ruleValue(rules []models.Rule, column string) string { for _, r := range rules { if v := strings.TrimSpace(r.Value(column)); v != "" { return v } } return "" } // pick returns the first non-empty candidate and its source. type candidate struct { value string source string } func pick(cands ...candidate) (string, string) { for _, c := range cands { if strings.TrimSpace(c.value) != "" { return strings.TrimSpace(c.value), c.source } } return "", SourceDefault } // robotsLayer is one layer of the robots precedence chain. type robotsLayer struct { value string source string } // resolveRobots merges the layers per directive (UI-SPEC C5): each // directive comes from the first layer that sets it, index,follow last. // The source is the layer that decided indexing. A malformed stored value // is skipped. func resolveRobots(layers []robotsLayer) (Robots, string) { var out Robots source := SourceDefault for _, layer := range layers { r, err := ParseRobots(layer.value) if err != nil || r.Empty() { continue } if out.Index == "" && r.Index != "" { source = layer.source } out = out.Inherit(r) } return out.Inherit(Robots{Index: "index", Follow: "follow"}), source } // Resolve answers rawURL, a path or an absolute URL of the site: the // redirect decision first, then the effective SEO (D-20a, D-103, P-21). // The pipeline is: // 1. normalisation (D-20b): every address rule at once; // 2. redirects on the normalised path, followed to the final target // (a 410 answers gone, a cycle not_found); // 3. strict URLs (D-20c): a path that names a record without being its // address 301s to the record's address; // 4. the effective SEO with the D-02 precedence per value (P-14): URL rule // > record > per-type template or global settings > content-derived // values. // // When normalisation changed the URL or a redirect matched, the answer's // location is the final address, so a visitor makes one hop. A path with // neither a published record nor a URL rule answers not_found, except the // home page. Outside production every answer is noindex,nofollow with // staging true (D-22). Absolute URLs come from SiteURL, or from the // request's own URL when that is the site's host or its www twin; never // from a request Host header. func (r *Resolver) Resolve(ctx context.Context, rawURL string) (Result, error) { path, query, err := parseTarget(rawURL) if err != nil { return Result{}, err } rawQuery := rawQueryOf(rawURL) svc := r.Service site := svc.SiteURL() db, err := svc.DB(ctx) if err != nil { return Result{}, err } settings, err := LoadSettings(db) if err != nil { return Result{}, err } opts := NormalizeOptionsOf(settings) self := site + joinQuery(path, rawQuery) // 1. Normalisation. An absolute URL of the site (or its www twin) // keeps its scheme and host for the scheme and host rules; any other // host is ignored. in := &url.URL{Path: path, RawQuery: rawQuery} if u, err := url.Parse(strings.TrimSpace(rawURL)); err == nil && u.IsAbs() && sameSite(site, u.Host) { in.Scheme, in.Host = u.Scheme, u.Host } norm, changed := Normalize(in, opts) origin := site if norm.Host != "" { origin = norm.Scheme + "://" + norm.Host } npath, nquery := norm.Path, norm.RawQuery // 2. Redirects on the normalised path. set, err := LoadRedirectSet(db) if err != nil { return Result{}, err } chain, err := set.Follow(site, npath, nquery, MaxRedirectHops) if err != nil { return Result{}, err } if len(chain.Hops) > 0 { if chain.Loop || chain.Gone { return r.redirectAnswer(self, chain), nil } location := chain.Final if fpath, fquery, internal := internalTarget(site, chain.Final); internal { final, _ := Normalize(&url.URL{Path: fpath, RawQuery: fquery}, opts) fpath = final.Path st, err := r.Strict(ctx, fpath, opts) if err != nil { return Result{}, err } if st.Key != "" && st.Mismatch && st.Canonical != "" { fpath = st.Canonical } location = origin + joinQuery(escapePath(fpath), final.RawQuery) } res := Result{Action: ActionRedirect, Status: chain.Status, Location: location, URL: self, Staging: svc.Staging(), Sources: map[string]string{}} return res, nil } // 3. Strict URLs. st, err := r.Strict(ctx, npath, opts) if err != nil { return Result{}, err } if st.Key != "" && st.Mismatch { if st.Canonical == "" { return Result{Action: ActionNotFound, Status: 404, URL: self, Staging: svc.Staging(), Sources: map[string]string{}}, nil } return r.moved(self, origin+joinQuery(escapePath(st.Canonical), nquery)), nil } if changed { return r.moved(self, origin+joinQuery(escapePath(npath), nquery)), nil } // 4. The effective SEO. return r.resolveAt(ctx, db, settings, npath, query, nquery, st.Key, st.Match, st.Record, false) } // moved is a 301 answer to location. func (r *Resolver) moved(self, location string) Result { return Result{Action: ActionRedirect, Status: int(models.Status301), Location: location, URL: self, Staging: r.Service.Staging(), Sources: map[string]string{}} } // escapePath percent-encodes a decoded path for a Location. func escapePath(p string) string { return (&url.URL{Path: p}).EscapedPath() } // ResolveRecord answers the effective SEO of a record in one locale at its // own address, published or not (the SEO tab's effective-values partial). // ok is false when the record has no address in that locale. func (r *Resolver) ResolveRecord(ctx context.Context, key, id, locale string) (Result, bool, error) { svc := r.Service reg := svc.Registry() if reg == nil { return Result{}, false, nil } rec, err := reg.Describe(ctx, key, id, svc.UnprefixedLocale()) if err != nil || rec == nil { return Result{}, false, err } var path string for _, l := range rec.Locales { if l.Locale == locale { path = l.URL } } if strings.TrimSpace(path) == "" { return Result{}, false, nil } target, query, err := parseTarget(path) if err != nil { return Result{}, false, nil } db, err := svc.DB(ctx) if err != nil { return Result{}, false, err } settings, err := LoadSettings(db) if err != nil { return Result{}, false, err } res, err := r.resolveAt(ctx, db, settings, target, query, rawQueryOf(path), key, &pact.ContentMatchResult{ID: id, Locale: locale, Exact: true}, rec, true) return res, err == nil, err } // resolveAt computes the answer for a path whose owner lookup is done: // key, match and rec name the record behind it (key "" when none). func (r *Resolver) resolveAt(ctx context.Context, db *gorm.DB, settings models.Settings, path string, query url.Values, rawQuery, key string, match *pact.ContentMatchResult, rec *pact.ContentRecord, force bool) (Result, error) { svc := r.Service site := svc.SiteURL() kept, filtered := splitQuery(settings.Params(), path, query) self := site + path if q := encodeQuery(kept); q != "" { self += "?" + q } res := Result{Action: ActionNotFound, Status: 404, URL: self, Staging: svc.Staging(), Sources: map[string]string{}} rules, err := matchingRules(db, path, rawQuery) if err != nil { return Result{}, err } home := path == "/" if key == "" && len(rules) == 0 && !home && !force { return res, nil } locale := "" var entry models.Entry var content pact.ContentLocale typeSetting := models.TypeSetting{Inject: true} if key != "" { typ, _ := svc.Registry().ContentType(key) locale = match.Locale if entry, _, err = LoadEntry(db, typ.Content.Model, match.ID, locale); err != nil { return Result{}, err } content = contentLocale(rec, locale) if typeSetting, err = LoadTypeSetting(db, key); err != nil { return Result{}, err } res.TypeKey, res.RecordID = key, match.ID } if locale == "" { locale = svc.DefaultLocale(ctx, db) } siteName := svc.translatedSetting(ctx, db, &settings, "site_name", settings.SiteName, locale) defaultDescription := svc.translatedSetting(ctx, db, &settings, "default_description", settings.DefaultDescription, locale) vars := TemplateVars{Title: content.Title, Description: content.Description, Site: siteName} seo := &SEO{Verification: []MetaTag{}} src := res.Sources // Title: rule > record > type template, else the general template, on // the content title > content title > site name. The home page without // a record uses the home template. var titleTemplate candidate switch { case key != "": tpl := typeSetting.TitleTemplate if strings.TrimSpace(tpl) == "" { tpl = settings.TitleTemplate } titleTemplate = candidate{RenderTemplate(tpl, vars), SourceTemplate} case home: titleTemplate = candidate{RenderTemplate(settings.HomeTemplate, vars), SourceTemplate} } seo.Title, src["title"] = pick( candidate{ruleValue(rules, "title"), SourceRule}, candidate{entry.Title, SourceRecord}, titleTemplate, candidate{content.Title, SourceContent}, candidate{siteName, SourceGlobal}, ) // Description: rule > record > type template > content > settings. var descTemplate candidate if key != "" { descTemplate = candidate{RenderTemplate(typeSetting.DescriptionTemplate, vars), SourceTemplate} } seo.Description, src["description"] = pick( candidate{ruleValue(rules, "description"), SourceRule}, candidate{entry.Description, SourceRecord}, descTemplate, candidate{content.Description, SourceContent}, candidate{defaultDescription, SourceGlobal}, ) // Canonical: rule > record > self. canonical, canonicalSource := pick( candidate{ruleValue(rules, "canonical"), SourceRule}, candidate{entry.Canonical, SourceRecord}, ) if canonical == "" { canonical, canonicalSource = self, SourceDefault } else { canonical = AbsoluteURL(site, canonical) } seo.Canonical, src["canonical"] = canonical, canonicalSource // Robots per directive: rule > record > never-indexed paths, search // paths and filter URLs > type default > index,follow. layers := []robotsLayer{ {ruleValue(rules, "robots"), SourceRule}, {entry.Robots, SourceRecord}, } switch { case MatchPathList(settings.NoindexPaths, path): layers = append(layers, robotsLayer{"noindex,nofollow", SourceGlobal}) case MatchPathList(settings.SearchPaths, path) || filtered: layers = append(layers, robotsLayer{"noindex,follow", SourceGlobal}) } if key != "" { layers = append(layers, robotsLayer{typeSetting.Robots, SourceTemplate}) } robots, robotsSource := resolveRobots(layers) if res.Staging && !r.production { robots, robotsSource = Robots{Index: "noindex", Follow: "nofollow"}, SourceDefault } seo.Robots, src["robots"] = robots.Header(), robotsSource seo.XRobotsTag = robots.Header() // Social values (D-04): overrides, else the effective title and // description; the image falls back to the content's, then the // settings' default social image. seo.OG.Title, src["og_title"] = pick( candidate{ruleValue(rules, "og_title"), SourceRule}, candidate{entry.OGTitle, SourceRecord}, candidate{seo.Title, src["title"]}, ) seo.OG.Description, src["og_description"] = pick( candidate{ruleValue(rules, "og_description"), SourceRule}, candidate{entry.OGDescription, SourceRecord}, candidate{seo.Description, src["description"]}, ) image, imageSource := pick( candidate{ruleValue(rules, "og_image"), SourceRule}, candidate{entry.OGImage, SourceRecord}, candidate{content.Image, SourceContent}, candidate{settings.DefaultOGImage, SourceGlobal}, ) if image != "" { image = AbsoluteURL(site, image) } seo.OG.Image, src["og_image"] = image, imageSource seo.OG.URL = seo.Canonical seo.Twitter.Card = "summary" if image != "" { seo.Twitter.Card = "summary_large_image" } seo.Twitter.Site = strings.TrimSpace(settings.TwitterSite) // Verification meta (D-05) on the home page only. if home { seo.Verification = verificationMeta(settings) } res.Action, res.Status, res.Locale, res.SEO = ActionOK, 200, locale, seo res.Outbound = OutboundOf(settings) return res, nil } // redirectAnswer is the answer of a redirect chain that leads nowhere: // gone for a chain that ends on a 410, not_found (logged once per URL) for // a cycle (P-22). func (r *Resolver) redirectAnswer(self string, ch Chain) Result { res := Result{URL: self, Staging: r.Service.Staging(), Sources: map[string]string{}} if ch.Gone && !ch.Loop { res.Action, res.Status = ActionGone, int(models.Status410) return res } logRedirectCycle(self, ch) res.Action, res.Status = ActionNotFound, 404 return res } // rawQueryOf returns the raw query of the resolver input. func rawQueryOf(raw string) string { u, err := url.Parse(strings.TrimSpace(raw)) if err != nil { return "" } return u.RawQuery } // verificationMeta lists the verification meta tags of the settings. func verificationMeta(s models.Settings) []MetaTag { out := []MetaTag{} for _, m := range []MetaTag{ {Name: "google-site-verification", Content: s.VerifyGoogle}, {Name: "msvalidate.01", Content: s.VerifyBing}, {Name: "yandex-verification", Content: s.VerifyYandex}, } { if strings.TrimSpace(m.Content) != "" { m.Content = strings.TrimSpace(m.Content) out = append(out, m) } } for _, m := range s.OtherMeta() { if strings.TrimSpace(m.Name) != "" && strings.TrimSpace(m.Content) != "" { out = append(out, MetaTag{Name: strings.TrimSpace(m.Name), Content: strings.TrimSpace(m.Content)}) } } return out }