package classes import ( "context" "net/url" "strings" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/sm-sitemap-plugin/contract" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) // SitemapFilter is the SEO plugin's contract.Filter for the sitemap plugin // (D-95, D-24): an address whose effective SEO is noindex, whose canonical // URL points elsewhere, or that redirects leaves the XML and HTML sitemaps // with the reason noindex, canonical or redirect. It decides on the // production values: the non-production lock (every page noindex outside // production, D-22) never excludes an address, so staging and production // list the same set. Service is asked on every call, because the filter is // published in Register, before the service exists. type SitemapFilter struct { Service func() *Service } var _ contract.Filter = SitemapFilter{} // filterBatch is what one Decide call reads once for all its candidates. type filterBatch struct { svc *Service resolver *Resolver db *gorm.DB settings models.Settings opts NormalizeOptions set *RedirectSet site string } // Decide answers one Decision per candidate, in order. Without a booted // service every address stays listed. func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ([]contract.Decision, error) { out := make([]contract.Decision, len(batch)) var svc *Service if f.Service != nil { svc = f.Service() } if svc == nil || len(batch) == 0 { return out, nil } b, err := svc.newFilterBatch(ctx) if err != nil { return nil, err } var pageIDs []string for _, c := range batch { if c.TypeKey == PageTypeKey { pageIDs = append(pageIDs, c.RecordID) } } kinds, err := pageKinds(b.db, pageIDs) if err != nil { return nil, err } for i, c := range batch { d, err := b.decide(ctx, c) if err != nil { return nil, err } if c.TypeKey == PageTypeKey && !d.Exclude { d.Group = PageGroup(kinds[c.RecordID]) } out[i] = d } return out, nil } // Groups lists the file groups the decisions name: the registry's page // kinds (pages, offers, services, landing-pages), titled in every locale // of the site with the kind's name (P33-18). func (f SitemapFilter) Groups(ctx context.Context) ([]contract.Group, error) { var svc *Service if f.Service != nil { svc = f.Service() } if svc == nil { return nil, nil } db, err := svc.DB(ctx) if err != nil { return nil, err } locales := svc.Locales(ctx, db) tr := svc.translator() out := make([]contract.Group, 0, len(models.PageKinds)) for _, kind := range models.PageKinds { titles := map[string]string{} key := PageKindLabels[kind] for _, locale := range locales { if tr == nil { titles[locale] = key continue } titles[locale] = tr.GetIn(locale, key, nil) } out = append(out, contract.Group{Key: PageGroup(kind), Titles: titles}) } return out, nil } // newFilterBatch reads the settings and the redirects once for a batch. func (s *Service) newFilterBatch(ctx context.Context) (*filterBatch, error) { db, err := s.DB(ctx) if err != nil { return nil, err } settings, err := LoadSettings(db) if err != nil { return nil, err } set, err := LoadRedirectSet(db) if err != nil { return nil, err } return &filterBatch{ svc: s, resolver: &Resolver{Service: s, production: true}, db: db, settings: settings, opts: NormalizeOptionsOf(settings), set: set, site: s.SiteURL(), }, nil } // decide is the decision of one candidate. func (b *filterBatch) decide(ctx context.Context, c contract.Candidate) (contract.Decision, error) { d, _, err := b.decideTarget(ctx, c) return d, err } // decideTarget is the decision of one candidate and, for a canonical or a // redirect, the address the sitemaps would list instead: a redirected // address first (it is no page), then noindex, then a canonical URL // elsewhere. func (b *filterBatch) decideTarget(ctx context.Context, c contract.Candidate) (contract.Decision, string, error) { path, query, rawQuery, ok := b.candidatePath(c.URL) if !ok { return contract.Decision{}, "", nil } norm, changed := Normalize(&url.URL{Path: path, RawQuery: rawQuery}, b.opts) if changed { return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, joinQuery(norm.Path, norm.RawQuery), nil } chain, err := b.set.Follow(b.site, norm.Path, norm.RawQuery, MaxRedirectHops) if err != nil { return contract.Decision{}, "", err } if len(chain.Hops) > 0 { return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, displayURL(b.site, chain.Final), nil } key := "" var match *pact.ContentMatchResult if reg := b.svc.Registry(); reg != nil && c.RecordID != "" { if typ, ok := reg.ContentType(c.TypeKey); ok && typ.Content != nil { key = c.TypeKey match = &pact.ContentMatchResult{ID: c.RecordID, Locale: c.Locale, Exact: true} } } res, err := b.resolver.resolveAt(ctx, b.db, b.settings, path, query, rawQuery, key, match, nil, true) if err != nil { return contract.Decision{}, "", err } if res.SEO == nil { return contract.Decision{}, "", nil } if r, err := ParseRobots(res.SEO.Robots); err == nil && r.Index == "noindex" { return contract.Decision{Exclude: true, Reason: contract.ReasonNoindex}, "", nil } if !sameAddress(res.SEO.Canonical, res.URL) { return contract.Decision{Exclude: true, Reason: contract.ReasonCanonical}, displayURL(b.site, res.SEO.Canonical), nil } return contract.Decision{}, "", nil } // SitemapExclusion is why a record's address is left out of the sitemaps: // a contract reason and, for canonical and redirect, the address listed // instead. type SitemapExclusion struct { Locale string Reason string Target string } // RecordSitemapExclusions lists why the sitemaps leave out the addresses // of record id of content type key, one per reason (the first locale that // has it): the type switched off in the sitemap settings (read through // contract.Listing), else the SEO plugin's own decision on each address // (noindex, canonical elsewhere, redirect). Nothing for an unpublished // record: the sitemaps never list it. func (s *Service) RecordSitemapExclusions(ctx context.Context, key, id string) ([]SitemapExclusion, error) { reg := s.Registry() if reg == nil { return nil, nil } rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale()) if err != nil || rec == nil || !rec.Published { return nil, err } if l := s.listing(); l != nil { st, err := l.Settings(ctx) if err != nil { return nil, err } for _, off := range st.TypesOff { if off == key { return []SitemapExclusion{{Reason: contract.ReasonType}}, nil } } } b, err := s.newFilterBatch(ctx) if err != nil { return nil, err } var out []SitemapExclusion seen := map[string]bool{} for _, l := range rec.Locales { if strings.TrimSpace(l.URL) == "" { continue } d, target, err := b.decideTarget(ctx, contract.Candidate{TypeKey: key, RecordID: id, Locale: l.Locale, URL: l.URL}) if err != nil { return nil, err } if d.Exclude && !seen[d.Reason] { seen[d.Reason] = true out = append(out, SitemapExclusion{Locale: l.Locale, Reason: d.Reason, Target: target}) } } return out, nil } // SitemapListed is the set of addresses the sitemaps list, read through // contract.Listing: their paths with the query, as the owner of a URL // names them. ok is false without a published Listing. At most // MaxListedAddresses are read. func (s *Service) SitemapListed(ctx context.Context) (map[string]bool, bool, error) { l := s.listing() if l == nil { return nil, false, nil } st, err := l.Settings(ctx) if err != nil { return nil, false, err } out := map[string]bool{} read := 0 for _, g := range st.Groups { after := "" for read < MaxListedAddresses { page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, After: after, Limit: 1000}) if err != nil { return nil, false, err } for _, e := range page.Entries { read++ if u, err := url.Parse(e.URL); err == nil { path := u.Path if path == "" { path = "/" } out[joinQuery(path, u.RawQuery)] = true } } if page.Next == "" || page.Next == after { break } after = page.Next } } return out, true, nil } // MaxListedAddresses caps the sitemap addresses SitemapListed reads. const MaxListedAddresses = 100000 // candidatePath turns a candidate's address (a path, or an absolute URL on // the site or its www twin) into the decoded path and its query; ok is // false for an address the plugin cannot judge (another host, malformed). func (b *filterBatch) candidatePath(raw string) (string, url.Values, string, bool) { raw = strings.TrimSpace(raw) if raw == "" { return "", nil, "", false } if u, err := url.Parse(raw); err == nil && u.IsAbs() && !sameSite(b.site, u.Host) { return "", nil, "", false } path, query, err := parseTarget(raw) if err != nil { return "", nil, "", false } return path, query, rawQueryOf(raw), true } // sameAddress reports whether two absolute URLs name the same page: the // host without letter case, the path and the query as written. func sameAddress(a, b string) bool { if a == b { return true } ua, errA := url.Parse(strings.TrimSpace(a)) ub, errB := url.Parse(strings.TrimSpace(b)) if errA != nil || errB != nil { return false } return strings.EqualFold(ua.Host, ub.Host) && ua.EscapedPath() == ub.EscapedPath() && ua.RawQuery == ub.RawQuery }