package classes import ( "context" "net/url" "strings" "git.golem15.com/golem15/sm-seo-plugin/models" "git.golem15.com/golem15/sm-sitemap-plugin/contract" "git.golem15.com/golem15/summercms/modules/pact" "gorm.io/gorm" ) // SitemapFilter is the SEO plugin's contract.Filter for the sitemap plugin // (D-95, D-24): an address whose effective SEO is noindex, whose canonical // URL points elsewhere, or that redirects leaves the XML and HTML sitemaps // with the reason noindex, canonical or redirect. It decides on the // production values: the non-production lock (every page noindex outside // production, D-22) never excludes an address, so staging and production // list the same set. Service is asked on every call, because the filter is // published in Register, before the service exists. type SitemapFilter struct { Service func() *Service } var _ contract.Filter = SitemapFilter{} // filterBatch is what one Decide call reads once for all its candidates. type filterBatch struct { svc *Service resolver *Resolver db *gorm.DB settings models.Settings opts NormalizeOptions set *RedirectSet site string } // Decide answers one Decision per candidate, in order. Without a booted // service every address stays listed. func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ([]contract.Decision, error) { out := make([]contract.Decision, len(batch)) var svc *Service if f.Service != nil { svc = f.Service() } if svc == nil || len(batch) == 0 { return out, nil } b, err := svc.newFilterBatch(ctx) if err != nil { return nil, err } var pageIDs []string for _, c := range batch { if c.TypeKey == PageTypeKey { pageIDs = append(pageIDs, c.RecordID) } } kinds, err := pageKinds(b.db, pageIDs) if err != nil { return nil, err } for i, c := range batch { d, err := b.decide(ctx, c) if err != nil { return nil, err } if c.TypeKey == PageTypeKey && !d.Exclude { d.Group = PageGroup(kinds[c.RecordID]) } out[i] = d } return out, nil } // Groups lists the file groups the decisions name: the registry's page // kinds (pages, offers, services, landing-pages), titled in every locale // of the site with the kind's name (P33-18). func (f SitemapFilter) Groups(ctx context.Context) ([]contract.Group, error) { var svc *Service if f.Service != nil { svc = f.Service() } if svc == nil { return nil, nil } db, err := svc.DB(ctx) if err != nil { return nil, err } locales := svc.Locales(ctx, db) tr := svc.translator() out := make([]contract.Group, 0, len(models.PageKinds)) for _, kind := range models.PageKinds { titles := map[string]string{} key := PageKindLabels[kind] for _, locale := range locales { if tr == nil { titles[locale] = key continue } titles[locale] = tr.GetIn(locale, key, nil) } out = append(out, contract.Group{Key: PageGroup(kind), Titles: titles}) } return out, nil } // newFilterBatch reads the settings and the redirects once for a batch. func (s *Service) newFilterBatch(ctx context.Context) (*filterBatch, error) { db, err := s.DB(ctx) if err != nil { return nil, err } settings, err := LoadSettings(db) if err != nil { return nil, err } set, err := LoadRedirectSet(db) if err != nil { return nil, err } return &filterBatch{ svc: s, resolver: &Resolver{Service: s, production: true}, db: db, settings: settings, opts: NormalizeOptionsOf(settings), set: set, site: s.SiteURL(), }, nil } // decide is the decision of one candidate: a redirected address first (it // is no page), then noindex, then a canonical URL elsewhere. func (b *filterBatch) decide(ctx context.Context, c contract.Candidate) (contract.Decision, error) { path, query, rawQuery, ok := b.candidatePath(c.URL) if !ok { return contract.Decision{}, nil } norm, changed := Normalize(&url.URL{Path: path, RawQuery: rawQuery}, b.opts) if changed { return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil } chain, err := b.set.Follow(b.site, norm.Path, norm.RawQuery, MaxRedirectHops) if err != nil { return contract.Decision{}, err } if len(chain.Hops) > 0 { return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil } key := "" var match *pact.ContentMatchResult if reg := b.svc.Registry(); reg != nil && c.RecordID != "" { if typ, ok := reg.ContentType(c.TypeKey); ok && typ.Content != nil { key = c.TypeKey match = &pact.ContentMatchResult{ID: c.RecordID, Locale: c.Locale, Exact: true} } } res, err := b.resolver.resolveAt(ctx, b.db, b.settings, path, query, rawQuery, key, match, nil, true) if err != nil { return contract.Decision{}, err } if res.SEO == nil { return contract.Decision{}, nil } if r, err := ParseRobots(res.SEO.Robots); err == nil && r.Index == "noindex" { return contract.Decision{Exclude: true, Reason: contract.ReasonNoindex}, nil } if !sameAddress(res.SEO.Canonical, res.URL) { return contract.Decision{Exclude: true, Reason: contract.ReasonCanonical}, nil } return contract.Decision{}, nil } // candidatePath turns a candidate's address (a path, or an absolute URL on // the site or its www twin) into the decoded path and its query; ok is // false for an address the plugin cannot judge (another host, malformed). func (b *filterBatch) candidatePath(raw string) (string, url.Values, string, bool) { raw = strings.TrimSpace(raw) if raw == "" { return "", nil, "", false } if u, err := url.Parse(raw); err == nil && u.IsAbs() && !sameSite(b.site, u.Host) { return "", nil, "", false } path, query, err := parseTarget(raw) if err != nil { return "", nil, "", false } return path, query, rawQueryOf(raw), true } // sameAddress reports whether two absolute URLs name the same page: the // host without letter case, the path and the query as written. func sameAddress(a, b string) bool { if a == b { return true } ua, errA := url.Parse(strings.TrimSpace(a)) ub, errB := url.Parse(strings.TrimSpace(b)) if errA != nil || errB != nil { return false } return strings.EqualFold(ua.Host, ub.Host) && ua.EscapedPath() == ub.EscapedPath() && ua.RawQuery == ub.RawQuery }