sm-seo-plugin/classes/sitemap_filter.go

package classes

import (
	"context"
	"net/url"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
	"git.golem15.com/golem15/summercms/modules/pact"
	"gorm.io/gorm"
)

// SitemapFilter is the SEO plugin's contract.Filter for the sitemap plugin
// (D-95, D-24): an address whose effective SEO is noindex, whose canonical
// URL points elsewhere, or that redirects leaves the XML and HTML sitemaps
// with the reason noindex, canonical or redirect. It decides on the
// production values: the non-production lock (every page noindex outside
// production, D-22) never excludes an address, so staging and production
// list the same set. Service is asked on every call, because the filter is
// published in Register, before the service exists.
type SitemapFilter struct {
	Service func() *Service
}

var _ contract.Filter = SitemapFilter{}

// filterBatch is what one Decide call reads once for all its candidates.
type filterBatch struct {
	svc      *Service
	resolver *Resolver
	db       *gorm.DB
	settings models.Settings
	opts     NormalizeOptions
	set      *RedirectSet
	site     string
}

// Decide answers one Decision per candidate, in order. Without a booted
// service every address stays listed.
func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ([]contract.Decision, error) {
	out := make([]contract.Decision, len(batch))
	var svc *Service
	if f.Service != nil {
		svc = f.Service()
	}
	if svc == nil || len(batch) == 0 {
		return out, nil
	}
	b, err := svc.newFilterBatch(ctx)
	if err != nil {
		return nil, err
	}
	var pageIDs []string
	for _, c := range batch {
		if c.TypeKey == PageTypeKey {
			pageIDs = append(pageIDs, c.RecordID)
		}
	}
	kinds, err := pageKinds(b.db, pageIDs)
	if err != nil {
		return nil, err
	}
	for i, c := range batch {
		d, err := b.decide(ctx, c)
		if err != nil {
			return nil, err
		}
		if c.TypeKey == PageTypeKey && !d.Exclude {
			d.Group = PageGroup(kinds[c.RecordID])
		}
		out[i] = d
	}
	return out, nil
}

// Groups lists the file groups the decisions name: the registry's page
// kinds (pages, offers, services, landing-pages), titled in every locale
// of the site with the kind's name (P33-18).
func (f SitemapFilter) Groups(ctx context.Context) ([]contract.Group, error) {
	var svc *Service
	if f.Service != nil {
		svc = f.Service()
	}
	if svc == nil {
		return nil, nil
	}
	db, err := svc.DB(ctx)
	if err != nil {
		return nil, err
	}
	locales := svc.Locales(ctx, db)
	tr := svc.translator()
	out := make([]contract.Group, 0, len(models.PageKinds))
	for _, kind := range models.PageKinds {
		titles := map[string]string{}
		key := PageKindLabels[kind]
		for _, locale := range locales {
			if tr == nil {
				titles[locale] = key
				continue
			}
			titles[locale] = tr.GetIn(locale, key, nil)
		}
		out = append(out, contract.Group{Key: PageGroup(kind), Titles: titles})
	}
	return out, nil
}

// newFilterBatch reads the settings and the redirects once for a batch.
func (s *Service) newFilterBatch(ctx context.Context) (*filterBatch, error) {
	db, err := s.DB(ctx)
	if err != nil {
		return nil, err
	}
	settings, err := LoadSettings(db)
	if err != nil {
		return nil, err
	}
	set, err := LoadRedirectSet(db)
	if err != nil {
		return nil, err
	}
	return &filterBatch{
		svc:      s,
		resolver: &Resolver{Service: s, production: true},
		db:       db,
		settings: settings,
		opts:     NormalizeOptionsOf(settings),
		set:      set,
		site:     s.SiteURL(),
	}, nil
}

// decide is the decision of one candidate.
func (b *filterBatch) decide(ctx context.Context, c contract.Candidate) (contract.Decision, error) {
	d, _, err := b.decideTarget(ctx, c)
	return d, err
}

// decideTarget is the decision of one candidate and, for a canonical or a
// redirect, the address the sitemaps would list instead: a redirected
// address first (it is no page), then noindex, then a canonical URL
// elsewhere.
func (b *filterBatch) decideTarget(ctx context.Context, c contract.Candidate) (contract.Decision, string, error) {
	path, query, rawQuery, ok := b.candidatePath(c.URL)
	if !ok {
		return contract.Decision{}, "", nil
	}
	norm, changed := Normalize(&url.URL{Path: path, RawQuery: rawQuery}, b.opts)
	if changed {
		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, joinQuery(norm.Path, norm.RawQuery), nil
	}
	chain, err := b.set.Follow(b.site, norm.Path, norm.RawQuery, MaxRedirectHops)
	if err != nil {
		return contract.Decision{}, "", err
	}
	if len(chain.Hops) > 0 {
		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, displayURL(b.site, chain.Final), nil
	}
	key := ""
	var match *pact.ContentMatchResult
	if reg := b.svc.Registry(); reg != nil && c.RecordID != "" {
		if typ, ok := reg.ContentType(c.TypeKey); ok && typ.Content != nil {
			key = c.TypeKey
			match = &pact.ContentMatchResult{ID: c.RecordID, Locale: c.Locale, Exact: true}
		}
	}
	res, err := b.resolver.resolveAt(ctx, b.db, b.settings, path, query, rawQuery, key, match, nil, true, nil)
	if err != nil {
		return contract.Decision{}, "", err
	}
	if res.SEO == nil {
		return contract.Decision{}, "", nil
	}
	if r, err := ParseRobots(res.SEO.Robots); err == nil && r.Index == "noindex" {
		return contract.Decision{Exclude: true, Reason: contract.ReasonNoindex}, "", nil
	}
	if !sameAddress(res.SEO.Canonical, res.URL) {
		return contract.Decision{Exclude: true, Reason: contract.ReasonCanonical}, displayURL(b.site, res.SEO.Canonical), nil
	}
	return contract.Decision{}, "", nil
}

// SitemapExclusion is why a record's address is left out of the sitemaps:
// a contract reason and, for canonical and redirect, the address listed
// instead.
type SitemapExclusion struct {
	Locale string
	Reason string
	Target string
}

// RecordSitemapExclusions lists why the sitemaps leave out the addresses
// of record id of content type key, one per reason (the first locale that
// has it): the type switched off in the sitemap settings (read through
// contract.Listing), else the SEO plugin's own decision on each address
// (noindex, canonical elsewhere, redirect). Nothing for an unpublished
// record: the sitemaps never list it.
func (s *Service) RecordSitemapExclusions(ctx context.Context, key, id string) ([]SitemapExclusion, error) {
	reg := s.Registry()
	if reg == nil {
		return nil, nil
	}
	rec, err := reg.Describe(ctx, key, id, s.UnprefixedLocale())
	if err != nil || rec == nil || !rec.Published {
		return nil, err
	}
	if l := s.listing(); l != nil {
		st, err := l.Settings(ctx)
		if err != nil {
			return nil, err
		}
		for _, off := range st.TypesOff {
			if off == key {
				return []SitemapExclusion{{Reason: contract.ReasonType}}, nil
			}
		}
	}
	b, err := s.newFilterBatch(ctx)
	if err != nil {
		return nil, err
	}
	var out []SitemapExclusion
	seen := map[string]bool{}
	for _, l := range rec.Locales {
		if strings.TrimSpace(l.URL) == "" {
			continue
		}
		d, target, err := b.decideTarget(ctx, contract.Candidate{TypeKey: key, RecordID: id, Locale: l.Locale, URL: l.URL})
		if err != nil {
			return nil, err
		}
		if d.Exclude && !seen[d.Reason] {
			seen[d.Reason] = true
			out = append(out, SitemapExclusion{Locale: l.Locale, Reason: d.Reason, Target: target})
		}
	}
	return out, nil
}

// SitemapListed is the set of addresses the sitemaps list, read through
// contract.Listing: their paths with the query, as the owner of a URL
// names them. ok is false without a published Listing. At most
// MaxListedAddresses are read.
func (s *Service) SitemapListed(ctx context.Context) (map[string]bool, bool, error) {
	l := s.listing()
	if l == nil {
		return nil, false, nil
	}
	st, err := l.Settings(ctx)
	if err != nil {
		return nil, false, err
	}
	out := map[string]bool{}
	read := 0
	for _, g := range st.Groups {
		after := ""
		for read < MaxListedAddresses {
			page, err := l.Entries(ctx, contract.EntriesQuery{Group: g.Key, After: after, Limit: 1000})
			if err != nil {
				return nil, false, err
			}
			for _, e := range page.Entries {
				read++
				if u, err := url.Parse(e.URL); err == nil {
					path := u.Path
					if path == "" {
						path = "/"
					}
					out[joinQuery(path, u.RawQuery)] = true
				}
			}
			if page.Next == "" || page.Next == after {
				break
			}
			after = page.Next
		}
	}
	return out, true, nil
}

// MaxListedAddresses caps the sitemap addresses SitemapListed reads.
const MaxListedAddresses = 100000

// candidatePath turns a candidate's address (a path, or an absolute URL on
// the site or its www twin) into the decoded path and its query; ok is
// false for an address the plugin cannot judge (another host, malformed).
func (b *filterBatch) candidatePath(raw string) (string, url.Values, string, bool) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return "", nil, "", false
	}
	if u, err := url.Parse(raw); err == nil && u.IsAbs() && !sameSite(b.site, u.Host) {
		return "", nil, "", false
	}
	path, query, err := parseTarget(raw)
	if err != nil {
		return "", nil, "", false
	}
	return path, query, rawQueryOf(raw), true
}

// sameAddress reports whether two absolute URLs name the same page: the
// host without letter case, the path and the query as written.
func sameAddress(a, b string) bool {
	if a == b {
		return true
	}
	ua, errA := url.Parse(strings.TrimSpace(a))
	ub, errB := url.Parse(strings.TrimSpace(b))
	if errA != nil || errB != nil {
		return false
	}
	return strings.EqualFold(ua.Host, ub.Host) && ua.EscapedPath() == ub.EscapedPath() && ua.RawQuery == ub.RawQuery
}