sm-seo-plugin/classes/sitemap_filter.go

package classes

import (
	"context"
	"net/url"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
	"git.golem15.com/golem15/summercms/modules/pact"
	"gorm.io/gorm"
)

// SitemapFilter is the SEO plugin's contract.Filter for the sitemap plugin
// (D-95, D-24): an address whose effective SEO is noindex, whose canonical
// URL points elsewhere, or that redirects leaves the XML and HTML sitemaps
// with the reason noindex, canonical or redirect. It decides on the
// production values: the non-production lock (every page noindex outside
// production, D-22) never excludes an address, so staging and production
// list the same set. Service is asked on every call, because the filter is
// published in Register, before the service exists.
type SitemapFilter struct {
	Service func() *Service
}

var _ contract.Filter = SitemapFilter{}

// filterBatch is what one Decide call reads once for all its candidates.
type filterBatch struct {
	svc      *Service
	resolver *Resolver
	db       *gorm.DB
	settings models.Settings
	opts     NormalizeOptions
	set      *RedirectSet
	site     string
}

// Decide answers one Decision per candidate, in order. Without a booted
// service every address stays listed.
func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ([]contract.Decision, error) {
	out := make([]contract.Decision, len(batch))
	var svc *Service
	if f.Service != nil {
		svc = f.Service()
	}
	if svc == nil || len(batch) == 0 {
		return out, nil
	}
	b, err := svc.newFilterBatch(ctx)
	if err != nil {
		return nil, err
	}
	var pageIDs []string
	for _, c := range batch {
		if c.TypeKey == PageTypeKey {
			pageIDs = append(pageIDs, c.RecordID)
		}
	}
	kinds, err := pageKinds(b.db, pageIDs)
	if err != nil {
		return nil, err
	}
	for i, c := range batch {
		d, err := b.decide(ctx, c)
		if err != nil {
			return nil, err
		}
		if c.TypeKey == PageTypeKey && !d.Exclude {
			d.Group = PageGroup(kinds[c.RecordID])
		}
		out[i] = d
	}
	return out, nil
}

// Groups lists the file groups the decisions name: the registry's page
// kinds (pages, offers, services, landing-pages), titled in every locale
// of the site with the kind's name (P33-18).
func (f SitemapFilter) Groups(ctx context.Context) ([]contract.Group, error) {
	var svc *Service
	if f.Service != nil {
		svc = f.Service()
	}
	if svc == nil {
		return nil, nil
	}
	db, err := svc.DB(ctx)
	if err != nil {
		return nil, err
	}
	locales := svc.Locales(ctx, db)
	tr := svc.translator()
	out := make([]contract.Group, 0, len(models.PageKinds))
	for _, kind := range models.PageKinds {
		titles := map[string]string{}
		key := PageKindLabels[kind]
		for _, locale := range locales {
			if tr == nil {
				titles[locale] = key
				continue
			}
			titles[locale] = tr.GetIn(locale, key, nil)
		}
		out = append(out, contract.Group{Key: PageGroup(kind), Titles: titles})
	}
	return out, nil
}

// newFilterBatch reads the settings and the redirects once for a batch.
func (s *Service) newFilterBatch(ctx context.Context) (*filterBatch, error) {
	db, err := s.DB(ctx)
	if err != nil {
		return nil, err
	}
	settings, err := LoadSettings(db)
	if err != nil {
		return nil, err
	}
	set, err := LoadRedirectSet(db)
	if err != nil {
		return nil, err
	}
	return &filterBatch{
		svc:      s,
		resolver: &Resolver{Service: s, production: true},
		db:       db,
		settings: settings,
		opts:     NormalizeOptionsOf(settings),
		set:      set,
		site:     s.SiteURL(),
	}, nil
}

// decide is the decision of one candidate: a redirected address first (it
// is no page), then noindex, then a canonical URL elsewhere.
func (b *filterBatch) decide(ctx context.Context, c contract.Candidate) (contract.Decision, error) {
	path, query, rawQuery, ok := b.candidatePath(c.URL)
	if !ok {
		return contract.Decision{}, nil
	}
	norm, changed := Normalize(&url.URL{Path: path, RawQuery: rawQuery}, b.opts)
	if changed {
		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil
	}
	chain, err := b.set.Follow(b.site, norm.Path, norm.RawQuery, MaxRedirectHops)
	if err != nil {
		return contract.Decision{}, err
	}
	if len(chain.Hops) > 0 {
		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil
	}
	key := ""
	var match *pact.ContentMatchResult
	if reg := b.svc.Registry(); reg != nil && c.RecordID != "" {
		if typ, ok := reg.ContentType(c.TypeKey); ok && typ.Content != nil {
			key = c.TypeKey
			match = &pact.ContentMatchResult{ID: c.RecordID, Locale: c.Locale, Exact: true}
		}
	}
	res, err := b.resolver.resolveAt(ctx, b.db, b.settings, path, query, rawQuery, key, match, nil, true)
	if err != nil {
		return contract.Decision{}, err
	}
	if res.SEO == nil {
		return contract.Decision{}, nil
	}
	if r, err := ParseRobots(res.SEO.Robots); err == nil && r.Index == "noindex" {
		return contract.Decision{Exclude: true, Reason: contract.ReasonNoindex}, nil
	}
	if !sameAddress(res.SEO.Canonical, res.URL) {
		return contract.Decision{Exclude: true, Reason: contract.ReasonCanonical}, nil
	}
	return contract.Decision{}, nil
}

// candidatePath turns a candidate's address (a path, or an absolute URL on
// the site or its www twin) into the decoded path and its query; ok is
// false for an address the plugin cannot judge (another host, malformed).
func (b *filterBatch) candidatePath(raw string) (string, url.Values, string, bool) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return "", nil, "", false
	}
	if u, err := url.Parse(raw); err == nil && u.IsAbs() && !sameSite(b.site, u.Host) {
		return "", nil, "", false
	}
	path, query, err := parseTarget(raw)
	if err != nil {
		return "", nil, "", false
	}
	return path, query, rawQueryOf(raw), true
}

// sameAddress reports whether two absolute URLs name the same page: the
// host without letter case, the path and the query as written.
func sameAddress(a, b string) bool {
	if a == b {
		return true
	}
	ua, errA := url.Parse(strings.TrimSpace(a))
	ub, errB := url.Parse(strings.TrimSpace(b))
	if errA != nil || errB != nil {
		return false
	}
	return strings.EqualFold(ua.Host, ub.Host) && ua.EscapedPath() == ub.EscapedPath() && ua.RawQuery == ub.RawQuery
}