feat: sitemap filter excludes noindex, canonicalised and redirected URLs (tracer)

- classes.SitemapFilter implements the sitemap plugin's contract.Filter on
  the production values: redirected, noindex and canonicalised addresses
  leave the sitemaps with their reason; staging lists the same set
- the plugin requires golem15.sitemap and publishes the filter in Register
- go.mod requires sm-sitemap-plugin, go.work mounts the sibling checkout

classes/resolver.go

@@ -103,6 +103,10 @@ type MetaTag struct {
 // Resolver answers the effective SEO of a URL.
 type Resolver struct {
 	Service *Service
+	// production computes the values as production serves them: the
+	// non-production lock (D-22) is not applied. The sitemap filter decides
+	// on these, so staging lists the same addresses as production.
+	production bool
 }
 
 // Resolver returns the service's resolver.
@@ -600,7 +604,7 @@ func (r *Resolver) resolveAt(ctx context.Context, db *gorm.DB, settings models.S
 		layers = append(layers, robotsLayer{typeSetting.Robots, SourceTemplate})
 	}
 	robots, robotsSource := resolveRobots(layers)
-	if res.Staging {
+	if res.Staging && !r.production {
 		robots, robotsSource = Robots{Index: "noindex", Follow: "nofollow"}, SourceDefault
 	}
 	seo.Robots, src["robots"] = robots.Header(), robotsSource

classes/sitemap_filter.go

@@ -0,0 +1,166 @@
+package classes
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/sm-sitemap-plugin/contract"
+	"git.golem15.com/golem15/summercms/modules/pact"
+	"gorm.io/gorm"
+)
+
+// SitemapFilter is the SEO plugin's contract.Filter for the sitemap plugin
+// (D-95, D-24): an address whose effective SEO is noindex, whose canonical
+// URL points elsewhere, or that redirects leaves the XML and HTML sitemaps
+// with the reason noindex, canonical or redirect. It decides on the
+// production values: the non-production lock (every page noindex outside
+// production, D-22) never excludes an address, so staging and production
+// list the same set. Service is asked on every call, because the filter is
+// published in Register, before the service exists.
+type SitemapFilter struct {
+	Service func() *Service
+}
+
+var _ contract.Filter = SitemapFilter{}
+
+// filterBatch is what one Decide call reads once for all its candidates.
+type filterBatch struct {
+	svc      *Service
+	resolver *Resolver
+	db       *gorm.DB
+	settings models.Settings
+	opts     NormalizeOptions
+	set      *RedirectSet
+	site     string
+}
+
+// Decide answers one Decision per candidate, in order. Without a booted
+// service every address stays listed.
+func (f SitemapFilter) Decide(ctx context.Context, batch []contract.Candidate) ([]contract.Decision, error) {
+	out := make([]contract.Decision, len(batch))
+	var svc *Service
+	if f.Service != nil {
+		svc = f.Service()
+	}
+	if svc == nil || len(batch) == 0 {
+		return out, nil
+	}
+	b, err := svc.newFilterBatch(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for i, c := range batch {
+		d, err := b.decide(ctx, c)
+		if err != nil {
+			return nil, err
+		}
+		out[i] = d
+	}
+	return out, nil
+}
+
+// Groups lists the file groups the decisions name.
+func (f SitemapFilter) Groups(context.Context) ([]contract.Group, error) {
+	return nil, nil
+}
+
+// newFilterBatch reads the settings and the redirects once for a batch.
+func (s *Service) newFilterBatch(ctx context.Context) (*filterBatch, error) {
+	db, err := s.DB(ctx)
+	if err != nil {
+		return nil, err
+	}
+	settings, err := LoadSettings(db)
+	if err != nil {
+		return nil, err
+	}
+	set, err := LoadRedirectSet(db)
+	if err != nil {
+		return nil, err
+	}
+	return &filterBatch{
+		svc:      s,
+		resolver: &Resolver{Service: s, production: true},
+		db:       db,
+		settings: settings,
+		opts:     NormalizeOptionsOf(settings),
+		set:      set,
+		site:     s.SiteURL(),
+	}, nil
+}
+
+// decide is the decision of one candidate: a redirected address first (it
+// is no page), then noindex, then a canonical URL elsewhere.
+func (b *filterBatch) decide(ctx context.Context, c contract.Candidate) (contract.Decision, error) {
+	path, query, rawQuery, ok := b.candidatePath(c.URL)
+	if !ok {
+		return contract.Decision{}, nil
+	}
+	norm, changed := Normalize(&url.URL{Path: path, RawQuery: rawQuery}, b.opts)
+	if changed {
+		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil
+	}
+	chain, err := b.set.Follow(b.site, norm.Path, norm.RawQuery, MaxRedirectHops)
+	if err != nil {
+		return contract.Decision{}, err
+	}
+	if len(chain.Hops) > 0 {
+		return contract.Decision{Exclude: true, Reason: contract.ReasonRedirect}, nil
+	}
+	key := ""
+	var match *pact.ContentMatchResult
+	if reg := b.svc.Registry(); reg != nil && c.RecordID != "" {
+		if typ, ok := reg.ContentType(c.TypeKey); ok && typ.Content != nil {
+			key = c.TypeKey
+			match = &pact.ContentMatchResult{ID: c.RecordID, Locale: c.Locale, Exact: true}
+		}
+	}
+	res, err := b.resolver.resolveAt(ctx, b.db, b.settings, path, query, rawQuery, key, match, nil, true)
+	if err != nil {
+		return contract.Decision{}, err
+	}
+	if res.SEO == nil {
+		return contract.Decision{}, nil
+	}
+	if r, err := ParseRobots(res.SEO.Robots); err == nil && r.Index == "noindex" {
+		return contract.Decision{Exclude: true, Reason: contract.ReasonNoindex}, nil
+	}
+	if !sameAddress(res.SEO.Canonical, res.URL) {
+		return contract.Decision{Exclude: true, Reason: contract.ReasonCanonical}, nil
+	}
+	return contract.Decision{}, nil
+}
+
+// candidatePath turns a candidate's address (a path, or an absolute URL on
+// the site or its www twin) into the decoded path and its query; ok is
+// false for an address the plugin cannot judge (another host, malformed).
+func (b *filterBatch) candidatePath(raw string) (string, url.Values, string, bool) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return "", nil, "", false
+	}
+	if u, err := url.Parse(raw); err == nil && u.IsAbs() && !sameSite(b.site, u.Host) {
+		return "", nil, "", false
+	}
+	path, query, err := parseTarget(raw)
+	if err != nil {
+		return "", nil, "", false
+	}
+	return path, query, rawQueryOf(raw), true
+}
+
+// sameAddress reports whether two absolute URLs name the same page: the
+// host without letter case, the path and the query as written.
+func sameAddress(a, b string) bool {
+	if a == b {
+		return true
+	}
+	ua, errA := url.Parse(strings.TrimSpace(a))
+	ub, errB := url.Parse(strings.TrimSpace(b))
+	if errA != nil || errB != nil {
+		return false
+	}
+	return strings.EqualFold(ua.Host, ub.Host) && ua.EscapedPath() == ub.EscapedPath() && ua.RawQuery == ub.RawQuery
+}