sm-seo-plugin/classes/issues.go

package classes

import (
	"context"
	"net/url"
	"sort"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/models"
)

// The kinds of the CMS-data issue report (D-13, D-14 CMS part). The
// crawler's checks of the rendered pages are Phase 33.
const (
	IssueMissingTitle        = "missing_title"
	IssueMissingDescription  = "missing_description"
	IssueDuplicateTitle      = "duplicate_title"
	IssueDuplicateDesc       = "duplicate_description"
	IssueRuleOverridesRecord = "rule_overrides_record"
	IssueRedirectChain       = "redirect_chain"
	IssueCanonicalNoindex    = "canonical_noindex"
	IssueCanonicalRedirected = "canonical_redirected"
	IssueNoindexBlocked      = "noindex_blocked"
	// IssueSitemapBlocked is a sitemap address the production robots.txt
	// keeps Googlebot from (D-24): an error, since the sitemap asks for a
	// page the crawler may not fetch.
	IssueSitemapBlocked = "sitemap_blocked"
)

// The severities an issue may name.
const (
	SeverityError = "error"
)

// Issue is one finding of the issue report: the address, the kind, the
// details of the finding and the MCP tool that fixes it.
type Issue struct {
	Kind    string         `json:"kind"`
	URL     string         `json:"url"`
	Locale  string         `json:"locale,omitempty"`
	Details map[string]any `json:"details,omitempty"`
	Tool    string         `json:"tool"`
	// Severity is set for kinds that are errors (sitemap_blocked); "" for
	// the others.
	Severity string `json:"severity,omitempty"`
}

// The tools the issues point to (pier prefixes them with seo_).
const (
	toolSetURLSEO      = "seo_set_url_seo"
	toolUpdateRule     = "seo_update_rule"
	toolUpdateRedirect = "seo_update_redirect"
	toolUpdateRobots   = "seo_update_robots"
)

// Issues builds the CMS-data issue report (D-13, D-14): missing effective
// titles and descriptions per record and locale, effective titles and
// descriptions shared by more than one address (records and URL rules),
// URL rules overriding a record, redirect chains with their final
// targets, canonicals pointing to a noindex or redirected address, and
// noindex addresses robots.txt keeps crawlers from. Records are read
// through their SEO entries (the generation job creates one per
// published record and locale).
func (s *Service) Issues(ctx context.Context) ([]Issue, error) {
	db, err := s.DB(ctx)
	if err != nil {
		return nil, err
	}
	var data issueData
	if err := db.Where("url <> ''").Order("url ASC, locale ASC").Find(&data.entries).Error; err != nil {
		return nil, err
	}
	if err := db.Order("url ASC").Find(&data.rules).Error; err != nil {
		return nil, err
	}
	data.locale = s.DefaultLocale(ctx, db)
	if data.redirects, err = LoadRedirects(db); err != nil {
		return nil, err
	}
	if data.robots, err = LoadRobots(db); err != nil {
		return nil, err
	}
	if data.noindex, err = noindexAddresses(db); err != nil {
		return nil, err
	}
	listed, _, err := s.SitemapListed(ctx)
	if err != nil {
		return nil, err
	}
	for address := range listed {
		data.listed = append(data.listed, address)
	}
	sort.Strings(data.listed)
	return s.issuesOf(ctx, data, s.canonicalProblem)
}

// issueData is what the issue report reads from the database: the SEO
// entries with an address (by url and locale), the URL rules (by url), the
// default content locale, the redirects, the robots row and the noindex
// addresses.
type issueData struct {
	entries   []models.Entry
	rules     []models.Rule
	locale    string
	redirects []models.Redirect
	robots    models.Robots
	noindex   []string
	// listed are the addresses the sitemaps list (path with query).
	listed []string
}

// issuesOf builds the report from data; canonical classifies a canonical
// value (canonicalProblem).
func (s *Service) issuesOf(ctx context.Context, data issueData, canonical func(context.Context, string) (string, string, error)) ([]Issue, error) {
	var out []Issue
	entries, rules := data.entries, data.rules

	// Missing and duplicate effective values.
	type place struct{ url, locale string }
	titles := map[string][]place{}
	descs := map[string][]place{}
	for _, e := range entries {
		if strings.TrimSpace(e.EffectiveTitle) == "" {
			out = append(out, Issue{Kind: IssueMissingTitle, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO})
		} else {
			k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveTitle))
			titles[k] = append(titles[k], place{e.URL, e.Locale})
		}
		if strings.TrimSpace(e.EffectiveDescription) == "" {
			out = append(out, Issue{Kind: IssueMissingDescription, URL: e.URL, Locale: e.Locale, Tool: toolSetURLSEO})
		} else {
			k := e.Locale + "|" + strings.ToLower(strings.TrimSpace(e.EffectiveDescription))
			descs[k] = append(descs[k], place{e.URL, e.Locale})
		}
	}
	locale := data.locale
	for _, r := range rules {
		if r.Match != models.MatchExact {
			continue
		}
		if t := strings.TrimSpace(r.Title); t != "" {
			k := locale + "|" + strings.ToLower(t)
			titles[k] = append(titles[k], place{r.URL, ""})
		}
		if d := strings.TrimSpace(r.Description); d != "" {
			k := locale + "|" + strings.ToLower(d)
			descs[k] = append(descs[k], place{r.URL, ""})
		}
	}
	duplicates := func(groups map[string][]place, kind string) {
		for key, list := range groups {
			if len(list) < 2 {
				continue
			}
			_, text, _ := strings.Cut(key, "|")
			for i, p := range list {
				others := make([]string, 0, len(list)-1)
				for j, o := range list {
					if j != i && o.url != p.url {
						others = append(others, o.url)
					}
				}
				if len(others) == 0 {
					continue
				}
				out = append(out, Issue{Kind: kind, URL: p.url, Locale: p.locale, Tool: toolSetURLSEO, Details: map[string]any{"value": text, "also": others}})
			}
		}
	}
	duplicates(titles, IssueDuplicateTitle)
	duplicates(descs, IssueDuplicateDesc)

	// URL rules overriding a record.
	if reg := s.Registry(); reg != nil {
		for _, r := range rules {
			if r.Match != models.MatchExact || strings.Contains(r.URL, "?") {
				continue
			}
			var fields []string
			for _, p := range models.RuleProperties {
				if strings.TrimSpace(r.Value(p.Column)) != "" {
					fields = append(fields, p.Column)
				}
			}
			if len(fields) == 0 {
				continue
			}
			key, match, err := reg.MatchPath(ctx, trimSlash(r.URL), s.UnprefixedLocale())
			if err != nil {
				return nil, err
			}
			if match == nil || !match.Exact {
				continue
			}
			out = append(out, Issue{Kind: IssueRuleOverridesRecord, URL: r.URL, Locale: match.Locale, Tool: toolUpdateRule, Details: map[string]any{
				"rule_id": r.ID, "type": key, "id": match.ID, "fields": fields,
			}})
		}
	}

	// Redirect chains.
	redirects := data.redirects
	set := NewRedirectSet(redirects)
	site := s.SiteURL()
	for _, r := range redirects {
		if r.Status == models.Status410 {
			continue
		}
		path, query, ok := redirectProbe(r)
		if !ok {
			continue
		}
		ch, err := set.Follow(site, path, query, MaxRedirectHops)
		if err != nil {
			return nil, err
		}
		if ch.Loop || len(ch.Hops) < 2 || ch.Hops[0].Rule.ID != r.ID {
			continue
		}
		out = append(out, Issue{Kind: IssueRedirectChain, URL: r.Source, Tool: toolUpdateRedirect, Details: map[string]any{
			"redirect_id": r.ID, "hops": len(ch.Hops), "final": displayURL(site, ch.Final), "path": displayPath(site, r, ch),
		}})
	}

	// Canonicals pointing to a noindex or redirected address.
	addCanonical := func(u, loc, value, tool string, details map[string]any) error {
		kind, target, err := canonical(ctx, value)
		if err != nil || kind == "" {
			return err
		}
		if details == nil {
			details = map[string]any{}
		}
		details["canonical"] = value
		if target != "" {
			details["target"] = target
		}
		out = append(out, Issue{Kind: kind, URL: u, Locale: loc, Tool: tool, Details: details})
		return nil
	}
	for _, e := range entries {
		if err := addCanonical(e.URL, e.Locale, e.Canonical, toolSetURLSEO, nil); err != nil {
			return nil, err
		}
	}
	for _, r := range rules {
		if err := addCanonical(r.URL, "", r.Canonical, toolUpdateRule, map[string]any{"rule_id": r.ID}); err != nil {
			return nil, err
		}
	}

	// Noindex addresses robots.txt blocks.
	file := ComposeRobots(data.robots.Content, data.robots.Bots, s.SitemapLines(ctx))
	for _, u := range s.blockedNoindex(data.noindex, data.robots.Content, data.robots.Bots) {
		out = append(out, Issue{Kind: IssueNoindexBlocked, URL: u, Tool: toolUpdateRobots})
	}

	// Sitemap addresses the production robots.txt blocks for Googlebot.
	for _, u := range data.listed {
		test := s.testRobotsFile(file, data.robots.Content, "Googlebot", u)
		if test.Allowed {
			continue
		}
		out = append(out, Issue{Kind: IssueSitemapBlocked, URL: u, Tool: toolUpdateRobots, Severity: SeverityError, Details: map[string]any{
			"line": test.Line, "rule": test.Rule,
		}})
	}

	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Kind != out[j].Kind {
			return out[i].Kind < out[j].Kind
		}
		if out[i].URL != out[j].URL {
			return out[i].URL < out[j].URL
		}
		return out[i].Locale < out[j].Locale
	})
	return out, nil
}

// canonicalProblem checks a canonical on this site (D-06): a redirect
// answers canonical_redirected with the final address, a page served
// noindex (in production) canonical_noindex. Other sites and empty values
// answer no kind.
func (s *Service) canonicalProblem(ctx context.Context, canonical string) (kind, target string, err error) {
	canonical = strings.TrimSpace(canonical)
	if canonical == "" {
		return "", "", nil
	}
	ref := canonical
	if u, err := url.Parse(canonical); err == nil && u.IsAbs() {
		if !sameSite(s.SiteURL(), u.Host) {
			return "", "", nil
		}
		ref = u.String()
	}
	res, err := s.Resolver().Resolve(ctx, ref)
	if err != nil {
		if err == ErrBadURL {
			return "", "", nil
		}
		return "", "", err
	}
	if res.Action == ActionRedirect {
		return IssueCanonicalRedirected, res.Location, nil
	}
	if s.Staging() || res.SEO == nil || !strings.Contains(res.SEO.Robots, "noindex") {
		return "", "", nil
	}
	return IssueCanonicalNoindex, "", nil
}