sm-seo-plugin/classes/crawl/checks.go

package crawl

import (
	"encoding/json"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/pact"
	"github.com/jimsmart/grobotstxt"
)

// The crawl kinds of UI-SPEC C23.
const (
	KindPageStatus        = "page_status"
	KindSitemapError      = "sitemap_error"
	KindSitemapNoindex    = "sitemap_noindex"
	KindSitemapBlocked    = "sitemap_blocked"
	KindSitemapMissing    = "sitemap_missing"
	KindBrokenLink        = "broken_link"
	KindRedirectedLink    = "redirected_link"
	KindCanonicalMissing  = "canonical_missing"
	KindCanonicalError    = "canonical_error"
	KindMissingTitle      = "missing_title"
	KindDuplicateTitle    = "duplicate_title"
	KindMissingDesc       = "missing_description"
	KindDuplicateDesc     = "duplicate_description"
	KindH1Missing         = "h1_missing"
	KindH1Multiple        = "h1_multiple"
	KindHeadingSkipped    = "heading_skipped"
	KindHeadingEmpty      = "heading_empty"
	KindH1TitleMismatch   = "h1_title_mismatch"
	KindJSONLDInvalid     = "jsonld_invalid"
	KindJSONLDMultiple    = "jsonld_multiple"
	KindJSONLDRequired    = "jsonld_required"
	KindJSONLDRecommended = "jsonld_recommended"
	KindHreflangRecip     = "hreflang_reciprocal"
	KindHreflangError     = "hreflang_error"
	KindGenericAnchor     = "generic_anchor"
	KindImageAlt          = "image_alt"
	KindImageSize         = "image_size"
	KindFeaturedLazy      = "featured_lazy"
	KindContentJS         = "content_js"
	KindOrphanPage        = "orphan_page"
	// KindH1InContent is the CMS-data check of a record whose body holds
	// an H1 although its page renders the title as the H1.
	KindH1InContent = "h1_in_content"
)

// severities are the default severities of the kinds table (UI-SPEC C23),
// the Phase 32 CMS kinds included.
var severities = map[string]string{
	KindPageStatus:          models.SeverityError,
	KindSitemapError:        models.SeverityError,
	KindSitemapNoindex:      models.SeverityError,
	KindSitemapBlocked:      models.SeverityError,
	KindSitemapMissing:      models.SeverityNotice,
	KindBrokenLink:          models.SeverityError,
	KindRedirectedLink:      models.SeverityNotice,
	KindCanonicalMissing:    models.SeverityWarning,
	KindCanonicalError:      models.SeverityError,
	"canonical_noindex":     models.SeverityError,
	"canonical_redirected":  models.SeverityWarning,
	KindMissingTitle:        models.SeverityWarning,
	KindDuplicateTitle:      models.SeverityWarning,
	KindMissingDesc:         models.SeverityNotice,
	KindDuplicateDesc:       models.SeverityNotice,
	"rule_overrides_record": models.SeverityNotice,
	"redirect_chain":        models.SeverityWarning,
	"noindex_blocked":       models.SeverityError,
	KindH1Missing:           models.SeverityError,
	KindH1Multiple:          models.SeverityError,
	KindH1InContent:         models.SeverityWarning,
	KindHeadingSkipped:      models.SeverityWarning,
	KindHeadingEmpty:        models.SeverityWarning,
	KindH1TitleMismatch:     models.SeverityNotice,
	KindJSONLDInvalid:       models.SeverityError,
	KindJSONLDMultiple:      models.SeverityWarning,
	KindJSONLDRequired:      models.SeverityError,
	KindJSONLDRecommended:   models.SeverityNotice,
	KindHreflangRecip:       models.SeverityWarning,
	KindHreflangError:       models.SeverityError,
	KindGenericAnchor:       models.SeverityWarning,
	KindImageAlt:            models.SeverityWarning,
	KindImageSize:           models.SeverityNotice,
	KindFeaturedLazy:        models.SeverityNotice,
	KindContentJS:           models.SeverityWarning,
	KindOrphanPage:          models.SeverityWarning,
}

// Severity returns a kind's severity (warning for an unknown kind).
func Severity(kind string) string {
	if s, ok := severities[kind]; ok {
		return s
	}
	return models.SeverityWarning
}

// Kinds lists every kind the audit stores, in the kinds table order.
func Kinds() []string {
	return []string{
		KindPageStatus, KindSitemapError, KindSitemapNoindex, KindSitemapBlocked,
		KindSitemapMissing, KindBrokenLink, KindRedirectedLink, KindCanonicalMissing,
		KindCanonicalError, "canonical_noindex", "canonical_redirected", KindMissingTitle,
		KindDuplicateTitle, KindMissingDesc, KindDuplicateDesc, "rule_overrides_record",
		"redirect_chain", "noindex_blocked", KindH1Missing, KindH1Multiple, KindH1InContent,
		KindHeadingSkipped, KindHeadingEmpty, KindH1TitleMismatch, KindJSONLDInvalid,
		KindJSONLDMultiple, KindJSONLDRequired, KindJSONLDRecommended, KindHreflangRecip,
		KindHreflangError, KindGenericAnchor, KindImageAlt, KindImageSize, KindFeaturedLazy,
		KindContentJS, KindOrphanPage,
	}
}

// Env is what the checks know besides the crawl: the record behind an
// address, whether the site runs outside production (the staging lock's
// noindex is never reported, RESEARCH Pitfall 12) and the generic link
// texts.
type Env struct {
	// Lookup returns the CMS record of an address, nil when none.
	Lookup func(key string) *Record
	// Staging is true outside production.
	Staging bool
	// GenericAnchors are the generic link texts (any case).
	GenericAnchors []string
	// Keyphrases maps an address to its SEO entry's focus keyphrase.
	Keyphrases map[string]string
}

func (e *Env) record(key string) *Record {
	if e == nil || e.Lookup == nil {
		return nil
	}
	return e.Lookup(key)
}

// locale is the issue locale of a page: its record's.
func locale(rec *Record) string {
	if rec == nil {
		return ""
	}
	return rec.Locale
}

// noindex reports whether a page asks not to be indexed (meta robots or
// X-Robots-Tag). On a non-production environment every page carries the
// staging lock's noindex, so it is never counted there.
func (e *Env) noindex(p *Page) bool {
	if e != nil && e.Staging {
		return false
	}
	if p.Facts != nil && p.Facts.Noindex() {
		return true
	}
	return p.Resp != nil && hasNoindex(p.Resp.XRobots)
}

// selfCanonical reports whether a page's canonical is missing or names
// the page itself.
func selfCanonical(res *Result, p *Page) bool {
	if p.Facts == nil || p.Facts.Canonical == "" || res.fetcher == nil {
		return true
	}
	u, err := url.Parse(p.Facts.Canonical)
	if err != nil {
		return true
	}
	return res.fetcher.Key(u) == p.Key
}

// indexable reports whether a page answers 200 with HTML, may be indexed
// and is its own canonical.
func (e *Env) indexable(res *Result, p *Page) bool {
	return p.OK() && !e.noindex(p) && selfCanonical(res, p)
}

// Check runs every check over a crawl's pages: the per-page checks of
// UI-SPEC C23 and the site-wide ones (sitemaps, statuses, duplicates).
func Check(res *Result, env *Env) []Finding {
	if env == nil {
		env = &Env{}
	}
	var out []Finding
	for _, key := range res.Order {
		p := res.Pages[key]
		rec := env.record(key)
		out = append(out, checkStatus(res, p, rec)...)
		if !p.OK() {
			continue
		}
		out = append(out, checkHead(res, p, rec)...)
		out = append(out, checkHeadings(p, rec)...)
		out = append(out, checkJSONLD(p, rec)...)
		out = append(out, checkHreflang(res, p, rec)...)
		out = append(out, checkGenericAnchors(res, p, rec, env.GenericAnchors)...)
		out = append(out, checkImages(p, rec)...)
		out = append(out, checkContentCMS(res, p, rec)...)
		out = append(out, checkRenderedPage(res, p, rec)...)
	}
	out = append(out, checkLinks(res, env)...)
	out = append(out, checkOrphans(res, env)...)
	out = append(out, checkSitemaps(res, env)...)
	out = append(out, checkDuplicates(res, env)...)
	return out
}

// checkStatus reports page_status for an address that answers an error
// (other than 404 and 410, which are broken links when linked) or cannot
// be fetched.
func checkStatus(res *Result, p *Page, rec *Record) []Finding {
	r := p.Resp
	if r == nil {
		return nil
	}
	switch {
	case r.Err != "" && r.Err != "canceled":
		return []Finding{{Kind: KindPageStatus, URL: p.Key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": 0, "error": r.Err}}}
	case r.Status >= 400 && r.Status != 404 && r.Status != 410:
		return []Finding{{Kind: KindPageStatus, URL: p.Key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": r.Status}}}
	}
	return nil
}

// checkHead reports a missing title, description or canonical and a
// canonical pointing to an error page.
func checkHead(res *Result, p *Page, rec *Record) []Finding {
	var out []Finding
	f := p.Facts
	loc := locale(rec)
	if strings.TrimSpace(f.Title) == "" {
		out = append(out, Finding{Kind: KindMissingTitle, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO})
	}
	if strings.TrimSpace(f.Description) == "" {
		out = append(out, Finding{Kind: KindMissingDesc, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO})
	}
	if f.Canonical == "" {
		out = append(out, Finding{Kind: KindCanonicalMissing, URL: p.Key, Locale: loc, Record: rec})
	} else if u, err := url.Parse(f.Canonical); err == nil && res.fetcher != nil {
		if key := res.fetcher.Key(u); key != "" && key != p.Key {
			if t := res.Page(key); t != nil && t.Resp != nil && (t.Resp.Err != "" || t.Resp.Status >= 400) {
				out = append(out, Finding{Kind: KindCanonicalError, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"url": key, "status": t.Resp.Status}})
			}
		}
	}
	if h1 := f.H1(); len(h1) > 0 && strings.TrimSpace(f.Title) != "" && strings.TrimSpace(h1[0]) != "" {
		a, b := normal(h1[0]), normal(f.Title)
		if !strings.Contains(b, a) && !strings.Contains(a, b) {
			out = append(out, Finding{Kind: KindH1TitleMismatch, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"h1": h1[0], "title": f.Title}})
		}
	}
	return out
}

// checkHeadings reports h1_missing, h1_multiple, heading_skipped and
// heading_empty. A heading inside the record's own content is fixed
// through the record's tool: H1 findings prefer Tools[headings], the
// others use Tools[edit] (P33-38).
func checkHeadings(p *Page, rec *Record) []Finding {
	var out []Finding
	loc := locale(rec)
	h1 := p.Facts.H1()
	switch {
	case len(h1) == 0:
		out = append(out, Finding{Kind: KindH1Missing, URL: p.Key, Locale: loc, Record: rec})
	case len(h1) > 1:
		f := Finding{Kind: KindH1Multiple, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}}
		if rec.hasHeading(1, "") {
			f.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
		}
		out = append(out, f)
	}
	var skips []map[string]any
	var first *Finding
	prev := 0
	for _, h := range p.Facts.Headings {
		if prev > 0 && h.Level > prev+1 {
			skips = append(skips, map[string]any{"from": prev, "to": h.Level, "text": h.Text})
			if first == nil {
				first = &Finding{Kind: KindHeadingSkipped, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"from": prev, "to": h.Level, "text": h.Text}}
				if rec.hasHeading(h.Level, h.Text) {
					first.Content = []string{pact.ContentToolEdit}
				}
			}
		}
		prev = h.Level
	}
	if first != nil {
		first.Details["skips"] = skips
		first.Details["count"] = len(skips)
		out = append(out, *first)
	}
	empty := 0
	var firstEmpty *Finding
	for _, h := range p.Facts.Headings {
		if strings.TrimSpace(h.Text) != "" {
			continue
		}
		empty++
		if firstEmpty == nil {
			firstEmpty = &Finding{Kind: KindHeadingEmpty, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"level": h.Level}}
			if hasEmptyHeading(rec, h.Level) {
				if h.Level == 1 {
					firstEmpty.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
				} else {
					firstEmpty.Content = []string{pact.ContentToolEdit}
				}
			}
		}
	}
	if firstEmpty != nil {
		firstEmpty.Details["count"] = empty
		out = append(out, *firstEmpty)
	}
	return out
}

// hasEmptyHeading reports whether the record's own headings hold an empty
// one of level.
func hasEmptyHeading(rec *Record, level int) bool {
	if rec == nil {
		return false
	}
	for _, h := range rec.Headings {
		if h.Level == level && strings.TrimSpace(h.Text) == "" {
			return true
		}
	}
	return false
}

// checkSitemaps compares the sitemaps with what the crawl found:
// sitemap_error for a listed address that does not answer 200,
// sitemap_noindex for a listed noindex page, sitemap_blocked for a listed
// address robots.txt keeps Googlebot from, and sitemap_missing for an
// indexable page found only through links.
func checkSitemaps(res *Result, env *Env) []Finding {
	var out []Finding
	var keys []string
	for key := range res.Listed {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	robots := res.Robots
	for _, key := range keys {
		file := res.Listed[key]
		rec := env.record(key)
		if p := res.Page(key); p != nil && p.Resp != nil {
			switch {
			case p.Resp.Err == "canceled":
			case p.Resp.Err != "" || p.Resp.Status != 200:
				out = append(out, Finding{Kind: KindSitemapError, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"status": p.Resp.Status, "file": file}})
			case env.noindex(p):
				out = append(out, Finding{Kind: KindSitemapNoindex, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"file": file}})
			}
		}
		if robots != "" && res.fetcher != nil {
			m := grobotstxt.NewRobotsMatcher()
			if !m.AgentAllowed(robots, "Googlebot", res.fetcher.Absolute(key)) {
				line := m.MatchingLine()
				rule := ""
				if lines := strings.Split(robots, "\n"); line > 0 && line <= len(lines) {
					rule = strings.TrimSpace(lines[line-1])
				}
				out = append(out, Finding{Kind: KindSitemapBlocked, URL: key, Locale: locale(rec), Record: rec, Tool: ToolUpdateRobots, Details: map[string]any{"line": line, "rule": rule, "file": file}})
			}
		}
	}
	links := inbound(res)
	for _, key := range res.Order {
		p := res.Pages[key]
		if _, listed := res.Listed[key]; listed || !env.indexable(res, p) {
			continue
		}
		rec := env.record(key)
		f := Finding{Kind: KindSitemapMissing, URL: key, Locale: locale(rec), Record: rec, Tool: ToolCreatePage, Details: map[string]any{"count": len(links[key])}}
		if rec != nil {
			f.Tool = ToolSetURLSEO
		}
		out = append(out, f)
	}
	return out
}

// inbound maps each address to the pages linking to it (each once, a page
// linking to itself left out), in crawl order.
func inbound(res *Result) map[string][]string {
	out := map[string][]string{}
	if res.fetcher == nil {
		return out
	}
	for _, from := range res.Order {
		p := res.Pages[from]
		if p.Facts == nil {
			continue
		}
		seen := map[string]bool{}
		for _, l := range p.Facts.Links {
			to := res.fetcher.Key(l.URL)
			if to == "" || to == from || seen[to] {
				continue
			}
			seen[to] = true
			out[to] = append(out[to], from)
		}
	}
	return out
}

// checkDuplicates reports titles and descriptions shared by indexable
// pages of the crawl (compared without case and extra spaces).
func checkDuplicates(res *Result, env *Env) []Finding {
	titles := map[string][]string{}
	descs := map[string][]string{}
	for _, key := range res.Order {
		p := res.Pages[key]
		if !env.indexable(res, p) {
			continue
		}
		if t := normal(p.Facts.Title); t != "" {
			titles[t] = append(titles[t], key)
		}
		if d := normal(p.Facts.Description); d != "" {
			descs[d] = append(descs[d], key)
		}
	}
	var out []Finding
	emit := func(groups map[string][]string, kind string) {
		var values []string
		for v := range groups {
			values = append(values, v)
		}
		sort.Strings(values)
		for _, v := range values {
			list := groups[v]
			if len(list) < 2 {
				continue
			}
			for _, key := range list {
				others := make([]string, 0, len(list)-1)
				for _, o := range list {
					if o != key {
						others = append(others, o)
					}
				}
				rec := env.record(key)
				out = append(out, Finding{Kind: kind, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"value": v, "count": len(others), "also": others}})
			}
		}
	}
	emit(titles, KindDuplicateTitle)
	emit(descs, KindDuplicateDesc)
	return out
}

// publisherTypes are the node types the SEO settings generate (publisher,
// local business, the site): their missing properties are fixed through
// seo_update_settings.
var publisherTypes = map[string]bool{
	"Organization": true, "Person": true, "WebSite": true, "LocalBusiness": true,
	"ProfessionalService": true, "Store": true, "FoodEstablishment": true,
	"HealthAndBeautyBusiness": true, "MedicalBusiness": true, "LegalService": true,
	"HomeAndConstructionBusiness": true, "AutomotiveBusiness": true, "FinancialService": true,
}

// checkJSONLD reads every JSON-LD script of a page: jsonld_invalid for one
// that is not JSON, jsonld_multiple for more than one script, and the
// graph validated with the rules of the admin preview (schema.Validate,
// D-46): jsonld_required and jsonld_recommended. Types without Google
// rules (FAQPage and the like) find nothing.
func checkJSONLD(p *Page, rec *Record) []Finding {
	scripts := p.Facts.JSONLD
	if len(scripts) == 0 {
		return nil
	}
	loc := locale(rec)
	var out []Finding
	var errs []string
	var nodes []any
	for _, raw := range scripts {
		var v any
		dec := json.NewDecoder(strings.NewReader(raw))
		dec.UseNumber()
		if err := dec.Decode(&v); err != nil {
			errs = append(errs, err.Error())
			continue
		}
		switch x := v.(type) {
		case map[string]any:
			if g, ok := x["@graph"].([]any); ok {
				nodes = append(nodes, g...)
			} else {
				nodes = append(nodes, x)
			}
		case []any:
			nodes = append(nodes, x...)
		}
	}
	if len(errs) > 0 {
		f := Finding{Kind: KindJSONLDInvalid, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"error": errs[0], "count": len(errs)}}
		if rec != nil && rec.CustomJSONLD {
			f.Tool = ToolSetURLSEO
		}
		out = append(out, f)
	}
	if len(scripts) > 1 {
		out = append(out, Finding{Kind: KindJSONLDMultiple, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(scripts)}})
	}
	if len(nodes) == 0 {
		return out
	}
	type miss struct{ typ, prop string }
	var required, recommended []miss
	for _, rep := range schema.Validate(map[string]any{"@graph": nodes}) {
		for _, f := range rep.Findings {
			m := miss{rep.Type, f.Property}
			if f.Level == schema.LevelRequired {
				required = append(required, m)
			} else {
				recommended = append(recommended, m)
			}
		}
	}
	emit := func(kind string, list []miss) {
		if len(list) == 0 {
			return
		}
		all := make([]map[string]any, 0, len(list))
		for _, m := range list {
			all = append(all, map[string]any{"type": m.typ, "property": m.prop})
		}
		tool := ToolSetURLSEO
		if publisherTypes[list[0].typ] {
			tool = ToolUpdateSettings
		}
		out = append(out, Finding{Kind: kind, URL: p.Key, Locale: loc, Record: rec, Tool: tool, Details: map[string]any{
			"type": list[0].typ, "property": list[0].prop, "count": len(list), "missing": all,
		}})
	}
	emit(KindJSONLDRequired, required)
	emit(KindJSONLDRecommended, recommended)
	return out
}

// checkHreflang follows a page's hreflang alternates: hreflang_error for
// an alternate that does not answer 200, hreflang_reciprocal for one that
// answers but does not link back.
func checkHreflang(res *Result, p *Page, rec *Record) []Finding {
	if res.fetcher == nil {
		return nil
	}
	loc := locale(rec)
	var broken, oneWay []map[string]any
	for _, a := range p.Facts.Alternates {
		key := res.fetcher.Key(a.URL)
		if key == "" || key == p.Key || strings.EqualFold(a.Lang, "x-default") {
			continue
		}
		t := res.Page(key)
		if t == nil || t.Resp == nil || t.Resp.Err == "canceled" {
			continue
		}
		if !t.OK() {
			broken = append(broken, map[string]any{"locale": a.Lang, "url": key, "status": t.Resp.Status})
			continue
		}
		back := false
		for _, b := range t.Facts.Alternates {
			if res.fetcher.Key(b.URL) == p.Key {
				back = true
				break
			}
		}
		if !back {
			oneWay = append(oneWay, map[string]any{"locale": a.Lang, "url": key})
		}
	}
	var out []Finding
	if len(broken) > 0 {
		d := map[string]any{"count": len(broken), "alternates": broken}
		for k, v := range broken[0] {
			d[k] = v
		}
		out = append(out, Finding{Kind: KindHreflangError, URL: p.Key, Locale: loc, Record: rec, Tool: ToolCreateRedirect, Details: d})
	}
	if len(oneWay) > 0 {
		d := map[string]any{"count": len(oneWay), "alternates": oneWay}
		for k, v := range oneWay[0] {
			d[k] = v
		}
		f := Finding{Kind: KindHreflangRecip, URL: p.Key, Locale: loc, Record: rec, Details: d}
		if rec != nil {
			f.Content = []string{pact.ContentToolEdit}
		}
		out = append(out, f)
	}
	return out
}

// maxListed caps the images a finding names.
const maxListed = 10

// checkImages reports image_alt (an <img> without an alt attribute; an
// empty alt is a decorative image), image_size (no width or height) and
// featured_lazy (the og:image loads lazily). An image the record lists is
// fixed through the record's image_alt tool.
func checkImages(p *Page, rec *Record) []Finding {
	loc := locale(rec)
	var noAlt, noSize []string
	var lazy string
	recordImage := false
	for _, img := range p.Facts.Images {
		if !img.HasAlt {
			noAlt = append(noAlt, img.Src)
			if rec != nil {
				for _, ri := range rec.Images {
					if ri.URL != "" && (strings.HasSuffix(img.Src, ri.URL) || ri.URL == img.Src) {
						recordImage = true
					}
				}
			}
		}
		if strings.TrimSpace(img.Width) == "" || strings.TrimSpace(img.Height) == "" {
			noSize = append(noSize, img.Src)
		}
		if p.Facts.OGImage != "" && img.Src == p.Facts.OGImage && img.Loading == "lazy" {
			lazy = img.Src
		}
	}
	cut := func(list []string) []string {
		if len(list) > maxListed {
			return list[:maxListed]
		}
		return list
	}
	var out []Finding
	if len(noAlt) > 0 {
		f := Finding{Kind: KindImageAlt, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(noAlt), "images": cut(noAlt)}}
		if recordImage {
			f.Content = []string{pact.ContentToolImageAlt, pact.ContentToolEdit}
		}
		out = append(out, f)
	}
	if len(noSize) > 0 {
		out = append(out, Finding{Kind: KindImageSize, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(noSize), "images": cut(noSize)}})
	}
	if lazy != "" {
		out = append(out, Finding{Kind: KindFeaturedLazy, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"image": lazy}})
	}
	return out
}

// checkContentCMS compares the server-rendered HTML with what the CMS
// record says (D-70, D-71; check cms): the record title must appear in an
// H1, the text must reach half of the record's word count, and the
// record's CMS-known links must be present. A miss is content_js.
func checkContentCMS(res *Result, p *Page, rec *Record) []Finding {
	if rec == nil {
		return nil
	}
	var what []string
	details := map[string]any{}
	if title := normal(rec.Title); title != "" {
		found := false
		for _, h := range p.Facts.H1() {
			if strings.Contains(normal(h), title) {
				found = true
			}
		}
		if !found {
			details["title"] = rec.Title
			what = append(what, "the title “"+rec.Title+"” in the H1")
		}
	}
	if rec.Words != nil && *rec.Words > 0 && p.Facts.Words*2 < *rec.Words {
		details["words_raw"] = p.Facts.Words
		details["words_cms"] = *rec.Words
		what = append(what, strconv.Itoa(p.Facts.Words)+" of "+strconv.Itoa(*rec.Words)+" words")
	}
	if len(rec.Links) > 0 && res.fetcher != nil {
		have := map[string]bool{}
		for _, l := range p.Facts.Links {
			if k := res.fetcher.Key(l.URL); k != "" {
				have[k] = true
			}
		}
		var missing []string
		for _, l := range rec.Links {
			if !have[l] {
				missing = append(missing, l)
			}
		}
		if len(missing) > 0 {
			details["links"] = missing
			what = append(what, "links "+strings.Join(missing, ", "))
		}
	}
	if len(what) == 0 {
		return nil
	}
	return []Finding{{Kind: KindContentJS, URL: p.Key, Locale: locale(rec), Record: rec, Check: checkCMS, Details: details, What: strings.Join(what, "; ")}}
}

// checkRenderedPage runs the raw versus rendered comparison of a page the
// renderer rendered.
func checkRenderedPage(res *Result, p *Page, rec *Record) []Finding {
	if p.Rendered == nil || res.fetcher == nil {
		return nil
	}
	out := compareRendered(p.Facts, p.Rendered, res.fetcher.Hosts())
	for i := range out {
		out[i].URL, out[i].Locale, out[i].Record = p.Key, locale(rec), rec
	}
	return out
}