feat: crawler checks for status, sitemaps, canonical, titles and headings

- page_status, sitemap_error, sitemap_noindex, sitemap_blocked and
  sitemap_missing against the fetched sitemap files and robots.txt
- canonical_missing and canonical_error, missing and duplicate titles and
  descriptions over the crawl
- h1_missing, h1_multiple, heading_skipped, heading_empty and
  h1_title_mismatch; content headings route to the record's tool
- the staging lock's noindex is never counted (Pitfall 12)
- TestCrawlerChecks on the planted site with a clean control page

classes/crawl/checks.go

@@ -1,8 +1,13 @@
 package crawl
 
 import (
+	"net/url"
+	"sort"
+	"strings"
+
 	"git.golem15.com/golem15/sm-seo-plugin/models"
 	"git.golem15.com/golem15/summercms/modules/pact"
+	"github.com/jimsmart/grobotstxt"
 )
 
 // The crawl kinds of UI-SPEC C23.
@@ -107,10 +112,16 @@ func Kinds() []string {
 }
 
 // Env is what the checks know besides the crawl: the record behind an
-// address.
+// address, whether the site runs outside production (the staging lock's
+// noindex is never reported, RESEARCH Pitfall 12) and the generic link
+// texts.
 type Env struct {
 	// Lookup returns the CMS record of an address, nil when none.
 	Lookup func(key string) *Record
+	// Staging is true outside production.
+	Staging bool
+	// GenericAnchors are the generic link texts (any case).
+	GenericAnchors []string
 }
 
 func (e *Env) record(key string) *Record {
@@ -120,32 +131,312 @@ func (e *Env) record(key string) *Record {
 	return e.Lookup(key)
 }
 
-// Check runs the per-page checks over a crawl's pages.
+// locale is the issue locale of a page: its record's.
+func locale(rec *Record) string {
+	if rec == nil {
+		return ""
+	}
+	return rec.Locale
+}
+
+// noindex reports whether a page asks not to be indexed (meta robots or
+// X-Robots-Tag). On a non-production environment every page carries the
+// staging lock's noindex, so it is never counted there.
+func (e *Env) noindex(p *Page) bool {
+	if e != nil && e.Staging {
+		return false
+	}
+	if p.Facts != nil && p.Facts.Noindex() {
+		return true
+	}
+	return p.Resp != nil && hasNoindex(p.Resp.XRobots)
+}
+
+// selfCanonical reports whether a page's canonical is missing or names
+// the page itself.
+func selfCanonical(res *Result, p *Page) bool {
+	if p.Facts == nil || p.Facts.Canonical == "" || res.fetcher == nil {
+		return true
+	}
+	u, err := url.Parse(p.Facts.Canonical)
+	if err != nil {
+		return true
+	}
+	return res.fetcher.Key(u) == p.Key
+}
+
+// indexable reports whether a page answers 200 with HTML, may be indexed
+// and is its own canonical.
+func (e *Env) indexable(res *Result, p *Page) bool {
+	return p.OK() && !e.noindex(p) && selfCanonical(res, p)
+}
+
+// Check runs every check over a crawl's pages: the per-page checks of
+// UI-SPEC C23 and the site-wide ones (sitemaps, statuses, duplicates).
 func Check(res *Result, env *Env) []Finding {
+	if env == nil {
+		env = &Env{}
+	}
 	var out []Finding
 	for _, key := range res.Order {
 		p := res.Pages[key]
+		rec := env.record(key)
+		out = append(out, checkStatus(res, p, rec)...)
 		if !p.OK() {
 			continue
 		}
-		out = append(out, checkHeadings(p, env.record(key))...)
+		out = append(out, checkHead(res, p, rec)...)
+		out = append(out, checkHeadings(p, rec)...)
+	}
+	out = append(out, checkSitemaps(res, env)...)
+	out = append(out, checkDuplicates(res, env)...)
+	return out
+}
+
+// checkStatus reports page_status for an address that answers an error
+// (other than 404 and 410, which are broken links when linked) or cannot
+// be fetched.
+func checkStatus(res *Result, p *Page, rec *Record) []Finding {
+	r := p.Resp
+	if r == nil {
+		return nil
+	}
+	switch {
+	case r.Err != "" && r.Err != "canceled":
+		return []Finding{{Kind: KindPageStatus, URL: p.Key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": 0, "error": r.Err}}}
+	case r.Status >= 400 && r.Status != 404 && r.Status != 410:
+		return []Finding{{Kind: KindPageStatus, URL: p.Key, Locale: locale(rec), Record: rec, Tool: ToolCreateRedirect, Details: map[string]any{"status": r.Status}}}
+	}
+	return nil
+}
+
+// checkHead reports a missing title, description or canonical and a
+// canonical pointing to an error page.
+func checkHead(res *Result, p *Page, rec *Record) []Finding {
+	var out []Finding
+	f := p.Facts
+	loc := locale(rec)
+	if strings.TrimSpace(f.Title) == "" {
+		out = append(out, Finding{Kind: KindMissingTitle, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO})
+	}
+	if strings.TrimSpace(f.Description) == "" {
+		out = append(out, Finding{Kind: KindMissingDesc, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO})
+	}
+	if f.Canonical == "" {
+		out = append(out, Finding{Kind: KindCanonicalMissing, URL: p.Key, Locale: loc, Record: rec})
+	} else if u, err := url.Parse(f.Canonical); err == nil && res.fetcher != nil {
+		if key := res.fetcher.Key(u); key != "" && key != p.Key {
+			if t := res.Page(key); t != nil && t.Resp != nil && (t.Resp.Err != "" || t.Resp.Status >= 400) {
+				out = append(out, Finding{Kind: KindCanonicalError, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"url": key, "status": t.Resp.Status}})
+			}
+		}
+	}
+	if h1 := f.H1(); len(h1) > 0 && strings.TrimSpace(f.Title) != "" && strings.TrimSpace(h1[0]) != "" {
+		a, b := normal(h1[0]), normal(f.Title)
+		if !strings.Contains(b, a) && !strings.Contains(a, b) {
+			out = append(out, Finding{Kind: KindH1TitleMismatch, URL: p.Key, Locale: loc, Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"h1": h1[0], "title": f.Title}})
+		}
 	}
 	return out
 }
 
-// checkHeadings reports h1_missing and h1_multiple. An H1 inside the
-// record's own content is fixed through the record's headings tool.
+// checkHeadings reports h1_missing, h1_multiple, heading_skipped and
+// heading_empty. A heading inside the record's own content is fixed
+// through the record's tool: H1 findings prefer Tools[headings], the
+// others use Tools[edit] (P33-38).
 func checkHeadings(p *Page, rec *Record) []Finding {
+	var out []Finding
+	loc := locale(rec)
 	h1 := p.Facts.H1()
 	switch {
 	case len(h1) == 0:
-		return []Finding{{Kind: KindH1Missing, URL: p.Key, Record: rec}}
+		out = append(out, Finding{Kind: KindH1Missing, URL: p.Key, Locale: loc, Record: rec})
 	case len(h1) > 1:
-		f := Finding{Kind: KindH1Multiple, URL: p.Key, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}}
+		f := Finding{Kind: KindH1Multiple, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}}
 		if rec.hasHeading(1, "") {
 			f.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
 		}
-		return []Finding{f}
+		out = append(out, f)
 	}
-	return nil
+	var skips []map[string]any
+	var first *Finding
+	prev := 0
+	for _, h := range p.Facts.Headings {
+		if prev > 0 && h.Level > prev+1 {
+			skips = append(skips, map[string]any{"from": prev, "to": h.Level, "text": h.Text})
+			if first == nil {
+				first = &Finding{Kind: KindHeadingSkipped, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"from": prev, "to": h.Level, "text": h.Text}}
+				if rec.hasHeading(h.Level, h.Text) {
+					first.Content = []string{pact.ContentToolEdit}
+				}
+			}
+		}
+		prev = h.Level
+	}
+	if first != nil {
+		first.Details["skips"] = skips
+		first.Details["count"] = len(skips)
+		out = append(out, *first)
+	}
+	empty := 0
+	var firstEmpty *Finding
+	for _, h := range p.Facts.Headings {
+		if strings.TrimSpace(h.Text) != "" {
+			continue
+		}
+		empty++
+		if firstEmpty == nil {
+			firstEmpty = &Finding{Kind: KindHeadingEmpty, URL: p.Key, Locale: loc, Record: rec, Details: map[string]any{"level": h.Level}}
+			if hasEmptyHeading(rec, h.Level) {
+				if h.Level == 1 {
+					firstEmpty.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
+				} else {
+					firstEmpty.Content = []string{pact.ContentToolEdit}
+				}
+			}
+		}
+	}
+	if firstEmpty != nil {
+		firstEmpty.Details["count"] = empty
+		out = append(out, *firstEmpty)
+	}
+	return out
+}
+
+// hasEmptyHeading reports whether the record's own headings hold an empty
+// one of level.
+func hasEmptyHeading(rec *Record, level int) bool {
+	if rec == nil {
+		return false
+	}
+	for _, h := range rec.Headings {
+		if h.Level == level && strings.TrimSpace(h.Text) == "" {
+			return true
+		}
+	}
+	return false
+}
+
+// checkSitemaps compares the sitemaps with what the crawl found:
+// sitemap_error for a listed address that does not answer 200,
+// sitemap_noindex for a listed noindex page, sitemap_blocked for a listed
+// address robots.txt keeps Googlebot from, and sitemap_missing for an
+// indexable page found only through links.
+func checkSitemaps(res *Result, env *Env) []Finding {
+	var out []Finding
+	var keys []string
+	for key := range res.Listed {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	robots := res.Robots
+	for _, key := range keys {
+		file := res.Listed[key]
+		rec := env.record(key)
+		if p := res.Page(key); p != nil && p.Resp != nil {
+			switch {
+			case p.Resp.Err == "canceled":
+			case p.Resp.Err != "" || p.Resp.Status != 200:
+				out = append(out, Finding{Kind: KindSitemapError, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"status": p.Resp.Status, "file": file}})
+			case env.noindex(p):
+				out = append(out, Finding{Kind: KindSitemapNoindex, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"file": file}})
+			}
+		}
+		if robots != "" && res.fetcher != nil {
+			m := grobotstxt.NewRobotsMatcher()
+			if !m.AgentAllowed(robots, "Googlebot", res.fetcher.Absolute(key)) {
+				line := m.MatchingLine()
+				rule := ""
+				if lines := strings.Split(robots, "\n"); line > 0 && line <= len(lines) {
+					rule = strings.TrimSpace(lines[line-1])
+				}
+				out = append(out, Finding{Kind: KindSitemapBlocked, URL: key, Locale: locale(rec), Record: rec, Tool: ToolUpdateRobots, Details: map[string]any{"line": line, "rule": rule, "file": file}})
+			}
+		}
+	}
+	links := inbound(res)
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		if _, listed := res.Listed[key]; listed || !env.indexable(res, p) {
+			continue
+		}
+		rec := env.record(key)
+		f := Finding{Kind: KindSitemapMissing, URL: key, Locale: locale(rec), Record: rec, Tool: ToolCreatePage, Details: map[string]any{"count": len(links[key])}}
+		if rec != nil {
+			f.Tool = ToolSetURLSEO
+		}
+		out = append(out, f)
+	}
+	return out
+}
+
+// inbound maps each address to the pages linking to it (each once, a page
+// linking to itself left out), in crawl order.
+func inbound(res *Result) map[string][]string {
+	out := map[string][]string{}
+	if res.fetcher == nil {
+		return out
+	}
+	for _, from := range res.Order {
+		p := res.Pages[from]
+		if p.Facts == nil {
+			continue
+		}
+		seen := map[string]bool{}
+		for _, l := range p.Facts.Links {
+			to := res.fetcher.Key(l.URL)
+			if to == "" || to == from || seen[to] {
+				continue
+			}
+			seen[to] = true
+			out[to] = append(out[to], from)
+		}
+	}
+	return out
+}
+
+// checkDuplicates reports titles and descriptions shared by indexable
+// pages of the crawl (compared without case and extra spaces).
+func checkDuplicates(res *Result, env *Env) []Finding {
+	titles := map[string][]string{}
+	descs := map[string][]string{}
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		if !env.indexable(res, p) {
+			continue
+		}
+		if t := normal(p.Facts.Title); t != "" {
+			titles[t] = append(titles[t], key)
+		}
+		if d := normal(p.Facts.Description); d != "" {
+			descs[d] = append(descs[d], key)
+		}
+	}
+	var out []Finding
+	emit := func(groups map[string][]string, kind string) {
+		var values []string
+		for v := range groups {
+			values = append(values, v)
+		}
+		sort.Strings(values)
+		for _, v := range values {
+			list := groups[v]
+			if len(list) < 2 {
+				continue
+			}
+			for _, key := range list {
+				others := make([]string, 0, len(list)-1)
+				for _, o := range list {
+					if o != key {
+						others = append(others, o)
+					}
+				}
+				rec := env.record(key)
+				out = append(out, Finding{Kind: kind, URL: key, Locale: locale(rec), Record: rec, Tool: ToolSetURLSEO, Details: map[string]any{"value": v, "count": len(others), "also": others}})
+			}
+		}
+	}
+	emit(titles, KindDuplicateTitle)
+	emit(descs, KindDuplicateDesc)
+	return out
 }

classes/crawl/job.go

@@ -221,7 +221,7 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
 		if err := SavePages(tx, row.ID, res); err != nil {
 			return err
 		}
-		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx)})
+		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging()})
 		cms, err := svc.Issues(classes.WithTx(ctx, tx))
 		if err != nil {
 			return err

crawl_site_test.go

@@ -6,6 +6,7 @@ import (
 	"html"
 	"io"
 	"net/http"
+	"net/url"
 	"slices"
 	"sort"
 	"strings"
@@ -415,3 +416,168 @@ func findIssue(t *testing.T, list []apiIssue, kind, url string) apiIssue {
 	}
 	return found[0]
 }
+
+// plantedPage is one page of the audit site: the document (or a raw
+// answer), whether a sitemap lists it, whether the home page links to it
+// and the issues it plants ("kind" each, at its own address).
+type plantedPage struct {
+	doc    doc
+	status int
+	header []string
+	raw    string
+	listed bool
+	linked bool
+	kinds  []string
+}
+
+// auditPages is the planted audit site: one page per kind of UI-SPEC C23
+// and the clean control page /clean.
+func auditPages() []plantedPage {
+	jsonld := func(v string) string { return `<script type="application/ld+json">` + v + `</script>` }
+	alt := func(lang, href string) string {
+		return `<link rel="alternate" hreflang="` + lang + `" href="` + siteURL + href + `">`
+	}
+	page := func(d doc, listed, linked bool, kinds ...string) plantedPage {
+		return plantedPage{doc: d, listed: listed, linked: linked, kinds: kinds}
+	}
+	return []plantedPage{
+		page(doc{path: "/clean", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"FAQPage","@id":"#faq"}]}`),
+			body: `<p>Plain text of the control page.</p><img src="/deco.png" alt="" width="10" height="10"><img src="/photo.jpg" alt="A photo" width="10" height="10">`}, true, true),
+		// Statuses and sitemaps.
+		{doc: doc{path: "/p/status-500"}, status: 500, linked: true, kinds: []string{"page_status"}},
+		{doc: doc{path: "/p/sitemap-404"}, status: 404, listed: true, kinds: []string{"sitemap_error"}},
+		page(doc{path: "/p/noindex", robots: "noindex, follow"}, true, true, "sitemap_noindex"),
+		page(doc{path: "/p/blocked"}, true, true, "sitemap_blocked"),
+		page(doc{path: "/p/unlisted"}, false, true, "sitemap_missing"),
+		// Head.
+		page(doc{path: "/p/no-canonical", noCanonical: true}, true, true, "canonical_missing"),
+		page(doc{path: "/p/bad-canonical", canonical: siteURL + "/p/canonical-404"}, true, true, "canonical_error"),
+		page(doc{path: "/p/no-title", noTitle: true, h1: []string{"No title page"}}, true, true, "missing_title"),
+		page(doc{path: "/p/dup-title-a", title: "Same title"}, true, true, "duplicate_title"),
+		page(doc{path: "/p/dup-title-b", title: "Same title"}, true, true, "duplicate_title"),
+		page(doc{path: "/p/no-desc", noDesc: true}, true, true, "missing_description"),
+		page(doc{path: "/p/dup-desc-a", description: "Same description"}, true, true, "duplicate_description"),
+		page(doc{path: "/p/dup-desc-b", description: "Same description"}, true, true, "duplicate_description"),
+		// Headings.
+		page(doc{path: "/p/no-h1", h1: []string{}, body: "<h2>Only a subheading</h2>"}, true, true, "h1_missing"),
+		page(doc{path: "/p/two-h1", h1: []string{"Page /p/two-h1", "Second"}}, true, true, "h1_multiple"),
+		page(doc{path: "/p/skip", body: "<h3>Deep</h3>"}, true, true, "heading_skipped"),
+		page(doc{path: "/p/empty-heading", body: "<h2></h2><p>Text</p>"}, true, true, "heading_empty"),
+		page(doc{path: "/p/mismatch", title: "Alpha page", h1: []string{"Completely different"}}, true, true, "h1_title_mismatch"),
+		// JSON-LD.
+		page(doc{path: "/p/jsonld-bad", head: jsonld(`{"@context": "https://schema.org", bad`)}, true, true, "jsonld_invalid"),
+		page(doc{path: "/p/jsonld-two", head: jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"A"}`) + jsonld(`{"@context":"https://schema.org","@type":"WebPage","name":"B"}`)}, true, true, "jsonld_multiple"),
+		page(doc{path: "/p/jsonld-req", head: jsonld(`{"@context":"https://schema.org","@graph":[{"@type":"BreadcrumbList","@id":"#crumbs"}]}`)}, true, true, "jsonld_required"),
+		page(doc{path: "/p/jsonld-rec", head: jsonld(`{"@context":"https://schema.org","@type":"Organization","name":"Acme","url":"https://www.example.test/"}`)}, true, true, "jsonld_recommended"),
+		// hreflang.
+		page(doc{path: "/en/x", head: alt("en", "/en/x") + alt("pl", "/pl/x")}, true, true, "hreflang_reciprocal"),
+		page(doc{path: "/pl/x", lang: "pl", head: alt("pl", "/pl/x")}, true, true),
+		page(doc{path: "/p/hreflang-err", head: alt("en", "/p/hreflang-err") + alt("de", "/de/missing")}, true, true, "hreflang_error"),
+		// Links.
+		page(doc{path: "/p/generic", body: `<p><a href="/clean">kliknij tutaj</a> and <a href="/clean">Click here</a></p>`}, true, true, "generic_anchor"),
+		{doc: doc{path: "/p/gone"}, status: 404, linked: true, kinds: []string{"broken_link"}},
+		{doc: doc{path: "/p/moved"}, status: 301, header: []string{"Location", "/clean"}, linked: true, kinds: []string{"redirected_link"}},
+		// Images.
+		page(doc{path: "/p/img-noalt", body: `<img src="/a.jpg" width="10" height="10">`}, true, true, "image_alt"),
+		page(doc{path: "/p/img-nosize", body: `<img src="/b.jpg" alt="B">`}, true, true, "image_size"),
+		page(doc{path: "/p/lazy", head: `<meta property="og:image" content="` + siteURL + `/hero.jpg">`, body: `<img src="/hero.jpg" alt="Hero" width="10" height="10" loading="lazy">`}, true, true, "featured_lazy"),
+		// Raw HTML against the CMS records (content_js, check cms).
+		page(doc{path: "/c/js-h1", title: "Template heading"}, true, true, "content_js"),
+		page(doc{path: "/c/js-words", title: "Words page", body: "<p>Just a few words here.</p>"}, true, true, "content_js"),
+		page(doc{path: "/c/js-links", title: "Linked post", links: []string{"/clean"}}, true, true, "content_js"),
+		page(doc{path: "/c/parent", title: "Parent post"}, true, true),
+		// Raw against rendered (content_js, check rendered).
+		page(doc{path: "/p/render", body: "<p>Short raw text.</p>"}, true, true, "content_js"),
+		page(doc{path: "/p/render-h1", h1: []string{}}, true, true, "h1_missing", "content_js"),
+		// Orphans.
+		page(doc{path: "/p/orphan", title: "Orphan topic"}, true, false, "orphan_page"),
+		page(doc{path: "/p/mentions", body: "<p>We wrote about the orphan topic before.</p>"}, true, true),
+	}
+}
+
+// auditRecords are the CMS records behind the /c/ pages.
+func auditRecords() map[string][]crawlRecord {
+	return map[string][]crawlRecord{crawlPostType: {
+		{id: "1", path: "/c/js-h1", title: "Record heading"},
+		{id: "2", path: "/c/js-words", title: "Words page", words: 400},
+		{id: "3", path: "/c/js-links", title: "Linked post", parents: []pact.ContentRef{{Type: crawlPostType, ID: "4"}}},
+		{id: "4", path: "/c/parent", title: "Parent post"},
+	}}
+}
+
+// plantAudit serves the audit site and returns the expected crawl issues
+// ("kind url", sorted).
+func plantAudit(site *fakeSite) []string {
+	var listed, linked, want []string
+	listed = append(listed, "/")
+	for _, p := range auditPages() {
+		switch {
+		case p.raw != "":
+			site.set(p.doc.path, p.status, p.raw, p.header...)
+		case p.status != 0:
+			site.set(p.doc.path, p.status, p.doc.render(), p.header...)
+		default:
+			site.plant(p.doc)
+		}
+		if p.listed {
+			listed = append(listed, p.doc.path)
+		}
+		if p.linked {
+			linked = append(linked, p.doc.path)
+		}
+		for _, k := range p.kinds {
+			want = append(want, k+" "+p.doc.path)
+		}
+	}
+	home := doc{path: "/", title: "Home", links: linked, body: `<p><a href="https://evil.example.com/x">Partner</a> <a href="http://169.254.169.254/latest">Metadata</a> <a href="http://www.example.test/plain">Plain http</a></p>`}
+	site.plant(home)
+	site.sitemap(listed...)
+	site.set("/robots.txt", http.StatusOK, "User-agent: *\nDisallow: /p/blocked\n", "Content-Type", "text/plain")
+	sort.Strings(want)
+	return want
+}
+
+// fakeRenderer is the test renderer of the comparison: the DOM planted
+// for a path, else the raw page (an identical rendering).
+type fakeRenderer struct {
+	site   *fakeSite
+	dom    map[string]string
+	fail   error
+	mu     sync.Mutex
+	calls  int
+	closed bool
+}
+
+func (r *fakeRenderer) Render(_ context.Context, pageURL string) (string, error) {
+	r.mu.Lock()
+	r.calls++
+	r.mu.Unlock()
+	if r.fail != nil {
+		return "", r.fail
+	}
+	u, err := url.Parse(pageURL)
+	if err != nil {
+		return "", err
+	}
+	if d, ok := r.dom[u.Path]; ok {
+		return d, nil
+	}
+	r.site.mu.Lock()
+	defer r.site.mu.Unlock()
+	return r.site.pages[u.Path].body, nil
+}
+
+func (r *fakeRenderer) Close() error {
+	r.closed = true
+	return nil
+}
+
+// auditRenderer renders /p/render with a link and 80 more words, and
+// /p/render-h1 with an H1.
+func auditRenderer(site *fakeSite) *fakeRenderer {
+	words := strings.Repeat("rendered word ", 40)
+	return &fakeRenderer{site: site, dom: map[string]string{
+		"/p/render":    doc{path: "/p/render", body: `<p>Short raw text.</p><p>` + words + `</p><a href="/p/js-only">Only after JavaScript</a>`}.render(),
+		"/p/render-h1": doc{path: "/p/render-h1"}.render(),
+	}}
+}

crawler_test.go

@@ -1,8 +1,11 @@
 package seo_test
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
+	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
 	"git.golem15.com/golem15/summercms/modules/pact"
 )
 
@@ -46,3 +49,95 @@ func TestCrawlerTracer(t *testing.T) {
 		}
 	}
 }
+
+// checkedKinds are the crawl kinds TestCrawlerChecks compares.
+var checkedKinds = map[string]bool{
+	"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
+	"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
+	"duplicate_title": true, "missing_description": true, "duplicate_description": true,
+	"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
+	"h1_title_mismatch": true,
+}
+
+// onlyChecked keeps the "kind url" entries of checkedKinds.
+func onlyChecked(list []string) []string {
+	out := []string{}
+	for _, e := range list {
+		kind, _, _ := strings.Cut(e, " ")
+		if checkedKinds[kind] {
+			out = append(out, e)
+		}
+	}
+	return out
+}
+
+// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
+// C23 and the clean control page /clean): every planted issue is found
+// exactly once with its severity and details, nothing is found on /clean,
+// and the crawler never requests another host, a private address or plain
+// http (T-33-43).
+func TestCrawlerChecks(t *testing.T) {
+	env := bootCrawlEnv(t)
+	token := env.admin(t, true)
+	setCrawlRecords(t, auditRecords())
+	site := newFakeSite()
+	want := plantAudit(site)
+
+	ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
+	env.crawlOnce(t, token, ctx)
+	list := env.issues(t, token, "source=crawl")
+
+	got := onlyChecked(issueSet(list, "crawl"))
+	if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
+		t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
+	}
+	for _, i := range list {
+		if i.URL == "/clean" {
+			t.Fatalf("issue on the control page: %+v", i)
+		}
+		if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
+			t.Fatalf("issue without a fix route: %+v", i)
+		}
+	}
+	for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
+		if site.requested(part) {
+			t.Fatalf("the crawler requested %s: %v", part, site.requests)
+		}
+	}
+
+	detail := func(kind, url, key string, value any) {
+		t.Helper()
+		i := findIssue(t, list, kind, url)
+		if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
+			t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
+		}
+	}
+	severity := func(kind, url, want string) {
+		t.Helper()
+		if i := findIssue(t, list, kind, url); i.Severity != want {
+			t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
+		}
+	}
+	detail("page_status", "/p/status-500", "status", 500)
+	severity("page_status", "/p/status-500", "error")
+	detail("sitemap_error", "/p/sitemap-404", "status", 404)
+	detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
+	detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
+	detail("sitemap_blocked", "/p/blocked", "line", 2)
+	detail("sitemap_missing", "/p/unlisted", "count", 1)
+	severity("sitemap_missing", "/p/unlisted", "notice")
+	detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
+	detail("canonical_error", "/p/bad-canonical", "status", 404)
+	detail("duplicate_title", "/p/dup-title-a", "count", 1)
+	detail("h1_multiple", "/p/two-h1", "count", 2)
+	detail("heading_skipped", "/p/skip", "from", 1)
+	detail("heading_skipped", "/p/skip", "to", 3)
+	detail("heading_empty", "/p/empty-heading", "level", 2)
+	detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
+	if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
+		t.Fatalf("canonical_missing = %+v", i)
+	}
+	if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
+		t.Fatalf("missing_title = %+v", i)
+	}
+}

lang/en/lang.yaml

@@ -525,3 +525,46 @@ crawls:
     started: Crawl started. Follow it in SEO › Crawls; the issues appear in Site audit when it finishes.
     already_running: A crawl is already running (started :time). Wait for it, or cancel it in SEO › Crawls.
     no_site_url: The site address for crawls is not configured (golem15.seo.site_url). Ask the administrator to set it.
+issues:
+    kind_page_status: "Page returns an error"
+    sum_page_status: "Status :status"
+    kind_sitemap_error: "Sitemap lists a page that does not answer 200"
+    sum_sitemap_error: "Status :status in :file"
+    kind_sitemap_noindex: "Sitemap lists a noindex page"
+    sum_sitemap_noindex: "In :file"
+    kind_sitemap_blocked: "Sitemap lists an address blocked in robots.txt"
+    sum_sitemap_blocked: "Blocked by line :line"
+    kind_sitemap_missing: "Indexable page missing from the sitemaps"
+    sum_sitemap_missing: "Found through :count links"
+    kind_canonical_missing: "No canonical link"
+    sum_canonical_missing: "—"
+    kind_canonical_error: "Canonical points to an error page"
+    sum_canonical_error: ":url answers :status"
+    kind_canonical_noindex: "Canonical points to a noindex page"
+    sum_canonical_noindex: ":url"
+    kind_canonical_redirected: "Canonical points to a redirect"
+    sum_canonical_redirected: ":url → :target"
+    kind_missing_title: "Missing title"
+    sum_missing_title: "—"
+    kind_duplicate_title: "Duplicate title"
+    sum_duplicate_title: "Same as :count other pages"
+    kind_missing_description: "Missing meta description"
+    sum_missing_description: "—"
+    kind_duplicate_description: "Duplicate meta description"
+    sum_duplicate_description: "Same as :count other pages"
+    kind_rule_overrides_record: "URL rule overrides the record"
+    sum_rule_overrides_record: ":fields"
+    kind_redirect_chain: "Redirect chain"
+    sum_redirect_chain: ":count hops to :final"
+    kind_noindex_blocked: "noindex page blocked in robots.txt"
+    sum_noindex_blocked: "Line :line"
+    kind_h1_missing: "No H1 heading"
+    sum_h1_missing: "—"
+    kind_h1_multiple: "More than one H1"
+    sum_h1_multiple: ":count H1 headings"
+    kind_heading_skipped: "Skipped heading level"
+    sum_heading_skipped: "H:from → H:to"
+    kind_heading_empty: "Empty heading"
+    sum_heading_empty: "H:level"
+    kind_h1_title_mismatch: "H1 does not match the title"
+    sum_h1_title_mismatch: "“:h1”"

lang/pl/lang.yaml

@@ -539,3 +539,46 @@ crawls:
     started: Rozpoczęto skanowanie. Postęp znajdziesz w SEO › Skanowania; problemy pojawią się w Audycie witryny po jego zakończeniu.
     already_running: Skanowanie już trwa (rozpoczęte :time). Poczekaj na nie lub anuluj je w SEO › Skanowania.
     no_site_url: Adres witryny do skanowania nie jest skonfigurowany (golem15.seo.site_url). Poproś administratora o jego ustawienie.
+issues:
+    kind_page_status: "Strona zwraca błąd"
+    sum_page_status: "Kod :status"
+    kind_sitemap_error: "Mapa witryny zawiera stronę bez kodu 200"
+    sum_sitemap_error: "Kod :status w :file"
+    kind_sitemap_noindex: "Mapa witryny zawiera stronę z noindex"
+    sum_sitemap_noindex: "W :file"
+    kind_sitemap_blocked: "Mapa witryny zawiera adres zablokowany w robots.txt"
+    sum_sitemap_blocked: "Blokowany przez wiersz :line"
+    kind_sitemap_missing: "Indeksowalna strona poza mapami witryny"
+    sum_sitemap_missing: "Znaleziona przez linki: :count"
+    kind_canonical_missing: "Brak linku kanonicznego"
+    sum_canonical_missing: "—"
+    kind_canonical_error: "Adres kanoniczny wskazuje stronę z błędem"
+    sum_canonical_error: ":url odpowiada kodem :status"
+    kind_canonical_noindex: "Adres kanoniczny wskazuje stronę z noindex"
+    sum_canonical_noindex: ":url"
+    kind_canonical_redirected: "Adres kanoniczny wskazuje przekierowanie"
+    sum_canonical_redirected: ":url → :target"
+    kind_missing_title: "Brak tytułu"
+    sum_missing_title: "—"
+    kind_duplicate_title: "Powielony tytuł"
+    sum_duplicate_title: "Taki sam jak na innych stronach: :count"
+    kind_missing_description: "Brak opisu meta"
+    sum_missing_description: "—"
+    kind_duplicate_description: "Powielony opis meta"
+    sum_duplicate_description: "Taki sam jak na innych stronach: :count"
+    kind_rule_overrides_record: "Reguła adresu nadpisuje rekord"
+    sum_rule_overrides_record: ":fields"
+    kind_redirect_chain: "Łańcuch przekierowań"
+    sum_redirect_chain: "Liczba przekierowań: :count, do :final"
+    kind_noindex_blocked: "Strona z noindex zablokowana w robots.txt"
+    sum_noindex_blocked: "Wiersz :line"
+    kind_h1_missing: "Brak nagłówka H1"
+    sum_h1_missing: "—"
+    kind_h1_multiple: "Więcej niż jeden nagłówek H1"
+    sum_h1_multiple: "Nagłówki H1: :count"
+    kind_heading_skipped: "Pominięty poziom nagłówka"
+    sum_heading_skipped: "H:from → H:to"
+    kind_heading_empty: "Pusty nagłówek"
+    sum_heading_empty: "H:level"
+    kind_h1_title_mismatch: "H1 nie pasuje do tytułu"
+    sum_h1_title_mismatch: "„:h1”"