feat: crawler checks for status, sitemaps, canonical, titles and headings

- page_status, sitemap_error, sitemap_noindex, sitemap_blocked and
  sitemap_missing against the fetched sitemap files and robots.txt
- canonical_missing and canonical_error, missing and duplicate titles and
  descriptions over the crawl
- h1_missing, h1_multiple, heading_skipped, heading_empty and
  h1_title_mismatch; content headings route to the record's tool
- the staging lock's noindex is never counted (Pitfall 12)
- TestCrawlerChecks on the planted site with a clean control page

crawler_test.go

@@ -1,8 +1,11 @@
 package seo_test
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
+	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
 	"git.golem15.com/golem15/summercms/modules/pact"
 )
 
@@ -46,3 +49,95 @@ func TestCrawlerTracer(t *testing.T) {
 		}
 	}
 }
+
+// checkedKinds are the crawl kinds TestCrawlerChecks compares.
+var checkedKinds = map[string]bool{
+	"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
+	"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
+	"duplicate_title": true, "missing_description": true, "duplicate_description": true,
+	"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
+	"h1_title_mismatch": true,
+}
+
+// onlyChecked keeps the "kind url" entries of checkedKinds.
+func onlyChecked(list []string) []string {
+	out := []string{}
+	for _, e := range list {
+		kind, _, _ := strings.Cut(e, " ")
+		if checkedKinds[kind] {
+			out = append(out, e)
+		}
+	}
+	return out
+}
+
+// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
+// C23 and the clean control page /clean): every planted issue is found
+// exactly once with its severity and details, nothing is found on /clean,
+// and the crawler never requests another host, a private address or plain
+// http (T-33-43).
+func TestCrawlerChecks(t *testing.T) {
+	env := bootCrawlEnv(t)
+	token := env.admin(t, true)
+	setCrawlRecords(t, auditRecords())
+	site := newFakeSite()
+	want := plantAudit(site)
+
+	ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
+	env.crawlOnce(t, token, ctx)
+	list := env.issues(t, token, "source=crawl")
+
+	got := onlyChecked(issueSet(list, "crawl"))
+	if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
+		t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
+	}
+	for _, i := range list {
+		if i.URL == "/clean" {
+			t.Fatalf("issue on the control page: %+v", i)
+		}
+		if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
+			t.Fatalf("issue without a fix route: %+v", i)
+		}
+	}
+	for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
+		if site.requested(part) {
+			t.Fatalf("the crawler requested %s: %v", part, site.requests)
+		}
+	}
+
+	detail := func(kind, url, key string, value any) {
+		t.Helper()
+		i := findIssue(t, list, kind, url)
+		if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
+			t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
+		}
+	}
+	severity := func(kind, url, want string) {
+		t.Helper()
+		if i := findIssue(t, list, kind, url); i.Severity != want {
+			t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
+		}
+	}
+	detail("page_status", "/p/status-500", "status", 500)
+	severity("page_status", "/p/status-500", "error")
+	detail("sitemap_error", "/p/sitemap-404", "status", 404)
+	detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
+	detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
+	detail("sitemap_blocked", "/p/blocked", "line", 2)
+	detail("sitemap_missing", "/p/unlisted", "count", 1)
+	severity("sitemap_missing", "/p/unlisted", "notice")
+	detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
+	detail("canonical_error", "/p/bad-canonical", "status", 404)
+	detail("duplicate_title", "/p/dup-title-a", "count", 1)
+	detail("h1_multiple", "/p/two-h1", "count", 2)
+	detail("heading_skipped", "/p/skip", "from", 1)
+	detail("heading_skipped", "/p/skip", "to", 3)
+	detail("heading_empty", "/p/empty-heading", "level", 2)
+	detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
+	if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
+		t.Fatalf("canonical_missing = %+v", i)
+	}
+	if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
+		t.Fatalf("missing_title = %+v", i)
+	}
+}