feat: crawler finds a page with two H1 headings and stores the issue (tracer)

- golem15_seo_crawls, crawl pages, links, issues and ignores (P33-37) - classes/crawl: fetchguard client on the site host and its www twin, sitemap index and robots.txt seeds, two workers with a delay, limits - h1_missing and h1_multiple with the P33-38 fix route - POST /_seo/api/v1/crawls queues the golem15.seo.crawl job; GET /issues answers the stored issues with id, severity, source, fix, found_at, ignored and crawl_id
2026-11-04 15:05:45 +01:00
parent d38049a956
commit c2b61bf8e9
23 changed files with 2967 additions and 13 deletions
--- a/classes/crawl/checks.go
+++ b/classes/crawl/checks.go
@@ -0,0 +1,151 @@
+package crawl
+
+import (
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/pact"
+)
+
+// The crawl kinds of UI-SPEC C23.
+const (
+	KindPageStatus        = "page_status"
+	KindSitemapError      = "sitemap_error"
+	KindSitemapNoindex    = "sitemap_noindex"
+	KindSitemapBlocked    = "sitemap_blocked"
+	KindSitemapMissing    = "sitemap_missing"
+	KindBrokenLink        = "broken_link"
+	KindRedirectedLink    = "redirected_link"
+	KindCanonicalMissing  = "canonical_missing"
+	KindCanonicalError    = "canonical_error"
+	KindMissingTitle      = "missing_title"
+	KindDuplicateTitle    = "duplicate_title"
+	KindMissingDesc       = "missing_description"
+	KindDuplicateDesc     = "duplicate_description"
+	KindH1Missing         = "h1_missing"
+	KindH1Multiple        = "h1_multiple"
+	KindHeadingSkipped    = "heading_skipped"
+	KindHeadingEmpty      = "heading_empty"
+	KindH1TitleMismatch   = "h1_title_mismatch"
+	KindJSONLDInvalid     = "jsonld_invalid"
+	KindJSONLDMultiple    = "jsonld_multiple"
+	KindJSONLDRequired    = "jsonld_required"
+	KindJSONLDRecommended = "jsonld_recommended"
+	KindHreflangRecip     = "hreflang_reciprocal"
+	KindHreflangError     = "hreflang_error"
+	KindGenericAnchor     = "generic_anchor"
+	KindImageAlt          = "image_alt"
+	KindImageSize         = "image_size"
+	KindFeaturedLazy      = "featured_lazy"
+	KindContentJS         = "content_js"
+	KindOrphanPage        = "orphan_page"
+	// KindH1InContent is the CMS-data check of a record whose body holds
+	// an H1 although its page renders the title as the H1.
+	KindH1InContent = "h1_in_content"
+)
+
+// severities are the default severities of the kinds table (UI-SPEC C23),
+// the Phase 32 CMS kinds included.
+var severities = map[string]string{
+	KindPageStatus:          models.SeverityError,
+	KindSitemapError:        models.SeverityError,
+	KindSitemapNoindex:      models.SeverityError,
+	KindSitemapBlocked:      models.SeverityError,
+	KindSitemapMissing:      models.SeverityNotice,
+	KindBrokenLink:          models.SeverityError,
+	KindRedirectedLink:      models.SeverityNotice,
+	KindCanonicalMissing:    models.SeverityWarning,
+	KindCanonicalError:      models.SeverityError,
+	"canonical_noindex":     models.SeverityError,
+	"canonical_redirected":  models.SeverityWarning,
+	KindMissingTitle:        models.SeverityWarning,
+	KindDuplicateTitle:      models.SeverityWarning,
+	KindMissingDesc:         models.SeverityNotice,
+	KindDuplicateDesc:       models.SeverityNotice,
+	"rule_overrides_record": models.SeverityNotice,
+	"redirect_chain":        models.SeverityWarning,
+	"noindex_blocked":       models.SeverityError,
+	KindH1Missing:           models.SeverityError,
+	KindH1Multiple:          models.SeverityError,
+	KindH1InContent:         models.SeverityWarning,
+	KindHeadingSkipped:      models.SeverityWarning,
+	KindHeadingEmpty:        models.SeverityWarning,
+	KindH1TitleMismatch:     models.SeverityNotice,
+	KindJSONLDInvalid:       models.SeverityError,
+	KindJSONLDMultiple:      models.SeverityWarning,
+	KindJSONLDRequired:      models.SeverityError,
+	KindJSONLDRecommended:   models.SeverityNotice,
+	KindHreflangRecip:       models.SeverityWarning,
+	KindHreflangError:       models.SeverityError,
+	KindGenericAnchor:       models.SeverityWarning,
+	KindImageAlt:            models.SeverityWarning,
+	KindImageSize:           models.SeverityNotice,
+	KindFeaturedLazy:        models.SeverityNotice,
+	KindContentJS:           models.SeverityWarning,
+	KindOrphanPage:          models.SeverityWarning,
+}
+
+// Severity returns a kind's severity (warning for an unknown kind).
+func Severity(kind string) string {
+	if s, ok := severities[kind]; ok {
+		return s
+	}
+	return models.SeverityWarning
+}
+
+// Kinds lists every kind the audit stores, in the kinds table order.
+func Kinds() []string {
+	return []string{
+		KindPageStatus, KindSitemapError, KindSitemapNoindex, KindSitemapBlocked,
+		KindSitemapMissing, KindBrokenLink, KindRedirectedLink, KindCanonicalMissing,
+		KindCanonicalError, "canonical_noindex", "canonical_redirected", KindMissingTitle,
+		KindDuplicateTitle, KindMissingDesc, KindDuplicateDesc, "rule_overrides_record",
+		"redirect_chain", "noindex_blocked", KindH1Missing, KindH1Multiple, KindH1InContent,
+		KindHeadingSkipped, KindHeadingEmpty, KindH1TitleMismatch, KindJSONLDInvalid,
+		KindJSONLDMultiple, KindJSONLDRequired, KindJSONLDRecommended, KindHreflangRecip,
+		KindHreflangError, KindGenericAnchor, KindImageAlt, KindImageSize, KindFeaturedLazy,
+		KindContentJS, KindOrphanPage,
+	}
+}
+
+// Env is what the checks know besides the crawl: the record behind an
+// address.
+type Env struct {
+	// Lookup returns the CMS record of an address, nil when none.
+	Lookup func(key string) *Record
+}
+
+func (e *Env) record(key string) *Record {
+	if e == nil || e.Lookup == nil {
+		return nil
+	}
+	return e.Lookup(key)
+}
+
+// Check runs the per-page checks over a crawl's pages.
+func Check(res *Result, env *Env) []Finding {
+	var out []Finding
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		if !p.OK() {
+			continue
+		}
+		out = append(out, checkHeadings(p, env.record(key))...)
+	}
+	return out
+}
+
+// checkHeadings reports h1_missing and h1_multiple. An H1 inside the
+// record's own content is fixed through the record's headings tool.
+func checkHeadings(p *Page, rec *Record) []Finding {
+	h1 := p.Facts.H1()
+	switch {
+	case len(h1) == 0:
+		return []Finding{{Kind: KindH1Missing, URL: p.Key, Record: rec}}
+	case len(h1) > 1:
+		f := Finding{Kind: KindH1Multiple, URL: p.Key, Record: rec, Details: map[string]any{"count": len(h1), "h1": h1}}
+		if rec.hasHeading(1, "") {
+			f.Content = []string{pact.ContentToolHeadings, pact.ContentToolEdit}
+		}
+		return []Finding{f}
+	}
+	return nil
+}
--- a/classes/crawl/crawler.go
+++ b/classes/crawl/crawler.go
@@ -0,0 +1,425 @@
+package crawl
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"errors"
+	"io"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+// ErrNoSiteURL is a crawl started without a site address
+// (golem15.seo.site_url, else app.url).
+var ErrNoSiteURL = errors.New("golem15.seo: the site address for crawls is not configured")
+
+// The paths the crawler seeds from (RESEARCH Pattern 7).
+const (
+	HomePath    = "/"
+	SitemapPath = "/sitemap.xml"
+	RobotsPath  = "/robots.txt"
+)
+
+// maxSitemapFiles caps the sitemap files read from an index.
+const maxSitemapFiles = 500
+
+// How a page entered the frontier.
+const (
+	viaHome      = "home"
+	viaSitemap   = "sitemap"
+	viaLink      = "link"
+	viaRedirect  = "redirect"
+	viaCanonical = "canonical"
+	viaAlternate = "alternate"
+)
+
+// Page is one fetched address of a crawl: the answer, the facts of its raw
+// HTML, the facts of its JavaScript-rendered DOM when the comparison ran,
+// its depth from the home page and the sitemap file that lists it.
+type Page struct {
+	Key         string
+	Depth       int
+	Via         string
+	Resp        *Response
+	Facts       *PageFacts
+	Rendered    *PageFacts
+	InSitemap   bool
+	SitemapFile string
+	FetchedAt   time.Time
+}
+
+// OK reports whether the page answered 200 with HTML.
+func (p *Page) OK() bool {
+	return p != nil && p.Resp != nil && p.Resp.Err == "" && p.Resp.Status == 200 && p.Facts != nil
+}
+
+// SitemapFile is one sitemap the crawl read: its address, status and the
+// page addresses it lists.
+type SitemapFile struct {
+	Key    string
+	Status int
+	URLs   []string
+}
+
+// RenderState is the outcome of the rendered comparison (P33-55).
+type RenderState struct {
+	Status string
+	Pages  int
+	Note   string
+}
+
+// Result is what one crawl fetched, in frontier order.
+type Result struct {
+	Pages    map[string]*Page
+	Order    []string
+	Sitemaps []SitemapFile
+	// Listed maps a page address to the sitemap file listing it.
+	Listed       map[string]string
+	Robots       string
+	RobotsStatus int
+	LimitHit     bool
+	Canceled     bool
+	Render       RenderState
+	fetcher      *Fetcher
+}
+
+// Page returns the fetched page at key, nil when none.
+func (r *Result) Page(key string) *Page {
+	if r == nil {
+		return nil
+	}
+	return r.Pages[key]
+}
+
+// Options are the limits and seams of one crawl.
+type Options struct {
+	// MaxPages caps the fetched addresses (D-14, default 5000).
+	MaxPages int
+	// Delay is the pause between two requests (default 250 ms).
+	Delay time.Duration
+	// Renderer renders pages for the comparison; nil skips it.
+	Renderer Renderer
+	// RenderMax caps the rendered pages (default 200, crawl order).
+	RenderMax int
+	// Canceled is asked between requests; true stops the crawl and keeps
+	// what was fetched.
+	Canceled func(ctx context.Context) bool
+	// Progress is told the number of fetched pages.
+	Progress func(ctx context.Context, fetched int)
+}
+
+// item is one frontier entry.
+type item struct {
+	key   string
+	depth int
+	via   string
+}
+
+// pacer spaces requests at least delay apart across the workers.
+type pacer struct {
+	mu    sync.Mutex
+	next  time.Time
+	delay time.Duration
+}
+
+func (p *pacer) wait(ctx context.Context) error {
+	p.mu.Lock()
+	now := time.Now()
+	at := p.next
+	if at.Before(now) {
+		at = now
+	}
+	p.next = at.Add(p.delay)
+	p.mu.Unlock()
+	if d := time.Until(at); d > 0 {
+		t := time.NewTimer(d)
+		defer t.Stop()
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-t.C:
+		}
+	}
+	return nil
+}
+
+// Crawl fetches the site through f, a fetchguard client limited to the
+// site host and its www twin (fetchguard.AllowHostsMode, HTTPS, no
+// redirects followed; TrustedMode only from Go config): robots.txt, the
+// sitemap index and its files, then breadth-first from the home page and
+// the sitemap addresses over same-host links, redirect targets, canonicals
+// and hreflang alternates, two requests at a time with the delay between
+// requests, up to MaxPages. robots.txt rules are read for the report, not
+// obeyed (P33-41). Cancelling keeps what was fetched. After each HTML page
+// answering 200, a configured Renderer renders it (one tab at a time, up
+// to RenderMax pages) for the raw versus rendered comparison.
+func Crawl(ctx context.Context, f *Fetcher, opt Options) (*Result, error) {
+	if opt.MaxPages <= 0 {
+		opt.MaxPages = 5000
+	}
+	if opt.RenderMax <= 0 {
+		opt.RenderMax = DefaultRenderMax
+	}
+	res := &Result{Pages: map[string]*Page{}, Listed: map[string]string{}, Render: RenderState{Status: renderStatusSkipped}, fetcher: f}
+	if opt.Renderer != nil {
+		res.Render.Status = renderStatusDone
+	}
+	pace := &pacer{delay: opt.Delay}
+	canceled := func() bool {
+		if ctx.Err() != nil {
+			return true
+		}
+		return opt.Canceled != nil && opt.Canceled(ctx)
+	}
+
+	// robots.txt, for the sitemap_blocked check.
+	if err := pace.wait(ctx); err != nil {
+		return res, err
+	}
+	if r := f.Get(ctx, RobotsPath); r.Err == "" {
+		res.RobotsStatus = r.Status
+		if r.Status == 200 {
+			res.Robots = string(r.Body)
+		}
+	}
+
+	// The sitemap index and its files.
+	if err := readSitemaps(ctx, f, pace, res); err != nil {
+		return res, err
+	}
+
+	seen := map[string]bool{}
+	var queue []item
+	push := func(key string, depth int, via string) {
+		if key == "" || seen[key] {
+			return
+		}
+		seen[key] = true
+		queue = append(queue, item{key: key, depth: depth, via: via})
+	}
+	push(HomePath, 0, viaHome)
+	for _, sm := range res.Sitemaps {
+		for _, key := range sm.URLs {
+			push(key, 1, viaSitemap)
+		}
+	}
+
+	var renderMu sync.Mutex
+	rendered := 0
+	renderOff := opt.Renderer == nil
+	for len(queue) > 0 {
+		if canceled() {
+			res.Canceled = true
+			break
+		}
+		if len(res.Order) >= opt.MaxPages {
+			res.LimitHit = true
+			break
+		}
+		n := Workers
+		if room := opt.MaxPages - len(res.Order); n > room {
+			n = room
+		}
+		if n > len(queue) {
+			n = len(queue)
+		}
+		batch := queue[:n]
+		queue = queue[n:]
+		pages := make([]*Page, len(batch))
+		var wg sync.WaitGroup
+		for i, it := range batch {
+			wg.Add(1)
+			go func(i int, it item) {
+				defer wg.Done()
+				p := &Page{Key: it.key, Depth: it.depth, Via: it.via}
+				if err := pace.wait(ctx); err != nil {
+					p.Resp = &Response{Err: "canceled"}
+					pages[i] = p
+					return
+				}
+				p.Resp = f.Get(ctx, it.key)
+				p.FetchedAt = time.Now().UTC()
+				if p.Resp.Err == "" && p.Resp.Status == 200 && p.Resp.HTML() {
+					base, _ := url.Parse(f.Absolute(it.key))
+					p.Facts = Parse(base, p.Resp.Body)
+					renderMu.Lock()
+					if !renderOff && rendered < opt.RenderMax && ctx.Err() == nil {
+						rendered++
+						dom, err := opt.Renderer.Render(ctx, f.Absolute(it.key))
+						if err != nil {
+							renderOff = true
+							rendered--
+							res.Render = RenderState{Status: renderStatusFailed, Note: err.Error()}
+						} else {
+							p.Rendered = Parse(base, []byte(dom))
+						}
+					}
+					renderMu.Unlock()
+				}
+				pages[i] = p
+			}(i, it)
+		}
+		wg.Wait()
+		for _, p := range pages {
+			if file, ok := res.Listed[p.Key]; ok {
+				p.InSitemap, p.SitemapFile = true, file
+			}
+			res.Pages[p.Key] = p
+			res.Order = append(res.Order, p.Key)
+			p.Resp.Body = nil
+			for _, next := range discovered(f, p) {
+				push(next.key, p.Depth+1, next.via)
+			}
+		}
+		if opt.Progress != nil {
+			opt.Progress(ctx, len(res.Order))
+		}
+	}
+	if len(queue) > 0 && !res.Canceled && len(res.Order) >= opt.MaxPages {
+		res.LimitHit = true
+	}
+	res.Render.Pages = rendered
+	return res, nil
+}
+
+// discovered lists the same-host addresses a fetched page leads to: its
+// redirect target, its links, canonical and hreflang alternates.
+func discovered(f *Fetcher, p *Page) []item {
+	var out []item
+	add := func(raw string, via string) {
+		u, err := url.Parse(raw)
+		if err != nil {
+			return
+		}
+		u.Fragment = ""
+		if key := f.Key(u); key != "" {
+			out = append(out, item{key: key, via: via})
+		}
+	}
+	if p.Resp != nil && p.Resp.Location != "" {
+		add(p.Resp.Location, viaRedirect)
+	}
+	if p.Facts == nil {
+		return out
+	}
+	for _, l := range p.Facts.Links {
+		if l.URL != nil {
+			add(l.URL.String(), viaLink)
+		}
+	}
+	if p.Facts.Canonical != "" {
+		add(p.Facts.Canonical, viaCanonical)
+	}
+	for _, a := range p.Facts.Alternates {
+		if a.URL != nil {
+			add(a.URL.String(), viaAlternate)
+		}
+	}
+	return out
+}
+
+// readSitemaps reads /sitemap.xml: an index's files (same host only) or a
+// single urlset.
+func readSitemaps(ctx context.Context, f *Fetcher, pace *pacer, res *Result) error {
+	if err := pace.wait(ctx); err != nil {
+		return err
+	}
+	r := f.Get(ctx, SitemapPath)
+	if r.Err != "" || r.Status != 200 {
+		res.Sitemaps = append(res.Sitemaps, SitemapFile{Key: SitemapPath, Status: r.Status})
+		return nil
+	}
+	index, locs := parseSitemap(r.Body)
+	if !index {
+		res.addSitemap(f, SitemapFile{Key: SitemapPath, Status: 200}, locs)
+		return nil
+	}
+	for i, loc := range locs {
+		if i >= maxSitemapFiles {
+			break
+		}
+		u, err := url.Parse(loc)
+		if err != nil {
+			continue
+		}
+		key := f.Key(u)
+		if key == "" {
+			continue
+		}
+		if err := pace.wait(ctx); err != nil {
+			return err
+		}
+		sr := f.Get(ctx, key)
+		file := SitemapFile{Key: key, Status: sr.Status}
+		if sr.Err != "" || sr.Status != 200 {
+			res.Sitemaps = append(res.Sitemaps, file)
+			continue
+		}
+		_, urls := parseSitemap(sr.Body)
+		res.addSitemap(f, file, urls)
+	}
+	return nil
+}
+
+func (res *Result) addSitemap(f *Fetcher, file SitemapFile, locs []string) {
+	for _, loc := range locs {
+		u, err := url.Parse(loc)
+		if err != nil {
+			continue
+		}
+		key := f.Key(u)
+		if key == "" {
+			continue
+		}
+		if _, dup := res.Listed[key]; !dup {
+			res.Listed[key] = file.Key
+		}
+		file.URLs = append(file.URLs, key)
+	}
+	res.Sitemaps = append(res.Sitemaps, file)
+}
+
+// parseSitemap reads the <loc> values of a sitemap document; index is true
+// for a <sitemapindex>.
+func parseSitemap(body []byte) (index bool, locs []string) {
+	dec := xml.NewDecoder(io.LimitReader(bytes.NewReader(body), MaxBody))
+	depth := 0
+	inLoc := false
+	var parent string
+	var stack []string
+	for {
+		tok, err := dec.Token()
+		if err != nil {
+			break
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			depth++
+			name := strings.ToLower(t.Name.Local)
+			if depth == 1 && name == "sitemapindex" {
+				index = true
+			}
+			stack = append(stack, name)
+			if name == "loc" && len(stack) >= 2 {
+				parent = stack[len(stack)-2]
+				inLoc = parent == "url" || parent == "sitemap"
+			}
+		case xml.EndElement:
+			depth--
+			if len(stack) > 0 {
+				stack = stack[:len(stack)-1]
+			}
+			inLoc = false
+		case xml.CharData:
+			if inLoc {
+				if s := strings.TrimSpace(string(t)); s != "" {
+					locs = append(locs, s)
+				}
+			}
+		}
+	}
+	return index, locs
+}
--- a/classes/crawl/fetch.go
+++ b/classes/crawl/fetch.go
@@ -0,0 +1,191 @@
+// Package crawl is the site audit's crawler (D-14): it fetches the
+// rendered site like a search engine, checks every page (UI-SPEC C23),
+// stores the findings with the CMS-data issues and names how each one is
+// fixed (P33-38).
+package crawl
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.golem15.com/golem15/summercms/modules/compass"
+	"git.golem15.com/golem15/summercms/modules/fetchguard"
+)
+
+// The config keys of the crawler.
+const (
+	// TrustedConfigKey lets the crawler fetch plain http and private
+	// addresses (staging, local sites). Only Go config can set it.
+	TrustedConfigKey = "golem15.seo.crawl_trusted"
+)
+
+// The crawler's limits (D-14, RESEARCH Pattern 7).
+const (
+	// MaxBody caps one answer's body.
+	MaxBody = 5 << 20
+	// FetchTimeout is one request's timeout.
+	FetchTimeout = 10 * time.Second
+	// Workers is the number of requests at a time.
+	Workers = 2
+	// MaxLinksPerPage caps the links stored and followed per page.
+	MaxLinksPerPage = 1000
+)
+
+// UserAgent is the crawler's identity (P33-41).
+func UserAgent(site string) string {
+	return "SummerCMS-SEO-Crawler/1 (+" + site + ")"
+}
+
+// Response is one fetched address: its status, Location, content type,
+// X-Robots-Tag and body, or the fetch error.
+type Response struct {
+	Status      int
+	Location    string
+	ContentType string
+	XRobots     string
+	Body        []byte
+	Err         string
+}
+
+// HTML reports whether the answer is an HTML page.
+func (r *Response) HTML() bool {
+	ct := strings.ToLower(r.ContentType)
+	return ct == "" || strings.Contains(ct, "text/html") || strings.Contains(ct, "application/xhtml")
+}
+
+// Fetcher fetches the site's own addresses through a fetchguard client
+// limited to the site host and its www twin: HTTPS only, private
+// addresses refused at dial time and redirects never followed. Trusted
+// mode (golem15.seo.crawl_trusted, Go config only) allows plain http and
+// private addresses, still only on those hosts.
+type Fetcher struct {
+	client  *fetchguard.Client
+	site    *url.URL
+	hosts   []string
+	trusted bool
+	ua      string
+}
+
+// SiteHosts returns the host of site and its www twin.
+func SiteHosts(host string) []string {
+	host = strings.ToLower(strings.TrimSpace(host))
+	if host == "" {
+		return nil
+	}
+	if bare, ok := strings.CutPrefix(host, "www."); ok {
+		return []string{host, bare}
+	}
+	return []string{host, "www." + host}
+}
+
+// Trusted reports whether config turns trusted mode on.
+func Trusted(cfg *compass.Config) bool {
+	return cfg != nil && cfg.Bool(TrustedConfigKey)
+}
+
+// NewFetcher builds the fetcher of site (an absolute origin).
+func NewFetcher(site string, cfg *compass.Config) (*Fetcher, error) {
+	u, err := url.Parse(strings.TrimRight(site, "/"))
+	if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
+		return nil, ErrNoSiteURL
+	}
+	f := &Fetcher{site: u, hosts: SiteHosts(u.Hostname()), trusted: Trusted(cfg), ua: UserAgent(u.Scheme + "://" + u.Host)}
+	policy := fetchguard.Policy{Mode: fetchguard.AllowHostsMode, AllowHosts: f.hosts, MaxBytes: MaxBody, Timeout: FetchTimeout}
+	if f.trusted {
+		policy.Mode = fetchguard.TrustedMode
+	}
+	f.client, err = fetchguard.NewClient(policy, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// Hosts returns the site host and its www twin.
+func (f *Fetcher) Hosts() []string { return f.hosts }
+
+// IsTrusted reports whether the fetcher runs in trusted mode.
+func (f *Fetcher) IsTrusted() bool { return f.trusted }
+
+// Site returns the site origin.
+func (f *Fetcher) Site() *url.URL { return f.site }
+
+// UA returns the crawler's User-Agent.
+func (f *Fetcher) UA() string { return f.ua }
+
+// Allowed reports whether u may be fetched: https (http only in trusted
+// mode) on the site host or its www twin, without credentials. In trusted
+// mode the host list is checked here, since fetchguard checks none then.
+func (f *Fetcher) Allowed(u *url.URL) bool {
+	if u == nil || u.User != nil {
+		return false
+	}
+	switch strings.ToLower(u.Scheme) {
+	case "https":
+	case "http":
+		if !f.trusted {
+			return false
+		}
+	default:
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	for _, h := range f.hosts {
+		if host == h {
+			return true
+		}
+	}
+	return false
+}
+
+// Key is the address of u on the site: its path with the query, or ""
+// when u is not on the site.
+func (f *Fetcher) Key(u *url.URL) string {
+	if !f.Allowed(u) {
+		return ""
+	}
+	p := u.EscapedPath()
+	if p == "" {
+		p = "/"
+	}
+	if u.RawQuery != "" {
+		p += "?" + u.RawQuery
+	}
+	return p
+}
+
+// Absolute returns the site URL of an address key.
+func (f *Fetcher) Absolute(key string) string {
+	return f.site.Scheme + "://" + f.site.Host + key
+}
+
+// Get fetches one address key of the site.
+func (f *Fetcher) Get(ctx context.Context, key string) *Response {
+	header := http.Header{}
+	header.Set("User-Agent", f.ua)
+	header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5")
+	res, err := f.client.Get(ctx, f.Absolute(key), header)
+	if err != nil {
+		reason := "network_error"
+		if fe, ok := err.(*fetchguard.Error); ok {
+			reason = string(fe.Reason)
+		}
+		return &Response{Err: reason}
+	}
+	out := &Response{
+		Status:      res.StatusCode,
+		ContentType: res.Header.Get("Content-Type"),
+		XRobots:     strings.Join(res.Header.Values("X-Robots-Tag"), ", "),
+		Body:        res.Body,
+	}
+	if loc := res.Header.Get("Location"); loc != "" {
+		base, _ := url.Parse(f.Absolute(key))
+		if ref, err := url.Parse(loc); err == nil && base != nil {
+			out.Location = base.ResolveReference(ref).String()
+		}
+	}
+	return out
+}
--- a/classes/crawl/fix.go
+++ b/classes/crawl/fix.go
@@ -0,0 +1,145 @@
+package crawl
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/pact"
+)
+
+// The SEO tools an issue may name (pier prefixes them with seo_).
+const (
+	ToolSetURLSEO      = "seo_set_url_seo"
+	ToolUpdateRobots   = "seo_update_robots"
+	ToolCreateRedirect = "seo_create_redirect"
+	ToolUpdatePage     = "seo_update_page"
+	ToolCreatePage     = "seo_create_page"
+	ToolUpdateSettings = "seo_update_settings"
+)
+
+// Record is the CMS record behind a crawled page: its content type, id
+// and locale, the descriptor facts the checks compare the page with
+// (title, word count, headings, images, the highest heading level) and the
+// fixing tools its type names (pact.ContentDescriptor.Tools).
+type Record struct {
+	Type       string
+	ID         string
+	Locale     string
+	Title      string
+	Words      *int
+	Headings   []pact.ContentHeading
+	Images     []pact.ContentImage
+	TopHeading int
+	// Links are the record's CMS-known links: the address of its primary
+	// parent, which the page's breadcrumb trail links to.
+	Links []string
+	Tools map[string]string
+	// CustomJSONLD is true when the record's SEO tab stores custom JSON-LD.
+	CustomJSONLD bool
+}
+
+// tool returns the record's tool for the first operation of ops it
+// names, "" when it names none.
+func (r *Record) tool(ops ...string) string {
+	if r == nil {
+		return ""
+	}
+	for _, op := range ops {
+		if t := strings.TrimSpace(r.Tools[op]); t != "" {
+			return t
+		}
+	}
+	return ""
+}
+
+// hasHeading reports whether the record's own headings hold one of level
+// (any level when level is 0) whose text is text (any text when empty).
+func (r *Record) hasHeading(level int, text string) bool {
+	if r == nil {
+		return false
+	}
+	for _, h := range r.Headings {
+		if level != 0 && h.Level != level {
+			continue
+		}
+		if text == "" || normal(h.Text) == normal(text) {
+			return true
+		}
+	}
+	return false
+}
+
+// Finding is one problem a check found on a page, before it is stored:
+// the kind, the address and locale, the details of the kind, and how it
+// is fixed (P33-38). Tool is a fixed CMS tool (seo_*); Content lists the
+// record operations (pact.ContentTool*) that fix a finding inside the
+// record's own content, tried in order; What names the template element
+// when it is fixed in code.
+type Finding struct {
+	Kind    string
+	URL     string
+	Locale  string
+	Details map[string]any
+	Tool    string
+	Content []string
+	Record  *Record
+	What    string
+	// Check tells the two content_js findings apart (cms, rendered).
+	Check string
+}
+
+// Route applies P33-38: a finding inside the record's own content names
+// the record type's fitting tool (H1 findings prefer Tools[headings],
+// other content findings Tools[edit]); else a fixed CMS tool; else fix
+// code with the template element in details.what. Every finding gets one
+// of the two.
+func Route(f Finding) (fix, tool, what string) {
+	if len(f.Content) > 0 {
+		if t := f.Record.tool(f.Content...); t != "" {
+			return models.FixTool, t, ""
+		}
+	}
+	if f.Tool != "" {
+		return models.FixTool, f.Tool, ""
+	}
+	what = f.What
+	if what == "" {
+		what = defaultWhat[f.Kind]
+	}
+	if what == "" {
+		what = "the page template"
+	}
+	return models.FixCode, "", what
+}
+
+// defaultWhat names the template element of a kind fixed in code.
+var defaultWhat = map[string]string{
+	KindCanonicalMissing: `<link rel="canonical"> in the page head`,
+	KindH1Missing:        "the <h1> of the page template",
+	KindH1Multiple:       "the <h1> headings of the page template",
+	KindHeadingSkipped:   "the heading levels of the page template",
+	KindHeadingEmpty:     "the empty heading of the page template",
+	KindJSONLDInvalid:    `the <script type="application/ld+json"> of the page template`,
+	KindJSONLDMultiple:   `the <script type="application/ld+json"> blocks of the page template`,
+	KindHreflangRecip:    `the <link rel="alternate" hreflang> links of the page template`,
+	KindImageAlt:         "the alt attribute of the template's <img>",
+	KindImageSize:        "the width and height attributes of the template's <img>",
+	KindFeaturedLazy:     `loading="lazy" on the main image`,
+	KindContentJS:        "server-side rendering of the page content",
+	KindGenericAnchor:    "the link text in the page template",
+	KindRedirectedLink:   "the link in the page template",
+	KindOrphanPage:       "a link to this page from the site navigation or another page",
+}
+
+// Fingerprint identifies a finding across crawls: the kind, the address,
+// the locale and the check (content_js), so an ignore survives later
+// crawls (P33-37).
+func Fingerprint(kind, url, locale, check string) string {
+	sum := sha256.Sum256([]byte(kind + "\x00" + url + "\x00" + locale + "\x00" + check))
+	return hex.EncodeToString(sum[:16])
+}
+
+// normal is text compared without case and extra whitespace.
+func normal(s string) string { return strings.ToLower(strings.Join(strings.Fields(s), " ")) }
--- a/classes/crawl/job.go
+++ b/classes/crawl/job.go
@@ -0,0 +1,361 @@
+package crawl
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"time"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/sm-seo-plugin/classes/schema"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/conga"
+	"git.golem15.com/golem15/summercms/modules/diary"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"git.golem15.com/golem15/summercms/modules/pact"
+	"gorm.io/gorm"
+)
+
+// CrawlKind is the River kind of the crawl job.
+const CrawlKind = "golem15.seo.crawl"
+
+// CrawlArgs are the arguments of the crawl job: the crawl row it runs.
+type CrawlArgs struct {
+	CrawlID uint `json:"crawl_id"`
+}
+
+// Kind is CrawlKind.
+func (CrawlArgs) Kind() string { return CrawlKind }
+
+// The crawl limits used until the Site audit settings say otherwise.
+const (
+	DefaultMaxPages = 5000
+	DefaultDelay    = 250 * time.Millisecond
+)
+
+// AlreadyRunningError refuses a start while a crawl is queued or running
+// (P33-39): one crawl runs at a time.
+type AlreadyRunningError struct {
+	Crawl models.Crawl
+}
+
+func (e *AlreadyRunningError) Error() string { return "golem15.seo: a crawl is already running" }
+
+// Since is the time the running crawl started, else its creation time.
+func (e *AlreadyRunningError) Since() time.Time {
+	if e.Crawl.StartedAt != nil {
+		return *e.Crawl.StartedAt
+	}
+	if e.Crawl.CreatedAt != nil {
+		return *e.Crawl.CreatedAt
+	}
+	return time.Time{}
+}
+
+// Start queues a crawl (D-14, P33-39) inside tx: the settings row is
+// locked so two starts cannot both pass the check that no crawl is queued
+// or running, then the crawl row is written and the conga job dispatched
+// in the same transaction. A missing site address refuses with
+// ErrNoSiteURL, a running crawl with *AlreadyRunningError.
+func Start(ctx context.Context, svc *classes.Service, tx *gorm.DB, trigger, name string) (models.Crawl, error) {
+	site := svc.SiteURL()
+	if site == "" {
+		return models.Crawl{}, ErrNoSiteURL
+	}
+	if _, err := classes.SettingsRow(tx); err != nil {
+		return models.Crawl{}, err
+	}
+	if err := tx.Exec(`SELECT id FROM golem15_seo_settings WHERE id = 1 FOR UPDATE`).Error; err != nil {
+		return models.Crawl{}, err
+	}
+	active, err := ActiveCrawl(tx)
+	if err != nil {
+		return models.Crawl{}, err
+	}
+	if active != nil {
+		return models.Crawl{}, &AlreadyRunningError{Crawl: *active}
+	}
+	limits, err := LoadLimits(tx)
+	if err != nil {
+		return models.Crawl{}, err
+	}
+	now := time.Now().UTC()
+	row := models.Crawl{
+		Status: models.CrawlQueued, Trigger: trigger, TriggerName: name, Site: site,
+		MaxPages: limits.MaxPages, RenderStatus: models.RenderSkipped, CreatedAt: &now,
+	}
+	if err := tx.Create(&row).Error; err != nil {
+		return models.Crawl{}, err
+	}
+	m, err := conga.From(svc.App())
+	if err != nil {
+		return models.Crawl{}, err
+	}
+	id, err := m.Dispatch(ctx, tx, CrawlArgs{CrawlID: row.ID}, conga.DispatchOpts{Label: svc.Phrase("golem15.seo::lang.jobs.crawl", nil)})
+	if err != nil {
+		return models.Crawl{}, err
+	}
+	row.JobID = &id
+	if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("job_id", id).Error; err != nil {
+		return models.Crawl{}, err
+	}
+	return row, nil
+}
+
+// Limits are a crawl's politeness limits.
+type Limits struct {
+	MaxPages int
+	Delay    time.Duration
+}
+
+// LoadLimits reads the crawl limits.
+func LoadLimits(db *gorm.DB) (Limits, error) {
+	return Limits{MaxPages: DefaultMaxPages, Delay: DefaultDelay}, nil
+}
+
+// LoadCrawl reads one crawl.
+func LoadCrawl(db *gorm.DB, id uint) (models.Crawl, error) {
+	var rows []models.Crawl
+	if err := db.Where("id = ?", id).Limit(1).Find(&rows).Error; err != nil {
+		return models.Crawl{}, err
+	}
+	if len(rows) == 0 {
+		return models.Crawl{}, gorm.ErrRecordNotFound
+	}
+	return rows[0], nil
+}
+
+// Run runs one queued crawl: it fetches the site, checks every page,
+// stores the pages, links and findings with the CMS-data issues, and marks
+// the crawl finished, cancelled (pages and issues fetched so far are kept)
+// or failed (with the error; what was fetched before the failure is kept).
+// A crawl that is no longer queued is left alone.
+func Run(ctx context.Context, svc *classes.Service, id uint) error {
+	gdb, err := svc.GormDB()
+	if err != nil {
+		return err
+	}
+	db := gdb.WithContext(ctx)
+	row, err := LoadCrawl(db, id)
+	if err != nil {
+		return err
+	}
+	if row.Status != models.CrawlQueued {
+		return nil
+	}
+	now := time.Now().UTC()
+	moved := db.Model(&models.Crawl{}).Where("id = ? AND status = ?", id, models.CrawlQueued).
+		Updates(map[string]any{"status": models.CrawlRunning, "started_at": now})
+	if moved.Error != nil {
+		return moved.Error
+	}
+	if moved.RowsAffected == 0 {
+		return nil
+	}
+	res, runErr := run(ctx, svc, db, row)
+	return finish(ctx, svc, db, row, res, runErr)
+}
+
+// run fetches and checks the site of one crawl.
+func run(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl) (*Result, error) {
+	f, err := NewFetcher(row.Site, svc.Config())
+	if err != nil {
+		return nil, err
+	}
+	limits, err := LoadLimits(db)
+	if err != nil {
+		return nil, err
+	}
+	if row.MaxPages > 0 {
+		limits.MaxPages = row.MaxPages
+	}
+	var m *conga.Manager
+	if app := svc.App(); app != nil {
+		m, _ = conga.From(app)
+	}
+	opt := Options{
+		MaxPages: limits.MaxPages,
+		Delay:    limits.Delay,
+		Renderer: rendererFrom(ctx),
+		Canceled: func(ctx context.Context) bool {
+			var status string
+			if err := db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&status).Error; err == nil && status == models.CrawlCancelled {
+				return true
+			}
+			if m != nil && row.JobID != nil {
+				if c, err := m.CheckIfCanceled(ctx, *row.JobID); err == nil && c {
+					return true
+				}
+			}
+			return false
+		},
+		Progress: func(ctx context.Context, n int) {
+			_ = db.Model(&models.Crawl{}).Where("id = ?", row.ID).Update("pages", n).Error
+			if m != nil && row.JobID != nil {
+				_ = m.UpdateJobState(ctx, *row.JobID, n, nil)
+			}
+		},
+	}
+	if m != nil && row.JobID != nil {
+		_ = m.StartJob(ctx, *row.JobID, limits.MaxPages)
+	}
+	return Crawl(ctx, f, opt)
+}
+
+// finish stores what a crawl found and writes its final state.
+func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.Crawl, res *Result, runErr error) error {
+	status := models.CrawlFinished
+	errText := ""
+	if runErr != nil {
+		status, errText = models.CrawlFailed, runErr.Error()
+	}
+	var current string
+	_ = db.Raw(`SELECT status FROM golem15_seo_crawls WHERE id = ?`, row.ID).Scan(&current).Error
+	if current == models.CrawlCancelled || (res != nil && res.Canceled) {
+		status = models.CrawlCancelled
+	}
+	store := func(ctx context.Context, tx *gorm.DB) error {
+		if res == nil {
+			return nil
+		}
+		if err := SavePages(tx, row.ID, res); err != nil {
+			return err
+		}
+		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx)})
+		cms, err := svc.Issues(classes.WithTx(ctx, tx))
+		if err != nil {
+			return err
+		}
+		id := row.ID
+		return SaveIssues(tx, &id, findings, cms)
+	}
+	if err := lagoon.Transaction(ctx, db, store); err != nil {
+		status, errText = models.CrawlFailed, err.Error()
+	}
+	updates := map[string]any{"status": status, "error": errText, "finished_at": time.Now().UTC()}
+	if res != nil {
+		updates["pages"] = len(res.Order)
+		updates["limit_hit"] = res.LimitHit
+		updates["render_status"] = res.Render.Status
+		updates["render_pages"] = res.Render.Pages
+		updates["render_note"] = res.Render.Note
+	}
+	if err := db.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
+		return err
+	}
+	if m, err := conga.From(svc.App()); err == nil && row.JobID != nil {
+		meta := map[string]any{"crawl_id": row.ID, "status": status}
+		switch status {
+		case models.CrawlFailed:
+			_ = m.FailJob(ctx, *row.JobID, meta)
+		case models.CrawlCancelled:
+			_ = m.StopJob(ctx, *row.JobID, meta)
+		default:
+			_ = m.CompleteJob(ctx, *row.JobID, meta)
+		}
+	}
+	if status == models.CrawlFailed {
+		pluginLog.Warn("golem15.seo: crawl failed", "crawl", row.ID, "error", errText)
+	}
+	return nil
+}
+
+// Lookup returns the CMS record behind a crawled address (cached per
+// address): the content type whose Match answers the path exactly, its
+// descriptor facts in the matched locale, its primary parent's address and
+// whether its SEO tab stores custom JSON-LD.
+func Lookup(ctx context.Context, svc *classes.Service, db *gorm.DB) func(key string) *Record {
+	cache := map[string]*Record{}
+	reg := svc.Registry()
+	unprefixed := svc.UnprefixedLocale()
+	return func(key string) *Record {
+		if reg == nil {
+			return nil
+		}
+		path, _, _ := strings.Cut(key, "?")
+		if r, ok := cache[path]; ok {
+			return r
+		}
+		cache[path] = nil
+		typeKey, match, err := reg.MatchPath(ctx, path, unprefixed)
+		if err != nil || match == nil || !match.Exact {
+			return nil
+		}
+		typ, ok := reg.ContentType(typeKey)
+		if !ok || typ.Content == nil {
+			return nil
+		}
+		rec, err := reg.Describe(ctx, typeKey, match.ID, unprefixed)
+		if err != nil || rec == nil {
+			return nil
+		}
+		out := &Record{Type: typeKey, ID: match.ID, Locale: match.Locale, TopHeading: rec.TopHeading, Tools: typ.Content.Tools}
+		var loc *pact.ContentLocale
+		for i := range rec.Locales {
+			if rec.Locales[i].Locale == match.Locale {
+				loc = &rec.Locales[i]
+				break
+			}
+		}
+		if loc == nil && len(rec.Locales) > 0 {
+			loc = &rec.Locales[0]
+		}
+		if loc != nil {
+			out.Locale = loc.Locale
+			out.Title, out.Words, out.Headings, out.Images = loc.Title, loc.Words, loc.Headings, loc.Images
+		}
+		if len(rec.Parents) > 0 {
+			if parent, err := reg.Describe(ctx, rec.Parents[0].Type, rec.Parents[0].ID, unprefixed); err == nil && parent != nil {
+				for _, pl := range parent.Locales {
+					if pl.Locale == out.Locale && pl.URL != "" {
+						out.Links = append(out.Links, pl.URL)
+					}
+				}
+			}
+		}
+		if entry, found, err := classes.LoadEntry(db, typ.Content.Model, match.ID, out.Locale); err == nil && found && entry.Data != nil {
+			if raw, _ := entry.Data["schema"].(string); raw != "" {
+				if v, err := schema.ParseValue(raw); err == nil && strings.TrimSpace(v.Custom) != "" {
+					out.CustomJSONLD = true
+				}
+			}
+		}
+		cache[path] = out
+		return out
+	}
+}
+
+// errCancelledCrawl refuses cancelling a crawl that is not queued or
+// running.
+var errCancelledCrawl = errors.New("golem15.seo: the crawl is not running")
+
+// Cancel stops a queued or running crawl inside tx: the row is marked
+// cancelled by name (the crawl keeps what it fetched) and its conga job
+// cancelled.
+func Cancel(ctx context.Context, svc *classes.Service, tx *gorm.DB, row models.Crawl, name string) error {
+	if !row.Active() {
+		return errCancelledCrawl
+	}
+	now := time.Now().UTC()
+	updates := map[string]any{"status": models.CrawlCancelled, "cancelled_by": name}
+	if row.Status == models.CrawlQueued {
+		updates["finished_at"] = now
+	}
+	if err := tx.Model(&models.Crawl{}).Where("id = ?", row.ID).Updates(updates).Error; err != nil {
+		return err
+	}
+	if row.JobID != nil {
+		if m, err := conga.From(svc.App()); err == nil {
+			if err := m.CancelJob(ctx, *row.JobID); err != nil {
+				pluginLog.Warn("golem15.seo: cancel crawl job", "crawl", row.ID, "error", err)
+			}
+		}
+	}
+	return nil
+}
+
+// ErrNotRunning is a cancel of a crawl that is not queued or running.
+var ErrNotRunning = errCancelledCrawl
+
+// pluginLog is the crawler's logger (component seo).
+var pluginLog = diary.For("seo")
--- a/classes/crawl/parse.go
+++ b/classes/crawl/parse.go
@@ -0,0 +1,293 @@
+package crawl
+
+import (
+	"bytes"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Heading is one heading of a page in document order.
+type Heading struct {
+	Level int    `json:"level"`
+	Text  string `json:"text"`
+}
+
+// Link is one <a href> of a page, resolved against the page's address.
+type Link struct {
+	URL  *url.URL
+	Href string
+	Text string
+	Rel  string
+}
+
+// Image is one <img> of a page.
+type Image struct {
+	Src     string
+	Alt     string
+	HasAlt  bool
+	Width   string
+	Height  string
+	Loading string
+}
+
+// Alternate is one <link rel="alternate" hreflang>.
+type Alternate struct {
+	Lang string
+	URL  *url.URL
+	Href string
+}
+
+// PageFacts is what one HTML document says, parsed with
+// golang.org/x/net/html (no regular expressions over HTML): the language,
+// title, meta description, canonical, meta robots, og:image, headings,
+// links, images, JSON-LD scripts, hreflang alternates and the main text
+// with its word count. The rendered comparison parses the JavaScript
+// rendered DOM into the same facts.
+type PageFacts struct {
+	Lang           string
+	Title          string
+	HasTitle       bool
+	Description    string
+	HasDescription bool
+	Canonical      string
+	Robots         string
+	OGImage        string
+	Headings       []Heading
+	Links          []Link
+	Images         []Image
+	JSONLD         []string
+	Alternates     []Alternate
+	Text           string
+	Words          int
+}
+
+// H1 returns the texts of the page's level-1 headings.
+func (p *PageFacts) H1() []string {
+	var out []string
+	for _, h := range p.Headings {
+		if h.Level == 1 {
+			out = append(out, h.Text)
+		}
+	}
+	return out
+}
+
+// Noindex reports whether the meta robots value says noindex.
+func (p *PageFacts) Noindex() bool { return hasNoindex(p.Robots) }
+
+// hasNoindex reports whether a robots value contains noindex or none.
+func hasNoindex(v string) bool {
+	for _, part := range strings.FieldsFunc(strings.ToLower(v), func(r rune) bool { return r == ',' || r == ' ' || r == ';' }) {
+		if part == "noindex" || part == "none" {
+			return true
+		}
+	}
+	return false
+}
+
+// Parse reads the facts of an HTML document fetched from base.
+func Parse(base *url.URL, body []byte) *PageFacts {
+	out := &PageFacts{}
+	doc, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return out
+	}
+	resolve := func(href string) *url.URL {
+		href = strings.TrimSpace(href)
+		if href == "" {
+			return nil
+		}
+		ref, err := url.Parse(href)
+		if err != nil {
+			return nil
+		}
+		if base == nil {
+			return ref
+		}
+		u := base.ResolveReference(ref)
+		u.Fragment = ""
+		return u
+	}
+	var main *html.Node
+	var bodyNode *html.Node
+	var walk func(n *html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.DataAtom {
+			case atom.Html:
+				out.Lang = strings.TrimSpace(attr(n, "lang"))
+			case atom.Title:
+				if !out.HasTitle {
+					out.HasTitle = true
+					out.Title = collapse(textOf(n))
+				}
+			case atom.Meta:
+				name := strings.ToLower(attr(n, "name"))
+				prop := strings.ToLower(attr(n, "property"))
+				switch {
+				case name == "description" && !out.HasDescription:
+					out.HasDescription = true
+					out.Description = collapse(attr(n, "content"))
+				case name == "robots" || name == "googlebot":
+					if v := strings.TrimSpace(attr(n, "content")); v != "" {
+						if out.Robots != "" {
+							out.Robots += ", "
+						}
+						out.Robots += v
+					}
+				case prop == "og:image" && out.OGImage == "":
+					if u := resolve(attr(n, "content")); u != nil {
+						out.OGImage = u.String()
+					}
+				}
+			case atom.Link:
+				rel := strings.ToLower(attr(n, "rel"))
+				switch {
+				case relHas(rel, "canonical") && out.Canonical == "":
+					if u := resolve(attr(n, "href")); u != nil {
+						out.Canonical = u.String()
+					}
+				case relHas(rel, "alternate") && hasAttr(n, "hreflang"):
+					href := attr(n, "href")
+					out.Alternates = append(out.Alternates, Alternate{Lang: strings.TrimSpace(attr(n, "hreflang")), URL: resolve(href), Href: href})
+				}
+			case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
+				out.Headings = append(out.Headings, Heading{Level: int(n.Data[1] - '0'), Text: collapse(textOf(n))})
+			case atom.A:
+				if hasAttr(n, "href") && len(out.Links) < MaxLinksPerPage {
+					href := attr(n, "href")
+					low := strings.ToLower(strings.TrimSpace(href))
+					if !strings.HasPrefix(low, "javascript:") && !strings.HasPrefix(low, "mailto:") && !strings.HasPrefix(low, "tel:") && !strings.HasPrefix(low, "#") {
+						if u := resolve(href); u != nil {
+							out.Links = append(out.Links, Link{URL: u, Href: href, Text: collapse(textOf(n)), Rel: strings.ToLower(attr(n, "rel"))})
+						}
+					}
+				}
+			case atom.Img:
+				alt, hasAlt := attrOK(n, "alt")
+				src := attr(n, "src")
+				if u := resolve(src); u != nil {
+					src = u.String()
+				}
+				out.Images = append(out.Images, Image{Src: src, Alt: alt, HasAlt: hasAlt, Width: attr(n, "width"), Height: attr(n, "height"), Loading: strings.ToLower(attr(n, "loading"))})
+			case atom.Script:
+				if strings.EqualFold(strings.TrimSpace(attr(n, "type")), "application/ld+json") {
+					out.JSONLD = append(out.JSONLD, textOf(n))
+				}
+				return
+			case atom.Main:
+				if main == nil {
+					main = n
+				}
+			case atom.Body:
+				bodyNode = n
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	root := main
+	if root == nil {
+		root = bodyNode
+	}
+	if root != nil {
+		var b strings.Builder
+		visibleText(root, &b)
+		out.Text = collapse(b.String())
+		if len(out.Text) > maxText {
+			out.Text = truncateUTF8(out.Text, maxText)
+		}
+		out.Words = len(strings.Fields(out.Text))
+	}
+	return out
+}
+
+// maxText caps the main text kept per page (characters of UTF-8, bytes
+// bound).
+const maxText = 100_000
+
+func truncateUTF8(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	for n > 0 && n < len(s) && (s[n]&0xC0) == 0x80 {
+		n--
+	}
+	return s[:n]
+}
+
+// visibleText writes the text of n without script, style, noscript and
+// template content, one space between block parts.
+func visibleText(n *html.Node, b *strings.Builder) {
+	if n.Type == html.ElementNode {
+		switch n.DataAtom {
+		case atom.Script, atom.Style, atom.Noscript, atom.Template, atom.Svg:
+			return
+		}
+	}
+	if n.Type == html.TextNode {
+		b.WriteString(n.Data)
+		b.WriteByte(' ')
+		return
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		visibleText(c, b)
+	}
+}
+
+func textOf(n *html.Node) string {
+	var b strings.Builder
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			b.WriteString(n.Data)
+			b.WriteByte(' ')
+		}
+		if n.Type == html.ElementNode && n.DataAtom == atom.Img {
+			if alt := attr(n, "alt"); alt != "" {
+				b.WriteString(alt)
+				b.WriteByte(' ')
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(n)
+	return strings.TrimSpace(b.String())
+}
+
+func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
+
+func attrOK(n *html.Node, key string) (string, bool) {
+	for _, a := range n.Attr {
+		if strings.EqualFold(a.Key, key) {
+			return a.Val, true
+		}
+	}
+	return "", false
+}
+
+func attr(n *html.Node, key string) string {
+	v, _ := attrOK(n, key)
+	return v
+}
+
+func hasAttr(n *html.Node, key string) bool {
+	_, ok := attrOK(n, key)
+	return ok
+}
+
+func relHas(rel, want string) bool {
+	for _, part := range strings.Fields(rel) {
+		if part == want {
+			return true
+		}
+	}
+	return false
+}
--- a/classes/crawl/render.go
+++ b/classes/crawl/render.go
@@ -0,0 +1,43 @@
+package crawl
+
+import "context"
+
+// Renderer renders a page of the site in a browser for the raw versus
+// rendered comparison (D-70, P33-55): Render answers the page's DOM after
+// JavaScript ran, Close releases the browser. A crawl without a Renderer
+// records the comparison as skipped; a Render error records it as failed
+// with the error and never fails the crawl.
+type Renderer interface {
+	Render(ctx context.Context, pageURL string) (string, error)
+	Close() error
+}
+
+// DefaultRenderMax is the number of pages rendered per crawl when
+// golem15.seo.crawl_render_max is unset.
+const DefaultRenderMax = 200
+
+// The render states of a crawl (models.RenderSkipped and the others).
+const (
+	renderStatusSkipped = "skipped"
+	renderStatusDone    = "done"
+	renderStatusFailed  = "failed"
+)
+
+type rendererKey struct{}
+
+// WithRenderer returns a context whose crawls use r instead of the
+// renderer built from config. It is the test seam of the comparison,
+// settable only from Go code (like fetchguard.WithTransport): no config
+// key, setting, request or MCP call reaches it.
+func WithRenderer(ctx context.Context, r Renderer) context.Context {
+	return context.WithValue(ctx, rendererKey{}, r)
+}
+
+// rendererFrom returns the renderer of ctx, nil when none.
+func rendererFrom(ctx context.Context) Renderer {
+	if ctx == nil {
+		return nil
+	}
+	r, _ := ctx.Value(rendererKey{}).(Renderer)
+	return r
+}
--- a/classes/crawl/store.go
+++ b/classes/crawl/store.go
@@ -0,0 +1,248 @@
+package crawl
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
+	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/lagoon"
+	"gorm.io/gorm"
+)
+
+// insertBatch is the rows written per INSERT.
+const insertBatch = 200
+
+// SavePages writes the pages and links one crawl fetched. The crawler
+// writes only its own tables (D-14).
+func SavePages(tx *gorm.DB, crawlID uint, res *Result) error {
+	if res == nil {
+		return nil
+	}
+	in := map[string]int{}
+	var links []models.CrawlLink
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		if p.Facts == nil {
+			continue
+		}
+		seen := map[string]bool{}
+		for _, l := range p.Facts.Links {
+			to := ""
+			internal := false
+			if res.fetcher != nil {
+				to = res.fetcher.Key(l.URL)
+				internal = to != ""
+			}
+			if !internal {
+				to = l.URL.String()
+			}
+			links = append(links, models.CrawlLink{CrawlID: crawlID, FromURL: key, ToURL: to, Text: truncateUTF8(l.Text, 500), Rel: l.Rel, Internal: internal})
+			if internal && to != key && !seen[to] {
+				seen[to] = true
+				in[to]++
+			}
+		}
+	}
+	pages := make([]models.CrawlPage, 0, len(res.Order))
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		row := models.CrawlPage{CrawlID: crawlID, URL: key, Depth: p.Depth, InSitemap: p.InSitemap, SitemapFile: p.SitemapFile, LinksIn: in[key]}
+		if !p.FetchedAt.IsZero() {
+			t := p.FetchedAt
+			row.FetchedAt = &t
+		}
+		if r := p.Resp; r != nil {
+			row.Status, row.Location, row.ContentType, row.XRobots = r.Status, r.Location, truncateUTF8(r.ContentType, 200), truncateUTF8(r.XRobots, 500)
+		}
+		if f := p.Facts; f != nil {
+			row.Title, row.Description, row.Canonical, row.Robots = truncateUTF8(f.Title, 1000), truncateUTF8(f.Description, 2000), truncateUTF8(f.Canonical, 2048), truncateUTF8(f.Robots, 500)
+			row.H1 = lagoon.Jsonable[[]string]{Data: f.H1(), Valid: true}
+			hs := make([]models.CrawlHeading, 0, len(f.Headings))
+			for _, h := range f.Headings {
+				hs = append(hs, models.CrawlHeading{Level: h.Level, Text: truncateUTF8(h.Text, 500)})
+			}
+			row.Headings = lagoon.Jsonable[[]models.CrawlHeading]{Data: hs, Valid: true}
+			row.Words, row.Text = f.Words, truncateUTF8(f.Text, models.MaxPageText)
+			row.LinksOut = len(f.Links)
+		}
+		pages = append(pages, row)
+	}
+	if len(pages) > 0 {
+		if err := tx.CreateInBatches(pages, insertBatch).Error; err != nil {
+			return err
+		}
+	}
+	if len(links) > 0 {
+		if err := tx.CreateInBatches(links, insertBatch).Error; err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// issueRow turns a finding into its stored row.
+func issueRow(crawlID *uint, source string, f Finding, now time.Time) models.Issue {
+	fix, tool, what := Route(f)
+	details := map[string]any{}
+	for k, v := range f.Details {
+		details[k] = v
+	}
+	if what != "" {
+		details["what"] = what
+	}
+	if f.Check != "" {
+		details["check"] = f.Check
+	}
+	sev := Severity(f.Kind)
+	t := now
+	return models.Issue{
+		CrawlID:     crawlID,
+		Source:      source,
+		Kind:        f.Kind,
+		Severity:    sev,
+		Rank:        models.SeverityRank(sev),
+		URL:         truncateUTF8(f.URL, 2048),
+		Locale:      f.Locale,
+		Details:     lagoon.Jsonable[map[string]any]{Data: details, Valid: true},
+		Tool:        tool,
+		Fix:         fix,
+		Fingerprint: Fingerprint(f.Kind, f.URL, f.Locale, f.Check),
+		FoundAt:     &t,
+	}
+}
+
+// cmsFinding turns a CMS-data issue of the classes report into a finding:
+// its tool is the one the report names; a CMS issue without one is fixed
+// in the record's admin form.
+func cmsFinding(issue classes.Issue) Finding {
+	f := Finding{Kind: issue.Kind, URL: issue.URL, Locale: issue.Locale, Details: issue.Details, Tool: issue.Tool}
+	if f.Tool == "" {
+		f.What = "the record in its admin form"
+	}
+	return f
+}
+
+// SaveIssues stores a crawl's findings and the CMS-data issues with it
+// (crawlID nil: the CMS issues of a site not crawled yet). A CMS issue
+// with the fingerprint of a crawl finding is the same problem and is not
+// stored twice.
+func SaveIssues(tx *gorm.DB, crawlID *uint, findings []Finding, cms []classes.Issue) error {
+	now := time.Now().UTC()
+	seen := map[string]bool{}
+	var rows []models.Issue
+	for _, f := range findings {
+		row := issueRow(crawlID, models.SourceCrawl, f, now)
+		if seen[row.Fingerprint] {
+			continue
+		}
+		seen[row.Fingerprint] = true
+		rows = append(rows, row)
+	}
+	for _, issue := range cms {
+		row := issueRow(crawlID, models.SourceCMS, cmsFinding(issue), now)
+		if seen[row.Fingerprint] {
+			continue
+		}
+		seen[row.Fingerprint] = true
+		rows = append(rows, row)
+	}
+	sort.SliceStable(rows, func(i, j int) bool {
+		if rows[i].Rank != rows[j].Rank {
+			return rows[i].Rank < rows[j].Rank
+		}
+		if rows[i].Kind != rows[j].Kind {
+			return rows[i].Kind < rows[j].Kind
+		}
+		return rows[i].URL < rows[j].URL
+	})
+	if len(rows) == 0 {
+		return nil
+	}
+	return tx.CreateInBatches(rows, insertBatch).Error
+}
+
+// LatestCrawl returns the newest crawl that is no longer queued or
+// running, nil when there is none: its issues are the audit's current
+// ones.
+func LatestCrawl(db *gorm.DB) (*models.Crawl, error) {
+	var rows []models.Crawl
+	if err := db.Where("status NOT IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	if len(rows) == 0 {
+		return nil, nil
+	}
+	return &rows[0], nil
+}
+
+// ActiveCrawl returns the queued or running crawl, nil when none.
+func ActiveCrawl(db *gorm.DB) (*models.Crawl, error) {
+	var rows []models.Crawl
+	if err := db.Where("status IN ?", []string{models.CrawlQueued, models.CrawlRunning}).Order("id DESC").Limit(1).Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	if len(rows) == 0 {
+		return nil, nil
+	}
+	return &rows[0], nil
+}
+
+// CurrentScope narrows an issues query to the audit's current issues: the
+// newest finished, failed or cancelled crawl's, or before any crawl the
+// stored CMS-data issues.
+func CurrentScope(db *gorm.DB, latest *models.Crawl) *gorm.DB {
+	if latest == nil {
+		return db.Where("golem15_seo_issues.crawl_id IS NULL")
+	}
+	return db.Where("golem15_seo_issues.crawl_id = ?", latest.ID)
+}
+
+// IgnoredSet returns which of the fingerprints are ignored.
+func IgnoredSet(db *gorm.DB, fingerprints []string) (map[string]bool, error) {
+	out := map[string]bool{}
+	if len(fingerprints) == 0 {
+		return out, nil
+	}
+	var rows []models.IssueIgnore
+	for start := 0; start < len(fingerprints); start += 1000 {
+		end := min(start+1000, len(fingerprints))
+		var part []models.IssueIgnore
+		if err := db.Where("fingerprint IN ?", fingerprints[start:end]).Find(&part).Error; err != nil {
+			return nil, err
+		}
+		rows = append(rows, part...)
+	}
+	for _, r := range rows {
+		out[r.Fingerprint] = true
+	}
+	return out, nil
+}
+
+// Current returns the crawl whose issues are the audit's current ones (nil
+// before any crawl). Before any crawl, the CMS-data checks run now and
+// their issues replace the stored ones, so the list shows the CMS data as
+// it is (UI-SPEC C23: when the list is opened with no crawl yet).
+func Current(ctx context.Context, svc *classes.Service, db *gorm.DB) (*models.Crawl, error) {
+	latest, err := LatestCrawl(db)
+	if err != nil || latest != nil {
+		return latest, err
+	}
+	return nil, RefreshCMS(ctx, svc, db)
+}
+
+// RefreshCMS replaces the stored CMS-data issues of a site not crawled
+// yet with the report's current ones, in one transaction.
+func RefreshCMS(ctx context.Context, svc *classes.Service, db *gorm.DB) error {
+	return lagoon.Transaction(ctx, db, func(ctx context.Context, tx *gorm.DB) error {
+		cms, err := svc.Issues(classes.WithTx(ctx, tx))
+		if err != nil {
+			return err
+		}
+		if err := tx.Where("crawl_id IS NULL").Delete(&models.Issue{}).Error; err != nil {
+			return err
+		}
+		return SaveIssues(tx, nil, nil, cms)
+	})
+}
--- a/classes/issues.go
+++ b/classes/issues.go
@@ -7,6 +7,7 @@ import (
 	"strings"

 	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"gorm.io/gorm"
 )

 // The kinds of the CMS-data issue report (D-13, D-14 CMS part). The
@@ -309,3 +310,11 @@ func (s *Service) canonicalProblem(ctx context.Context, canonical string) (kind,
 	}
 	return IssueCanonicalNoindex, "", nil
 }
+
+// Actor names who runs an audit operation (a crawl start, a cancel, an
+// ignore): the backend administrator's id and name, and whether the call
+// came through MCP.
+func (s *Service) Actor(ctx context.Context, db *gorm.DB) (id *uint, name string, mcp bool) {
+	id, name = s.author(ctx, db, models.ViaAdmin)
+	return id, name, ViaOf(ctx) == models.ViaMCP
+}