feat: orphan pages and merged CMS issues with fix routes

- orphan_page for listed indexable pages no crawled page links to, with
  pages mentioning the title or focus keyphrase as suggestions (D-82)
- CMS-data image_alt and h1_in_content from the content descriptors
  (TopHeading 2 only), stored with the crawl as source cms (D-85, D-50)
- seo-page names Tools edit seo_update_page; every stored issue carries
  a tool or fix code with details.what (P33-38)
- TestIssueMerge (ignored fingerprints survive the next crawl, fixed
  pages lose their issues) and TestStagingCrawl (Pitfall 12)

classes/crawl/checks.go

@@ -125,6 +125,8 @@ type Env struct {
 	Staging bool
 	// GenericAnchors are the generic link texts (any case).
 	GenericAnchors []string
+	// Keyphrases maps an address to its SEO entry's focus keyphrase.
+	Keyphrases map[string]string
 }
 
 func (e *Env) record(key string) *Record {
@@ -198,6 +200,7 @@ func Check(res *Result, env *Env) []Finding {
 		out = append(out, checkRenderedPage(res, p, rec)...)
 	}
 	out = append(out, checkLinks(res, env)...)
+	out = append(out, checkOrphans(res, env)...)
 	out = append(out, checkSitemaps(res, env)...)
 	out = append(out, checkDuplicates(res, env)...)
 	return out

classes/crawl/job.go

@@ -221,7 +221,11 @@ func finish(ctx context.Context, svc *classes.Service, db *gorm.DB, row models.C
 		if err := SavePages(tx, row.ID, res); err != nil {
 			return err
 		}
-		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors})
+		keyphrases, err := Keyphrases(tx)
+		if err != nil {
+			return err
+		}
+		findings := Check(res, &Env{Lookup: Lookup(ctx, svc, tx), Staging: svc.Staging(), GenericAnchors: DefaultGenericAnchors, Keyphrases: keyphrases})
 		cms, err := svc.Issues(classes.WithTx(ctx, tx))
 		if err != nil {
 			return err
@@ -325,6 +329,22 @@ func Lookup(ctx context.Context, svc *classes.Service, db *gorm.DB) func(key str
 	}
 }
 
+// Keyphrases maps each address with an SEO entry to its focus keyphrase.
+func Keyphrases(db *gorm.DB) (map[string]string, error) {
+	var rows []struct {
+		URL       string
+		Keyphrase string
+	}
+	if err := db.Raw(`SELECT url, keyphrase FROM golem15_seo_entries WHERE url <> '' AND keyphrase <> ''`).Scan(&rows).Error; err != nil {
+		return nil, err
+	}
+	out := make(map[string]string, len(rows))
+	for _, r := range rows {
+		out[r.URL] = r.Keyphrase
+	}
+	return out, nil
+}
+
 // errCancelledCrawl refuses cancelling a crawl that is not queued or
 // running.
 var errCancelledCrawl = errors.New("golem15.seo: the crawl is not running")

classes/crawl/orphans.go

@@ -0,0 +1,79 @@
+package crawl
+
+import (
+	"strings"
+
+	"git.golem15.com/golem15/summercms/modules/pact"
+)
+
+// maxSuggestions caps the pages suggested to link to an orphan.
+const maxSuggestions = 10
+
+// minPhrase is the shortest phrase a suggestion matches (in bytes).
+const minPhrase = 3
+
+// checkOrphans reports orphan_page (D-82): an indexable page a sitemap
+// lists that no crawled page links to (the home page is the crawl's
+// entry and never an orphan). Its suggestions are the pages whose main
+// text mentions the orphan's title or its focus keyphrase; the first
+// suggestion's record names the tool that adds the link, else the fix is
+// a link in the site's code.
+func checkOrphans(res *Result, env *Env) []Finding {
+	links := inbound(res)
+	var out []Finding
+	for _, key := range res.Order {
+		p := res.Pages[key]
+		if key == HomePath || !p.InSitemap || !env.indexable(res, p) || len(links[key]) > 0 {
+			continue
+		}
+		rec := env.record(key)
+		var phrases []string
+		add := func(s string) {
+			if s = normal(s); len(s) >= minPhrase {
+				for _, have := range phrases {
+					if have == s {
+						return
+					}
+				}
+				phrases = append(phrases, s)
+			}
+		}
+		if rec != nil {
+			add(rec.Title)
+		}
+		if h1 := p.Facts.H1(); len(h1) > 0 {
+			add(h1[0])
+		}
+		add(p.Facts.Title)
+		if env.Keyphrases != nil {
+			add(env.Keyphrases[key])
+		}
+		var suggestions []map[string]any
+		var first *Record
+		for _, other := range res.Order {
+			if len(suggestions) >= maxSuggestions {
+				break
+			}
+			op := res.Pages[other]
+			if other == key || !op.OK() {
+				continue
+			}
+			text := normal(op.Facts.Text)
+			for _, phrase := range phrases {
+				if strings.Contains(text, phrase) {
+					suggestions = append(suggestions, map[string]any{"url": other, "phrase": phrase})
+					if first == nil {
+						first = env.record(other)
+					}
+					break
+				}
+			}
+		}
+		f := Finding{Kind: KindOrphanPage, URL: key, Locale: locale(rec), Record: first, Details: map[string]any{"count": len(suggestions), "suggestions": suggestions}}
+		if first != nil {
+			f.Content = []string{pact.ContentToolEdit}
+		}
+		out = append(out, f)
+	}
+	return out
+}

classes/issues.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"git.golem15.com/golem15/sm-seo-plugin/models"
+	"git.golem15.com/golem15/summercms/modules/pact"
 	"gorm.io/gorm"
 )
 
@@ -26,6 +27,13 @@ const (
 	// keeps Googlebot from (D-24): an error, since the sitemap asks for a
 	// page the crawler may not fetch.
 	IssueSitemapBlocked = "sitemap_blocked"
+	// IssueImageAlt is a record image without alternative text that is
+	// not marked decorative (D-85), from the content descriptors.
+	IssueImageAlt = "image_alt"
+	// IssueH1InContent is an H1 in the body of a record whose page renders
+	// the title as the only H1 (descriptor TopHeading 2, D-50; Journal's
+	// H1 setting on).
+	IssueH1InContent = "h1_in_content"
 )
 
 // The severities an issue may name.
@@ -92,7 +100,92 @@ func (s *Service) Issues(ctx context.Context) ([]Issue, error) {
 		data.listed = append(data.listed, address)
 	}
 	sort.Strings(data.listed)
-	return s.issuesOf(ctx, data, s.canonicalProblem)
+	out, err := s.issuesOf(ctx, data, s.canonicalProblem)
+	if err != nil {
+		return nil, err
+	}
+	content, err := s.ContentIssues(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return append(out, content...), nil
+}
+
+// contentIssuesPage is the page size of the content walk.
+const contentIssuesPage = 200
+
+// ContentIssues are the CMS-data checks on the content descriptors of
+// every published record (D-85, D-50): image_alt for images with an empty
+// alt that are not decorative, and h1_in_content for an H1 in the body of
+// a record whose descriptor sets TopHeading 2 (off when the type sets no
+// rule, as Journal does with its H1 setting off). Each names the record
+// type's fitting tool (ContentDescriptor.Tools: image_alt or headings,
+// else edit); "" when the type names none.
+func (s *Service) ContentIssues(ctx context.Context) ([]Issue, error) {
+	reg := s.Registry()
+	if reg == nil {
+		return nil, nil
+	}
+	var out []Issue
+	for _, typ := range reg.ContentTypes() {
+		if typ.Content == nil || typ.Content.Records == nil {
+			continue
+		}
+		tool := func(ops ...string) string {
+			for _, op := range ops {
+				if t := strings.TrimSpace(typ.Content.Tools[op]); t != "" {
+					return t
+				}
+			}
+			return ""
+		}
+		after := ""
+		for {
+			page, err := reg.Records(ctx, typ.Key, after, contentIssuesPage, s.UnprefixedLocale())
+			if err != nil {
+				return nil, err
+			}
+			for _, rec := range page.Records {
+				if !rec.Published {
+					continue
+				}
+				for _, l := range rec.Locales {
+					if strings.TrimSpace(l.URL) == "" {
+						continue
+					}
+					var missing []string
+					for _, img := range l.Images {
+						if !img.Decorative && strings.TrimSpace(img.Alt) == "" && img.URL != "" {
+							missing = append(missing, img.URL)
+						}
+					}
+					if len(missing) > 0 {
+						out = append(out, Issue{Kind: IssueImageAlt, URL: l.URL, Locale: l.Locale, Tool: tool(pact.ContentToolImageAlt, pact.ContentToolEdit), Details: map[string]any{
+							"count": len(missing), "images": missing, "type": typ.Key, "id": rec.ID,
+						}})
+					}
+					if rec.TopHeading == 2 {
+						var h1 []string
+						for _, h := range l.Headings {
+							if h.Level == 1 {
+								h1 = append(h1, h.Text)
+							}
+						}
+						if len(h1) > 0 {
+							out = append(out, Issue{Kind: IssueH1InContent, URL: l.URL, Locale: l.Locale, Tool: tool(pact.ContentToolHeadings, pact.ContentToolEdit), Details: map[string]any{
+								"count": len(h1), "h1": h1, "type": typ.Key, "id": rec.ID,
+							}})
+						}
+					}
+				}
+			}
+			if page.Next == "" || page.Next == after || len(page.Records) == 0 {
+				break
+			}
+			after = page.Next
+		}
+	}
+	return out, nil
 }
 
 // issueData is what the issue report reads from the database: the SEO

classes/registry.go

@@ -247,6 +247,9 @@ func PageMenuItemType(service func() *Service) pact.MenuItemType {
 			Controller: PagesController,
 			Model:      PageModel,
 			Kind:       pact.ContentKindPage,
+			// A registry page's fields are changed through seo_update_page,
+			// the tool the site audit names for its content (P33-38).
+			Tools: map[string]string{pact.ContentToolEdit: "seo_update_page"},
 			Records: func(ctx context.Context, in pact.ContentList) (pact.ContentPage, error) {
 				s, err := svc()
 				if err != nil {