feat: orphan pages and merged CMS issues with fix routes

- orphan_page for listed indexable pages no crawled page links to, with
  pages mentioning the title or focus keyphrase as suggestions (D-82)
- CMS-data image_alt and h1_in_content from the content descriptors
  (TopHeading 2 only), stored with the crawl as source cms (D-85, D-50)
- seo-page names Tools edit seo_update_page; every stored issue carries
  a tool or fix code with details.what (P33-38)
- TestIssueMerge (ignored fingerprints survive the next crawl, fixed
  pages lose their issues) and TestStagingCrawl (Pitfall 12)

crawler_test.go

@@ -59,7 +59,7 @@ var checkedKinds = map[string]bool{
 	"h1_title_mismatch": true, "jsonld_invalid": true, "jsonld_multiple": true, "jsonld_required": true,
 	"jsonld_recommended": true, "hreflang_reciprocal": true, "hreflang_error": true, "generic_anchor": true,
 	"broken_link": true, "redirected_link": true, "image_alt": true, "image_size": true, "featured_lazy": true,
-	"content_js": true,
+	"content_js": true, "orphan_page": true,
 }
 
 // onlyChecked keeps the "kind url" entries of checkedKinds.
@@ -175,6 +175,10 @@ func TestCrawlerChecks(t *testing.T) {
 			t.Fatalf("content_js %s = %+v, want check %s naming %q", url, i, check, what)
 		}
 	}
+	detail("orphan_page", "/p/orphan", "count", 1)
+	if i := findIssue(t, list, "orphan_page", "/p/orphan"); !strings.Contains(fmt.Sprint(i.Details["suggestions"]), "/p/mentions") {
+		t.Fatalf("orphan suggestions = %v", i.Details)
+	}
 	content("/c/js-h1", "cms", "Record heading")
 	content("/c/js-words", "cms", "of 400 words")
 	content("/c/js-links", "cms", "/c/parent")
@@ -241,3 +245,123 @@ func TestRenderSkipped(t *testing.T) {
 	}
 	findIssue(t, env.issues(t, token, ""), "h1_multiple", "/p/two-h1")
 }
+
+// TestIssueMerge checks the CMS-data issues stored with the crawl and the
+// fix routes (SC-3, P33-38): before any crawl the list holds the CMS
+// issues; after it they are stored with the crawl as source cms; image_alt
+// names the record type's image_alt tool; h1_in_content appears for a post
+// and a category whose descriptor sets TopHeading 2 and not for TopHeading
+// 0, naming the headings tool; a skipped level in a category's pillar text
+// (a type without an edit tool) is fix code; no issue lacks a route. An
+// ignored fingerprint stays ignored in the next crawl, and a fixed page's
+// issue is gone from it.
+func TestIssueMerge(t *testing.T) {
+	env := bootCrawlEnv(t)
+	token := env.admin(t, true)
+	setCrawlRecords(t, map[string][]crawlRecord{
+		crawlPostType: {
+			{id: "1", path: "/c/post-a", title: "Post A", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1"}, {Level: 2, Text: "Part"}},
+				images: []pact.ContentImage{{URL: "/img/a.jpg"}, {URL: "/img/deco.png", Decorative: true}}},
+			{id: "2", path: "/c/post-b", title: "Post B", topHeading: 0, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1 of B"}}},
+		},
+		crawlCategoryType: {
+			{id: "10", path: "/cat/news", title: "News", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Pillar H1"}, {Level: 2, Text: "Intro"}, {Level: 4, Text: "Deep"}}},
+		},
+	})
+	site := newFakeSite()
+	site.sitemap("/", "/c/post-a", "/c/post-b", "/cat/news", "/p/fixme")
+	site.plant(
+		doc{path: "/", title: "Home", links: []string{"/c/post-a", "/c/post-b", "/cat/news", "/p/fixme"}},
+		doc{path: "/c/post-a", title: "Post A", body: `<h2>Part</h2><img src="/img/a.jpg" alt="Photo" width="1" height="1">`},
+		doc{path: "/c/post-b", title: "Post B"},
+		doc{path: "/cat/news", title: "News", body: "<h2>Intro</h2><h4>Deep</h4>"},
+		doc{path: "/p/fixme", title: "Fix me", noDesc: true},
+	)
+
+	before := env.issues(t, token, "")
+	if i := findIssue(t, before, "image_alt", "/c/post-a"); i.Source != "cms" || i.CrawlID != nil || i.Tool != "acme_set_alt" {
+		t.Fatalf("CMS image_alt before any crawl = %+v", i)
+	}
+
+	ctx := siteCtx(t, site)
+	first := env.crawlOnce(t, token, ctx)
+	list := env.issues(t, token, "")
+	alt := findIssue(t, list, "image_alt", "/c/post-a")
+	if alt.Source != "cms" || alt.CrawlID == nil || *alt.CrawlID != first || alt.Tool != "acme_set_alt" || alt.Fix != "tool" {
+		t.Fatalf("image_alt = %+v", alt)
+	}
+	if i := findIssue(t, list, "h1_in_content", "/c/post-a"); i.Tool != "acme_fix_headings" || i.Source != "cms" || i.Severity != "warning" {
+		t.Fatalf("h1_in_content post = %+v", i)
+	}
+	if i := findIssue(t, list, "h1_in_content", "/cat/news"); i.Tool != "acme_category_headings" {
+		t.Fatalf("h1_in_content category = %+v", i)
+	}
+	for _, i := range list {
+		if i.Kind == "h1_in_content" && i.URL == "/c/post-b" {
+			t.Fatalf("h1_in_content for TopHeading 0: %+v", i)
+		}
+		if i.Fix == "tool" && i.Tool == "" || i.Fix != "tool" && i.Fix != "code" {
+			t.Fatalf("issue without a route: %+v", i)
+		}
+		if i.Fix == "code" && i.Details["what"] == nil {
+			t.Fatalf("fix code without what: %+v", i)
+		}
+	}
+	skipped := findIssue(t, list, "heading_skipped", "/cat/news")
+	if skipped.Fix != "code" || skipped.Tool != "" {
+		t.Fatalf("heading_skipped in pillar text = %+v", skipped)
+	}
+	findIssue(t, list, "missing_description", "/p/fixme")
+
+	// Ignore the skipped level, fix the description, crawl again.
+	var fp string
+	if err := env.db.Raw(`SELECT fingerprint FROM golem15_seo_issues WHERE id = ?`, skipped.ID).Scan(&fp).Error; err != nil || fp == "" {
+		t.Fatalf("fingerprint: %q %v", fp, err)
+	}
+	if err := env.db.Exec(`INSERT INTO golem15_seo_issue_ignores (fingerprint, ignored_at) VALUES (?, NOW())`, fp).Error; err != nil {
+		t.Fatal(err)
+	}
+	site.plant(doc{path: "/p/fixme", title: "Fix me"})
+	second := env.crawlOnce(t, token, ctx)
+	list = env.issues(t, token, "")
+	again := findIssue(t, list, "heading_skipped", "/cat/news")
+	if !again.Ignored || again.CrawlID == nil || *again.CrawlID != second {
+		t.Fatalf("ignored issue in the next crawl = %+v", again)
+	}
+	for _, i := range list {
+		if i.URL == "/p/fixme" {
+			t.Fatalf("fixed page still has %+v", i)
+		}
+	}
+}
+
+// TestStagingCrawl checks RESEARCH Pitfall 12: on a non-production
+// environment every page carries the staging lock's noindex, and no
+// noindex-derived issue is reported; the pages still count as indexable.
+func TestStagingCrawl(t *testing.T) {
+	env := bootCrawlEnv(t, "SUMMER_ENV=staging")
+	if !env.seoPlugin(t).Service().Staging() {
+		t.Fatal("the environment is not staging")
+	}
+	token := env.admin(t, true)
+	site := newFakeSite()
+	site.sitemap("/", "/a", "/b")
+	for _, d := range []doc{
+		{path: "/", title: "Home", links: []string{"/a", "/b", "/c"}},
+		{path: "/a", title: "Same"},
+		{path: "/b", title: "Same"},
+		{path: "/c", title: "Unlisted"},
+	} {
+		d.robots = "noindex, nofollow"
+		site.set(d.path, 200, d.render(), "X-Robots-Tag", "noindex")
+	}
+	env.crawlOnce(t, token, siteCtx(t, site))
+	list := env.issues(t, token, "")
+	for _, i := range list {
+		if i.Kind == "sitemap_noindex" || i.Kind == "canonical_noindex" || i.Kind == "noindex_blocked" {
+			t.Fatalf("staging noindex reported: %+v", i)
+		}
+	}
+	findIssue(t, list, "duplicate_title", "/a")
+	findIssue(t, list, "sitemap_missing", "/c")
+}