sm-seo-plugin/crawler_test.go

package seo_test

import (
	"fmt"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/summercms/modules/pact"
)

// TestCrawlerTracer is the end-to-end path (D-14): an administrator
// starts a crawl through the API, the crawl job fetches the site with the
// crawler's User-Agent, and GET /issues lists the page with two H1
// headings as an error with how to fix it: code for H1 headings of the
// template, the record type's headings tool for an H1 inside the record's
// own content.
func TestCrawlerTracer(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, map[string][]crawlRecord{
		crawlPostType: {{id: "1", path: "/c/two-h1", title: "Post title", headings: []pact.ContentHeading{{Level: 1, Text: "Inside the body"}}}},
	})
	site := newFakeSite()
	site.sitemap("/", "/p/two-h1", "/c/two-h1")
	site.plant(
		doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/c/two-h1"}},
		doc{path: "/p/two-h1", title: "Template page", h1: []string{"Template page", "Another H1"}},
		doc{path: "/c/two-h1", title: "Post title", h1: []string{"Post title", "Inside the body"}},
	)

	env.crawlOnce(t, token, siteCtx(t, site))

	list := env.issues(t, token, "")
	tpl := findIssue(t, list, "h1_multiple", "/p/two-h1")
	if tpl.Severity != "error" || tpl.Fix != "code" || tpl.Tool != "" || tpl.Source != "crawl" || tpl.Details["what"] == nil || tpl.Details["count"] != float64(2) {
		t.Fatalf("template h1_multiple = %+v", tpl)
	}
	content := findIssue(t, list, "h1_multiple", "/c/two-h1")
	if content.Fix != "tool" || content.Tool != "acme_fix_headings" || content.CrawlID == nil || content.ID == 0 {
		t.Fatalf("content h1_multiple = %+v", content)
	}
	if !site.requested("/sitemap-pages-1.xml") || !site.requested("/c/two-h1") {
		t.Fatalf("crawler requests = %v", site.requests)
	}
	for _, ua := range site.agents {
		if ua != "SummerCMS-SEO-Crawler/1 (+https://www.example.test)" {
			t.Fatalf("User-Agent = %q", ua)
		}
	}
}

// checkedKinds are the crawl kinds TestCrawlerChecks compares.
var checkedKinds = map[string]bool{
	"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
	"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
	"duplicate_title": true, "missing_description": true, "duplicate_description": true,
	"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
	"h1_title_mismatch": true, "jsonld_invalid": true, "jsonld_multiple": true, "jsonld_required": true,
	"jsonld_recommended": true, "hreflang_reciprocal": true, "hreflang_error": true, "generic_anchor": true,
	"broken_link": true, "redirected_link": true, "image_alt": true, "image_size": true, "featured_lazy": true,
	"content_js": true, "orphan_page": true,
}

// onlyChecked keeps the "kind url" entries of checkedKinds.
func onlyChecked(list []string) []string {
	out := []string{}
	for _, e := range list {
		kind, _, _ := strings.Cut(e, " ")
		if checkedKinds[kind] {
			out = append(out, e)
		}
	}
	return out
}

// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
// C23 and the clean control page /clean): every planted issue is found
// exactly once with its severity and details, nothing is found on /clean,
// and the crawler never requests another host, a private address or plain
// http (T-33-43).
func TestCrawlerChecks(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, auditRecords())
	site := newFakeSite()
	want := plantAudit(site)

	ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
	env.crawlOnce(t, token, ctx)
	list := env.issues(t, token, "source=crawl")

	got := onlyChecked(issueSet(list, "crawl"))
	if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
		t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
	}
	for _, i := range list {
		if i.URL == "/clean" {
			t.Fatalf("issue on the control page: %+v", i)
		}
		if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
			t.Fatalf("issue without a fix route: %+v", i)
		}
	}
	for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
		if site.requested(part) {
			t.Fatalf("the crawler requested %s: %v", part, site.requests)
		}
	}

	detail := func(kind, url, key string, value any) {
		t.Helper()
		i := findIssue(t, list, kind, url)
		if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
			t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
		}
	}
	severity := func(kind, url, want string) {
		t.Helper()
		if i := findIssue(t, list, kind, url); i.Severity != want {
			t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
		}
	}
	detail("page_status", "/p/status-500", "status", 500)
	severity("page_status", "/p/status-500", "error")
	detail("sitemap_error", "/p/sitemap-404", "status", 404)
	detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
	detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
	detail("sitemap_blocked", "/p/blocked", "line", 2)
	detail("sitemap_missing", "/p/unlisted", "count", 1)
	severity("sitemap_missing", "/p/unlisted", "notice")
	detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
	detail("canonical_error", "/p/bad-canonical", "status", 404)
	detail("duplicate_title", "/p/dup-title-a", "count", 1)
	detail("h1_multiple", "/p/two-h1", "count", 2)
	detail("heading_skipped", "/p/skip", "from", 1)
	detail("heading_skipped", "/p/skip", "to", 3)
	detail("heading_empty", "/p/empty-heading", "level", 2)
	detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
	if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
		t.Fatalf("canonical_missing = %+v", i)
	}
	if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
		t.Fatalf("missing_title = %+v", i)
	}
	detail("jsonld_multiple", "/p/jsonld-two", "count", 2)
	detail("jsonld_required", "/p/jsonld-req", "type", "BreadcrumbList")
	detail("jsonld_required", "/p/jsonld-req", "property", "itemListElement")
	detail("jsonld_recommended", "/p/jsonld-rec", "property", "logo")
	if i := findIssue(t, list, "jsonld_recommended", "/p/jsonld-rec"); i.Tool != "seo_update_settings" || i.Severity != "notice" {
		t.Fatalf("jsonld_recommended = %+v", i)
	}
	if i := findIssue(t, list, "jsonld_invalid", "/p/jsonld-bad"); i.Details["error"] == nil || i.Fix != "code" {
		t.Fatalf("jsonld_invalid = %+v", i)
	}
	detail("hreflang_reciprocal", "/en/x", "locale", "pl")
	detail("hreflang_reciprocal", "/en/x", "url", "/pl/x")
	detail("hreflang_error", "/p/hreflang-err", "url", "/de/missing")
	detail("hreflang_error", "/p/hreflang-err", "status", 404)
	detail("generic_anchor", "/p/generic", "count", 2)
	detail("generic_anchor", "/p/generic", "text", "kliknij tutaj")
	detail("generic_anchor", "/p/generic", "target", "/clean")
	detail("broken_link", "/p/gone", "from", []string{"/"})
	if i := findIssue(t, list, "broken_link", "/p/gone"); i.Tool != "seo_create_redirect" || i.Severity != "error" {
		t.Fatalf("broken_link = %+v", i)
	}
	detail("redirected_link", "/p/moved", "target", "/clean")
	detail("image_alt", "/p/img-noalt", "count", 1)
	detail("image_size", "/p/img-nosize", "count", 1)
	detail("featured_lazy", "/p/lazy", "image", siteURL+"/hero.jpg")
	content := func(url, check, what string) {
		t.Helper()
		i := findIssue(t, list, "content_js", url)
		if i.Details["check"] != check || !strings.Contains(fmt.Sprint(i.Details["what"]), what) || i.Fix != "code" {
			t.Fatalf("content_js %s = %+v, want check %s naming %q", url, i, check, what)
		}
	}
	detail("orphan_page", "/p/orphan", "count", 1)
	if i := findIssue(t, list, "orphan_page", "/p/orphan"); !strings.Contains(fmt.Sprint(i.Details["suggestions"]), "/p/mentions") {
		t.Fatalf("orphan suggestions = %v", i.Details)
	}
	content("/c/js-h1", "cms", "Record heading")
	content("/c/js-words", "cms", "of 400 words")
	content("/c/js-links", "cms", "/c/parent")
	content("/p/render", "rendered", "/p/js-only")
	content("/p/render", "rendered", "words")
	content("/p/render-h1", "rendered", "H1")
	var crawlRow struct {
		RenderStatus string
		RenderPages  int
	}
	if err := env.db.Raw(`SELECT render_status, render_pages FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&crawlRow).Error; err != nil {
		t.Fatal(err)
	}
	if crawlRow.RenderStatus != "done" || crawlRow.RenderPages < 10 {
		t.Fatalf("render state = %+v", crawlRow)
	}
}

// TestRenderSkipped checks the comparison's absence and failure (D-70,
// P33-55): without a renderer (no golem15.seo.crawl_chrome_path) the crawl
// runs every raw check and records render_status skipped with no rendered
// finding; a renderer whose Render fails records render_status failed with
// its note, and the crawl still finishes with the raw checks.
func TestRenderSkipped(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, auditRecords())
	site := newFakeSite()
	plantAudit(site)

	renderState := func(id uint) (status string, pages int, note, crawlStatus string) {
		t.Helper()
		var row struct {
			RenderStatus string
			RenderPages  int
			RenderNote   string
			Status       string
		}
		if err := env.db.Raw(`SELECT render_status, render_pages, render_note, status FROM golem15_seo_crawls WHERE id = ?`, id).Scan(&row).Error; err != nil {
			t.Fatal(err)
		}
		return row.RenderStatus, row.RenderPages, row.RenderNote, row.Status
	}

	id := env.crawlOnce(t, token, siteCtx(t, site))
	status, pages, _, crawlStatus := renderState(id)
	if status != "skipped" || pages != 0 || crawlStatus != "finished" {
		t.Fatalf("without a renderer: render %s, %d pages, crawl %s", status, pages, crawlStatus)
	}
	list := env.issues(t, token, "")
	for _, i := range list {
		if i.Kind == "content_js" && i.Details["check"] == "rendered" {
			t.Fatalf("rendered finding without a renderer: %+v", i)
		}
	}
	findIssue(t, list, "content_js", "/c/js-h1")
	findIssue(t, list, "h1_multiple", "/p/two-h1")

	failing := &fakeRenderer{site: site, fail: fmt.Errorf("chrome exited")}
	id = env.crawlOnce(t, token, crawl.WithRenderer(siteCtx(t, site), failing))
	status, _, note, crawlStatus := renderState(id)
	if status != "failed" || !strings.Contains(note, "chrome exited") || crawlStatus != "finished" {
		t.Fatalf("failing renderer: render %s (%q), crawl %s", status, note, crawlStatus)
	}
	findIssue(t, env.issues(t, token, ""), "h1_multiple", "/p/two-h1")
}

// TestIssueMerge checks the CMS-data issues stored with the crawl and the
// fix routes (SC-3, P33-38): before any crawl the list holds the CMS
// issues; after it they are stored with the crawl as source cms; image_alt
// names the record type's image_alt tool; h1_in_content appears for a post
// and a category whose descriptor sets TopHeading 2 and not for TopHeading
// 0, naming the headings tool; a skipped level in a category's pillar text
// (a type without an edit tool) is fix code; no issue lacks a route. An
// ignored fingerprint stays ignored in the next crawl, and a fixed page's
// issue is gone from it.
func TestIssueMerge(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, map[string][]crawlRecord{
		crawlPostType: {
			{id: "1", path: "/c/post-a", title: "Post A", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1"}, {Level: 2, Text: "Part"}},
				images: []pact.ContentImage{{URL: "/img/a.jpg"}, {URL: "/img/deco.png", Decorative: true}}},
			{id: "2", path: "/c/post-b", title: "Post B", topHeading: 0, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1 of B"}}},
		},
		crawlCategoryType: {
			{id: "10", path: "/cat/news", title: "News", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Pillar H1"}, {Level: 2, Text: "Intro"}, {Level: 4, Text: "Deep"}}},
		},
	})
	site := newFakeSite()
	site.sitemap("/", "/c/post-a", "/c/post-b", "/cat/news", "/p/fixme")
	site.plant(
		doc{path: "/", title: "Home", links: []string{"/c/post-a", "/c/post-b", "/cat/news", "/p/fixme"}},
		doc{path: "/c/post-a", title: "Post A", body: `<h2>Part</h2><img src="/img/a.jpg" alt="Photo" width="1" height="1">`},
		doc{path: "/c/post-b", title: "Post B"},
		doc{path: "/cat/news", title: "News", body: "<h2>Intro</h2><h4>Deep</h4>"},
		doc{path: "/p/fixme", title: "Fix me", noDesc: true},
	)

	before := env.issues(t, token, "")
	if i := findIssue(t, before, "image_alt", "/c/post-a"); i.Source != "cms" || i.CrawlID != nil || i.Tool != "acme_set_alt" {
		t.Fatalf("CMS image_alt before any crawl = %+v", i)
	}

	ctx := siteCtx(t, site)
	first := env.crawlOnce(t, token, ctx)
	list := env.issues(t, token, "")
	alt := findIssue(t, list, "image_alt", "/c/post-a")
	if alt.Source != "cms" || alt.CrawlID == nil || *alt.CrawlID != first || alt.Tool != "acme_set_alt" || alt.Fix != "tool" {
		t.Fatalf("image_alt = %+v", alt)
	}
	if i := findIssue(t, list, "h1_in_content", "/c/post-a"); i.Tool != "acme_fix_headings" || i.Source != "cms" || i.Severity != "warning" {
		t.Fatalf("h1_in_content post = %+v", i)
	}
	if i := findIssue(t, list, "h1_in_content", "/cat/news"); i.Tool != "acme_category_headings" {
		t.Fatalf("h1_in_content category = %+v", i)
	}
	for _, i := range list {
		if i.Kind == "h1_in_content" && i.URL == "/c/post-b" {
			t.Fatalf("h1_in_content for TopHeading 0: %+v", i)
		}
		if i.Fix == "tool" && i.Tool == "" || i.Fix != "tool" && i.Fix != "code" {
			t.Fatalf("issue without a route: %+v", i)
		}
		if i.Fix == "code" && i.Details["what"] == nil {
			t.Fatalf("fix code without what: %+v", i)
		}
	}
	skipped := findIssue(t, list, "heading_skipped", "/cat/news")
	if skipped.Fix != "code" || skipped.Tool != "" {
		t.Fatalf("heading_skipped in pillar text = %+v", skipped)
	}
	findIssue(t, list, "missing_description", "/p/fixme")

	// Ignore the skipped level, fix the description, crawl again.
	var fp string
	if err := env.db.Raw(`SELECT fingerprint FROM golem15_seo_issues WHERE id = ?`, skipped.ID).Scan(&fp).Error; err != nil || fp == "" {
		t.Fatalf("fingerprint: %q %v", fp, err)
	}
	if err := env.db.Exec(`INSERT INTO golem15_seo_issue_ignores (fingerprint, ignored_at) VALUES (?, NOW())`, fp).Error; err != nil {
		t.Fatal(err)
	}
	site.plant(doc{path: "/p/fixme", title: "Fix me"})
	second := env.crawlOnce(t, token, ctx)
	list = env.issues(t, token, "")
	again := findIssue(t, list, "heading_skipped", "/cat/news")
	if !again.Ignored || again.CrawlID == nil || *again.CrawlID != second {
		t.Fatalf("ignored issue in the next crawl = %+v", again)
	}
	for _, i := range list {
		if i.URL == "/p/fixme" {
			t.Fatalf("fixed page still has %+v", i)
		}
	}
}

// TestStagingCrawl checks RESEARCH Pitfall 12: on a non-production
// environment every page carries the staging lock's noindex, and no
// noindex-derived issue is reported; the pages still count as indexable.
func TestStagingCrawl(t *testing.T) {
	env := bootCrawlEnv(t, "SUMMER_ENV=staging")
	if !env.seoPlugin(t).Service().Staging() {
		t.Fatal("the environment is not staging")
	}
	token := env.admin(t, true)
	site := newFakeSite()
	site.sitemap("/", "/a", "/b")
	for _, d := range []doc{
		{path: "/", title: "Home", links: []string{"/a", "/b", "/c"}},
		{path: "/a", title: "Same"},
		{path: "/b", title: "Same"},
		{path: "/c", title: "Unlisted"},
	} {
		d.robots = "noindex, nofollow"
		site.set(d.path, 200, d.render(), "X-Robots-Tag", "noindex")
	}
	env.crawlOnce(t, token, siteCtx(t, site))
	list := env.issues(t, token, "")
	for _, i := range list {
		if i.Kind == "sitemap_noindex" || i.Kind == "canonical_noindex" || i.Kind == "noindex_blocked" {
			t.Fatalf("staging noindex reported: %+v", i)
		}
	}
	findIssue(t, list, "duplicate_title", "/a")
	findIssue(t, list, "sitemap_missing", "/c")
}