sm-seo-plugin/crawler_test.go

package seo_test

import (
	"fmt"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/summercms/modules/pact"
)

// TestCrawlerTracer is the end-to-end path (D-14): an administrator
// starts a crawl through the API, the crawl job fetches the site with the
// crawler's User-Agent, and GET /issues lists the page with two H1
// headings as an error with how to fix it: code for H1 headings of the
// template, the record type's headings tool for an H1 inside the record's
// own content.
func TestCrawlerTracer(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, map[string][]crawlRecord{
		crawlPostType: {{id: "1", path: "/c/two-h1", title: "Post title", headings: []pact.ContentHeading{{Level: 1, Text: "Inside the body"}}}},
	})
	site := newFakeSite()
	site.sitemap("/", "/p/two-h1", "/c/two-h1")
	site.plant(
		doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/c/two-h1"}},
		doc{path: "/p/two-h1", title: "Template page", h1: []string{"Template page", "Another H1"}},
		doc{path: "/c/two-h1", title: "Post title", h1: []string{"Post title", "Inside the body"}},
	)

	env.crawlOnce(t, token, siteCtx(t, site))

	list := env.issues(t, token, "")
	tpl := findIssue(t, list, "h1_multiple", "/p/two-h1")
	if tpl.Severity != "error" || tpl.Fix != "code" || tpl.Tool != "" || tpl.Source != "crawl" || tpl.Details["what"] == nil || tpl.Details["count"] != float64(2) {
		t.Fatalf("template h1_multiple = %+v", tpl)
	}
	content := findIssue(t, list, "h1_multiple", "/c/two-h1")
	if content.Fix != "tool" || content.Tool != "acme_fix_headings" || content.CrawlID == nil || content.ID == 0 {
		t.Fatalf("content h1_multiple = %+v", content)
	}
	if !site.requested("/sitemap-pages-1.xml") || !site.requested("/c/two-h1") {
		t.Fatalf("crawler requests = %v", site.requests)
	}
	for _, ua := range site.agents {
		if ua != "SummerCMS-SEO-Crawler/1 (+https://www.example.test)" {
			t.Fatalf("User-Agent = %q", ua)
		}
	}
}

// checkedKinds are the crawl kinds TestCrawlerChecks compares.
var checkedKinds = map[string]bool{
	"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
	"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
	"duplicate_title": true, "missing_description": true, "duplicate_description": true,
	"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
	"h1_title_mismatch": true, "jsonld_invalid": true, "jsonld_multiple": true, "jsonld_required": true,
	"jsonld_recommended": true, "hreflang_reciprocal": true, "hreflang_error": true, "generic_anchor": true,
	"broken_link": true, "redirected_link": true, "image_alt": true, "image_size": true, "featured_lazy": true,
	"content_js": true,
}

// onlyChecked keeps the "kind url" entries of checkedKinds.
func onlyChecked(list []string) []string {
	out := []string{}
	for _, e := range list {
		kind, _, _ := strings.Cut(e, " ")
		if checkedKinds[kind] {
			out = append(out, e)
		}
	}
	return out
}

// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
// C23 and the clean control page /clean): every planted issue is found
// exactly once with its severity and details, nothing is found on /clean,
// and the crawler never requests another host, a private address or plain
// http (T-33-43).
func TestCrawlerChecks(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, auditRecords())
	site := newFakeSite()
	want := plantAudit(site)

	ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
	env.crawlOnce(t, token, ctx)
	list := env.issues(t, token, "source=crawl")

	got := onlyChecked(issueSet(list, "crawl"))
	if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
		t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
	}
	for _, i := range list {
		if i.URL == "/clean" {
			t.Fatalf("issue on the control page: %+v", i)
		}
		if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
			t.Fatalf("issue without a fix route: %+v", i)
		}
	}
	for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
		if site.requested(part) {
			t.Fatalf("the crawler requested %s: %v", part, site.requests)
		}
	}

	detail := func(kind, url, key string, value any) {
		t.Helper()
		i := findIssue(t, list, kind, url)
		if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
			t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
		}
	}
	severity := func(kind, url, want string) {
		t.Helper()
		if i := findIssue(t, list, kind, url); i.Severity != want {
			t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
		}
	}
	detail("page_status", "/p/status-500", "status", 500)
	severity("page_status", "/p/status-500", "error")
	detail("sitemap_error", "/p/sitemap-404", "status", 404)
	detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
	detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
	detail("sitemap_blocked", "/p/blocked", "line", 2)
	detail("sitemap_missing", "/p/unlisted", "count", 1)
	severity("sitemap_missing", "/p/unlisted", "notice")
	detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
	detail("canonical_error", "/p/bad-canonical", "status", 404)
	detail("duplicate_title", "/p/dup-title-a", "count", 1)
	detail("h1_multiple", "/p/two-h1", "count", 2)
	detail("heading_skipped", "/p/skip", "from", 1)
	detail("heading_skipped", "/p/skip", "to", 3)
	detail("heading_empty", "/p/empty-heading", "level", 2)
	detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
	if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
		t.Fatalf("canonical_missing = %+v", i)
	}
	if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
		t.Fatalf("missing_title = %+v", i)
	}
	detail("jsonld_multiple", "/p/jsonld-two", "count", 2)
	detail("jsonld_required", "/p/jsonld-req", "type", "BreadcrumbList")
	detail("jsonld_required", "/p/jsonld-req", "property", "itemListElement")
	detail("jsonld_recommended", "/p/jsonld-rec", "property", "logo")
	if i := findIssue(t, list, "jsonld_recommended", "/p/jsonld-rec"); i.Tool != "seo_update_settings" || i.Severity != "notice" {
		t.Fatalf("jsonld_recommended = %+v", i)
	}
	if i := findIssue(t, list, "jsonld_invalid", "/p/jsonld-bad"); i.Details["error"] == nil || i.Fix != "code" {
		t.Fatalf("jsonld_invalid = %+v", i)
	}
	detail("hreflang_reciprocal", "/en/x", "locale", "pl")
	detail("hreflang_reciprocal", "/en/x", "url", "/pl/x")
	detail("hreflang_error", "/p/hreflang-err", "url", "/de/missing")
	detail("hreflang_error", "/p/hreflang-err", "status", 404)
	detail("generic_anchor", "/p/generic", "count", 2)
	detail("generic_anchor", "/p/generic", "text", "kliknij tutaj")
	detail("generic_anchor", "/p/generic", "target", "/clean")
	detail("broken_link", "/p/gone", "from", []string{"/"})
	if i := findIssue(t, list, "broken_link", "/p/gone"); i.Tool != "seo_create_redirect" || i.Severity != "error" {
		t.Fatalf("broken_link = %+v", i)
	}
	detail("redirected_link", "/p/moved", "target", "/clean")
	detail("image_alt", "/p/img-noalt", "count", 1)
	detail("image_size", "/p/img-nosize", "count", 1)
	detail("featured_lazy", "/p/lazy", "image", siteURL+"/hero.jpg")
	content := func(url, check, what string) {
		t.Helper()
		i := findIssue(t, list, "content_js", url)
		if i.Details["check"] != check || !strings.Contains(fmt.Sprint(i.Details["what"]), what) || i.Fix != "code" {
			t.Fatalf("content_js %s = %+v, want check %s naming %q", url, i, check, what)
		}
	}
	content("/c/js-h1", "cms", "Record heading")
	content("/c/js-words", "cms", "of 400 words")
	content("/c/js-links", "cms", "/c/parent")
	content("/p/render", "rendered", "/p/js-only")
	content("/p/render", "rendered", "words")
	content("/p/render-h1", "rendered", "H1")
	var crawlRow struct {
		RenderStatus string
		RenderPages  int
	}
	if err := env.db.Raw(`SELECT render_status, render_pages FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&crawlRow).Error; err != nil {
		t.Fatal(err)
	}
	if crawlRow.RenderStatus != "done" || crawlRow.RenderPages < 10 {
		t.Fatalf("render state = %+v", crawlRow)
	}
}

// TestRenderSkipped checks the comparison's absence and failure (D-70,
// P33-55): without a renderer (no golem15.seo.crawl_chrome_path) the crawl
// runs every raw check and records render_status skipped with no rendered
// finding; a renderer whose Render fails records render_status failed with
// its note, and the crawl still finishes with the raw checks.
func TestRenderSkipped(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, auditRecords())
	site := newFakeSite()
	plantAudit(site)

	renderState := func(id uint) (status string, pages int, note, crawlStatus string) {
		t.Helper()
		var row struct {
			RenderStatus string
			RenderPages  int
			RenderNote   string
			Status       string
		}
		if err := env.db.Raw(`SELECT render_status, render_pages, render_note, status FROM golem15_seo_crawls WHERE id = ?`, id).Scan(&row).Error; err != nil {
			t.Fatal(err)
		}
		return row.RenderStatus, row.RenderPages, row.RenderNote, row.Status
	}

	id := env.crawlOnce(t, token, siteCtx(t, site))
	status, pages, _, crawlStatus := renderState(id)
	if status != "skipped" || pages != 0 || crawlStatus != "finished" {
		t.Fatalf("without a renderer: render %s, %d pages, crawl %s", status, pages, crawlStatus)
	}
	list := env.issues(t, token, "")
	for _, i := range list {
		if i.Kind == "content_js" && i.Details["check"] == "rendered" {
			t.Fatalf("rendered finding without a renderer: %+v", i)
		}
	}
	findIssue(t, list, "content_js", "/c/js-h1")
	findIssue(t, list, "h1_multiple", "/p/two-h1")

	failing := &fakeRenderer{site: site, fail: fmt.Errorf("chrome exited")}
	id = env.crawlOnce(t, token, crawl.WithRenderer(siteCtx(t, site), failing))
	status, _, note, crawlStatus := renderState(id)
	if status != "failed" || !strings.Contains(note, "chrome exited") || crawlStatus != "finished" {
		t.Fatalf("failing renderer: render %s (%q), crawl %s", status, note, crawlStatus)
	}
	findIssue(t, env.issues(t, token, ""), "h1_multiple", "/p/two-h1")
}