sm-seo-plugin/crawler_test.go

package seo_test

import (
	"fmt"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
	"git.golem15.com/golem15/summercms/modules/pact"
)

// TestCrawlerTracer is the end-to-end path (D-14): an administrator
// starts a crawl through the API, the crawl job fetches the site with the
// crawler's User-Agent, and GET /issues lists the page with two H1
// headings as an error with how to fix it: code for H1 headings of the
// template, the record type's headings tool for an H1 inside the record's
// own content.
func TestCrawlerTracer(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, map[string][]crawlRecord{
		crawlPostType: {{id: "1", path: "/c/two-h1", title: "Post title", headings: []pact.ContentHeading{{Level: 1, Text: "Inside the body"}}}},
	})
	site := newFakeSite()
	site.sitemap("/", "/p/two-h1", "/c/two-h1")
	site.plant(
		doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/c/two-h1"}},
		doc{path: "/p/two-h1", title: "Template page", h1: []string{"Template page", "Another H1"}},
		doc{path: "/c/two-h1", title: "Post title", h1: []string{"Post title", "Inside the body"}},
	)

	env.crawlOnce(t, token, siteCtx(t, site))

	list := env.issues(t, token, "")
	tpl := findIssue(t, list, "h1_multiple", "/p/two-h1")
	if tpl.Severity != "error" || tpl.Fix != "code" || tpl.Tool != "" || tpl.Source != "crawl" || tpl.Details["what"] == nil || tpl.Details["count"] != float64(2) {
		t.Fatalf("template h1_multiple = %+v", tpl)
	}
	content := findIssue(t, list, "h1_multiple", "/c/two-h1")
	if content.Fix != "tool" || content.Tool != "acme_fix_headings" || content.CrawlID == nil || content.ID == 0 {
		t.Fatalf("content h1_multiple = %+v", content)
	}
	if !site.requested("/sitemap-pages-1.xml") || !site.requested("/c/two-h1") {
		t.Fatalf("crawler requests = %v", site.requests)
	}
	for _, ua := range site.agents {
		if ua != "SummerCMS-SEO-Crawler/1 (+https://www.example.test)" {
			t.Fatalf("User-Agent = %q", ua)
		}
	}
}

// checkedKinds are the crawl kinds TestCrawlerChecks compares.
var checkedKinds = map[string]bool{
	"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
	"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
	"duplicate_title": true, "missing_description": true, "duplicate_description": true,
	"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
	"h1_title_mismatch": true,
}

// onlyChecked keeps the "kind url" entries of checkedKinds.
func onlyChecked(list []string) []string {
	out := []string{}
	for _, e := range list {
		kind, _, _ := strings.Cut(e, " ")
		if checkedKinds[kind] {
			out = append(out, e)
		}
	}
	return out
}

// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
// C23 and the clean control page /clean): every planted issue is found
// exactly once with its severity and details, nothing is found on /clean,
// and the crawler never requests another host, a private address or plain
// http (T-33-43).
func TestCrawlerChecks(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	setCrawlRecords(t, auditRecords())
	site := newFakeSite()
	want := plantAudit(site)

	ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
	env.crawlOnce(t, token, ctx)
	list := env.issues(t, token, "source=crawl")

	got := onlyChecked(issueSet(list, "crawl"))
	if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
		t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
	}
	for _, i := range list {
		if i.URL == "/clean" {
			t.Fatalf("issue on the control page: %+v", i)
		}
		if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
			t.Fatalf("issue without a fix route: %+v", i)
		}
	}
	for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
		if site.requested(part) {
			t.Fatalf("the crawler requested %s: %v", part, site.requests)
		}
	}

	detail := func(kind, url, key string, value any) {
		t.Helper()
		i := findIssue(t, list, kind, url)
		if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
			t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
		}
	}
	severity := func(kind, url, want string) {
		t.Helper()
		if i := findIssue(t, list, kind, url); i.Severity != want {
			t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
		}
	}
	detail("page_status", "/p/status-500", "status", 500)
	severity("page_status", "/p/status-500", "error")
	detail("sitemap_error", "/p/sitemap-404", "status", 404)
	detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
	detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
	detail("sitemap_blocked", "/p/blocked", "line", 2)
	detail("sitemap_missing", "/p/unlisted", "count", 1)
	severity("sitemap_missing", "/p/unlisted", "notice")
	detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
	detail("canonical_error", "/p/bad-canonical", "status", 404)
	detail("duplicate_title", "/p/dup-title-a", "count", 1)
	detail("h1_multiple", "/p/two-h1", "count", 2)
	detail("heading_skipped", "/p/skip", "from", 1)
	detail("heading_skipped", "/p/skip", "to", 3)
	detail("heading_empty", "/p/empty-heading", "level", 2)
	detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
	if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
		t.Fatalf("canonical_missing = %+v", i)
	}
	if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
		t.Fatalf("missing_title = %+v", i)
	}
}