Files
sm-seo-plugin/crawler_test.go
Jakub Zych e2af5b44c8 fix: one decoded path form for crawl keys and CMS lookups
Crawl keys, the rendered comparison and the link graph use the decoded
path (classes.PathKey), the form the resolver, the descriptors and the
SEO entries use; the request URL is percent-encoded again
(classes.EscapeKeyPath). A non-ASCII address now matches its CMS record
and its stored links. A path that does not decode to text keeps its
encoded form.
2026-11-04 22:59:17 +01:00

444 lines
18 KiB
Go

package seo_test
import (
"fmt"
"strings"
"testing"
"git.golem15.com/golem15/sm-seo-plugin/classes/crawl"
"git.golem15.com/golem15/summercms/modules/pact"
)
// TestCrawlerTracer is the end-to-end path (D-14): an administrator
// starts a crawl through the API, the crawl job fetches the site with the
// crawler's User-Agent, and GET /issues lists the page with two H1
// headings as an error with how to fix it: code for H1 headings of the
// template, the record type's headings tool for an H1 inside the record's
// own content.
func TestCrawlerTracer(t *testing.T) {
env := bootCrawlEnv(t)
token := env.admin(t, true)
setCrawlRecords(t, map[string][]crawlRecord{
crawlPostType: {{id: "1", path: "/c/two-h1", title: "Post title", headings: []pact.ContentHeading{{Level: 1, Text: "Inside the body"}}}},
})
site := newFakeSite()
site.sitemap("/", "/p/two-h1", "/c/two-h1")
site.plant(
doc{path: "/", title: "Home", links: []string{"/p/two-h1", "/c/two-h1"}},
doc{path: "/p/two-h1", title: "Template page", h1: []string{"Template page", "Another H1"}},
doc{path: "/c/two-h1", title: "Post title", h1: []string{"Post title", "Inside the body"}},
)
env.crawlOnce(t, token, siteCtx(t, site))
list := env.issues(t, token, "")
tpl := findIssue(t, list, "h1_multiple", "/p/two-h1")
if tpl.Severity != "error" || tpl.Fix != "code" || tpl.Tool != "" || tpl.Source != "crawl" || tpl.Details["what"] == nil || tpl.Details["count"] != float64(2) {
t.Fatalf("template h1_multiple = %+v", tpl)
}
content := findIssue(t, list, "h1_multiple", "/c/two-h1")
if content.Fix != "tool" || content.Tool != "acme_fix_headings" || content.CrawlID == nil || content.ID == 0 {
t.Fatalf("content h1_multiple = %+v", content)
}
if !site.requested("/sitemap-pages-1.xml") || !site.requested("/c/two-h1") {
t.Fatalf("crawler requests = %v", site.requests)
}
for _, ua := range site.agents {
if ua != "SummerCMS-SEO-Crawler/1 (+https://www.example.test)" {
t.Fatalf("User-Agent = %q", ua)
}
}
}
// checkedKinds are the crawl kinds TestCrawlerChecks compares.
var checkedKinds = map[string]bool{
"page_status": true, "sitemap_error": true, "sitemap_noindex": true, "sitemap_blocked": true,
"sitemap_missing": true, "canonical_missing": true, "canonical_error": true, "missing_title": true,
"duplicate_title": true, "missing_description": true, "duplicate_description": true,
"h1_missing": true, "h1_multiple": true, "heading_skipped": true, "heading_empty": true,
"h1_title_mismatch": true, "jsonld_invalid": true, "jsonld_multiple": true, "jsonld_required": true,
"jsonld_recommended": true, "hreflang_reciprocal": true, "hreflang_error": true, "generic_anchor": true,
"broken_link": true, "redirected_link": true, "image_alt": true, "image_size": true, "featured_lazy": true,
"content_js": true, "orphan_page": true,
}
// onlyChecked keeps the "kind url" entries of checkedKinds.
func onlyChecked(list []string) []string {
out := []string{}
for _, e := range list {
kind, _, _ := strings.Cut(e, " ")
if checkedKinds[kind] {
out = append(out, e)
}
}
return out
}
// TestCrawlerChecks crawls the planted site (one page per kind of UI-SPEC
// C23 and the clean control page /clean): every planted issue is found
// exactly once with its severity and details, nothing is found on /clean,
// and the crawler never requests another host, a private address or plain
// http (T-33-43).
func TestCrawlerChecks(t *testing.T) {
env := bootCrawlEnv(t)
token := env.admin(t, true)
setCrawlRecords(t, auditRecords())
site := newFakeSite()
want := plantAudit(site)
ctx := crawl.WithRenderer(siteCtx(t, site), auditRenderer(site))
env.crawlOnce(t, token, ctx)
list := env.issues(t, token, "source=crawl")
got := onlyChecked(issueSet(list, "crawl"))
if strings.Join(got, "\n") != strings.Join(onlyChecked(want), "\n") {
t.Fatalf("crawl issues:\n%s\n\nwant:\n%s", strings.Join(got, "\n"), strings.Join(onlyChecked(want), "\n"))
}
for _, i := range list {
if i.URL == "/clean" {
t.Fatalf("issue on the control page: %+v", i)
}
if i.Fix != "tool" && i.Fix != "code" || (i.Fix == "tool") != (i.Tool != "") {
t.Fatalf("issue without a fix route: %+v", i)
}
}
for _, part := range []string{"evil.example.com", "169.254.169.254", "http://www.example.test"} {
if site.requested(part) {
t.Fatalf("the crawler requested %s: %v", part, site.requests)
}
}
detail := func(kind, url, key string, value any) {
t.Helper()
i := findIssue(t, list, kind, url)
if fmt.Sprint(i.Details[key]) != fmt.Sprint(value) {
t.Fatalf("%s %s details.%s = %v, want %v (%v)", kind, url, key, i.Details[key], value, i.Details)
}
}
severity := func(kind, url, want string) {
t.Helper()
if i := findIssue(t, list, kind, url); i.Severity != want {
t.Fatalf("%s %s severity = %s, want %s", kind, url, i.Severity, want)
}
}
detail("page_status", "/p/status-500", "status", 500)
severity("page_status", "/p/status-500", "error")
detail("sitemap_error", "/p/sitemap-404", "status", 404)
detail("sitemap_error", "/p/sitemap-404", "file", "/sitemap-pages-1.xml")
detail("sitemap_noindex", "/p/noindex", "file", "/sitemap-pages-1.xml")
detail("sitemap_blocked", "/p/blocked", "line", 2)
detail("sitemap_missing", "/p/unlisted", "count", 1)
severity("sitemap_missing", "/p/unlisted", "notice")
detail("canonical_error", "/p/bad-canonical", "url", "/p/canonical-404")
detail("canonical_error", "/p/bad-canonical", "status", 404)
detail("duplicate_title", "/p/dup-title-a", "count", 1)
detail("h1_multiple", "/p/two-h1", "count", 2)
detail("heading_skipped", "/p/skip", "from", 1)
detail("heading_skipped", "/p/skip", "to", 3)
detail("heading_empty", "/p/empty-heading", "level", 2)
detail("h1_title_mismatch", "/p/mismatch", "h1", "Completely different")
if i := findIssue(t, list, "canonical_missing", "/p/no-canonical"); i.Fix != "code" || i.Severity != "warning" {
t.Fatalf("canonical_missing = %+v", i)
}
if i := findIssue(t, list, "missing_title", "/p/no-title"); i.Tool != "seo_set_url_seo" {
t.Fatalf("missing_title = %+v", i)
}
detail("jsonld_multiple", "/p/jsonld-two", "count", 2)
detail("jsonld_required", "/p/jsonld-req", "type", "BreadcrumbList")
detail("jsonld_required", "/p/jsonld-req", "property", "itemListElement")
detail("jsonld_recommended", "/p/jsonld-rec", "property", "logo")
if i := findIssue(t, list, "jsonld_recommended", "/p/jsonld-rec"); i.Tool != "seo_update_settings" || i.Severity != "notice" {
t.Fatalf("jsonld_recommended = %+v", i)
}
if i := findIssue(t, list, "jsonld_invalid", "/p/jsonld-bad"); i.Details["error"] == nil || i.Fix != "code" {
t.Fatalf("jsonld_invalid = %+v", i)
}
detail("hreflang_reciprocal", "/en/x", "locale", "pl")
detail("hreflang_reciprocal", "/en/x", "url", "/pl/x")
detail("hreflang_error", "/p/hreflang-err", "url", "/de/missing")
detail("hreflang_error", "/p/hreflang-err", "status", 404)
detail("generic_anchor", "/p/generic", "count", 2)
detail("generic_anchor", "/p/generic", "text", "kliknij tutaj")
detail("generic_anchor", "/p/generic", "target", "/clean")
detail("broken_link", "/p/gone", "from", []string{"/"})
if i := findIssue(t, list, "broken_link", "/p/gone"); i.Tool != "seo_create_redirect" || i.Severity != "error" {
t.Fatalf("broken_link = %+v", i)
}
detail("redirected_link", "/p/moved", "target", "/clean")
detail("image_alt", "/p/img-noalt", "count", 1)
detail("image_size", "/p/img-nosize", "count", 1)
detail("featured_lazy", "/p/lazy", "image", siteURL+"/hero.jpg")
content := func(url, check, what string) {
t.Helper()
i := findIssue(t, list, "content_js", url)
if i.Details["check"] != check || !strings.Contains(fmt.Sprint(i.Details["what"]), what) || i.Fix != "code" {
t.Fatalf("content_js %s = %+v, want check %s naming %q", url, i, check, what)
}
}
detail("orphan_page", "/p/orphan", "count", 1)
if i := findIssue(t, list, "orphan_page", "/p/orphan"); !strings.Contains(fmt.Sprint(i.Details["suggestions"]), "/p/mentions") {
t.Fatalf("orphan suggestions = %v", i.Details)
}
content("/c/js-h1", "cms", "Record heading")
content("/c/js-words", "cms", "of 400 words")
content("/c/js-links", "cms", "/c/parent")
content("/p/render", "rendered", "/p/js-only")
content("/p/render", "rendered", "words")
content("/p/render-h1", "rendered", "H1")
var crawlRow struct {
RenderStatus string
RenderPages int
}
if err := env.db.Raw(`SELECT render_status, render_pages FROM golem15_seo_crawls ORDER BY id DESC LIMIT 1`).Scan(&crawlRow).Error; err != nil {
t.Fatal(err)
}
if crawlRow.RenderStatus != "done" || crawlRow.RenderPages < 10 {
t.Fatalf("render state = %+v", crawlRow)
}
}
// TestRenderSkipped checks the comparison's absence and failure (D-70,
// P33-55): without a renderer (no golem15.seo.crawl_chrome_path) the crawl
// runs every raw check and records render_status skipped with no rendered
// finding; a renderer whose Render fails records render_status failed with
// its note, and the crawl still finishes with the raw checks.
func TestRenderSkipped(t *testing.T) {
env := bootCrawlEnv(t)
token := env.admin(t, true)
setCrawlRecords(t, auditRecords())
site := newFakeSite()
plantAudit(site)
renderState := func(id uint) (status string, pages int, note, crawlStatus string) {
t.Helper()
var row struct {
RenderStatus string
RenderPages int
RenderNote string
Status string
}
if err := env.db.Raw(`SELECT render_status, render_pages, render_note, status FROM golem15_seo_crawls WHERE id = ?`, id).Scan(&row).Error; err != nil {
t.Fatal(err)
}
return row.RenderStatus, row.RenderPages, row.RenderNote, row.Status
}
id := env.crawlOnce(t, token, siteCtx(t, site))
status, pages, _, crawlStatus := renderState(id)
if status != "skipped" || pages != 0 || crawlStatus != "finished" {
t.Fatalf("without a renderer: render %s, %d pages, crawl %s", status, pages, crawlStatus)
}
list := env.issues(t, token, "")
for _, i := range list {
if i.Kind == "content_js" && i.Details["check"] == "rendered" {
t.Fatalf("rendered finding without a renderer: %+v", i)
}
}
findIssue(t, list, "content_js", "/c/js-h1")
findIssue(t, list, "h1_multiple", "/p/two-h1")
failing := &fakeRenderer{site: site, fail: fmt.Errorf("chrome exited")}
id = env.crawlOnce(t, token, crawl.WithRenderer(siteCtx(t, site), failing))
status, _, note, crawlStatus := renderState(id)
if status != "failed" || !strings.Contains(note, "chrome exited") || crawlStatus != "finished" {
t.Fatalf("failing renderer: render %s (%q), crawl %s", status, note, crawlStatus)
}
findIssue(t, env.issues(t, token, ""), "h1_multiple", "/p/two-h1")
}
// TestIssueMerge checks the CMS-data issues stored with the crawl and the
// fix routes (SC-3, P33-38): before any crawl the list holds the CMS
// issues; after it they are stored with the crawl as source cms; image_alt
// names the record type's image_alt tool; h1_in_content appears for a post
// and a category whose descriptor sets TopHeading 2 and not for TopHeading
// 0, naming the headings tool; a skipped level in a category's pillar text
// (a type without an edit tool) is fix code; no issue lacks a route. An
// ignored fingerprint stays ignored in the next crawl, and a fixed page's
// issue is gone from it.
func TestIssueMerge(t *testing.T) {
env := bootCrawlEnv(t)
token := env.admin(t, true)
setCrawlRecords(t, map[string][]crawlRecord{
crawlPostType: {
{id: "1", path: "/c/post-a", title: "Post A", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1"}, {Level: 2, Text: "Part"}},
images: []pact.ContentImage{{URL: "/img/a.jpg"}, {URL: "/img/deco.png", Decorative: true}}},
{id: "2", path: "/c/post-b", title: "Post B", topHeading: 0, headings: []pact.ContentHeading{{Level: 1, Text: "Body H1 of B"}}},
},
crawlCategoryType: {
{id: "10", path: "/cat/news", title: "News", topHeading: 2, headings: []pact.ContentHeading{{Level: 1, Text: "Pillar H1"}, {Level: 2, Text: "Intro"}, {Level: 4, Text: "Deep"}}},
},
})
site := newFakeSite()
site.sitemap("/", "/c/post-a", "/c/post-b", "/cat/news", "/p/fixme")
site.plant(
doc{path: "/", title: "Home", links: []string{"/c/post-a", "/c/post-b", "/cat/news", "/p/fixme"}},
doc{path: "/c/post-a", title: "Post A", body: `<h2>Part</h2><img src="/img/a.jpg" alt="Photo" width="1" height="1">`},
doc{path: "/c/post-b", title: "Post B"},
doc{path: "/cat/news", title: "News", body: "<h2>Intro</h2><h4>Deep</h4>"},
doc{path: "/p/fixme", title: "Fix me", noDesc: true},
)
before := env.issues(t, token, "")
if i := findIssue(t, before, "image_alt", "/c/post-a"); i.Source != "cms" || i.CrawlID != nil || i.Tool != "acme_set_alt" {
t.Fatalf("CMS image_alt before any crawl = %+v", i)
}
ctx := siteCtx(t, site)
first := env.crawlOnce(t, token, ctx)
list := env.issues(t, token, "")
alt := findIssue(t, list, "image_alt", "/c/post-a")
if alt.Source != "cms" || alt.CrawlID == nil || *alt.CrawlID != first || alt.Tool != "acme_set_alt" || alt.Fix != "tool" {
t.Fatalf("image_alt = %+v", alt)
}
if i := findIssue(t, list, "h1_in_content", "/c/post-a"); i.Tool != "acme_fix_headings" || i.Source != "cms" || i.Severity != "warning" {
t.Fatalf("h1_in_content post = %+v", i)
}
if i := findIssue(t, list, "h1_in_content", "/cat/news"); i.Tool != "acme_category_headings" {
t.Fatalf("h1_in_content category = %+v", i)
}
for _, i := range list {
if i.Kind == "h1_in_content" && i.URL == "/c/post-b" {
t.Fatalf("h1_in_content for TopHeading 0: %+v", i)
}
if i.Fix == "tool" && i.Tool == "" || i.Fix != "tool" && i.Fix != "code" {
t.Fatalf("issue without a route: %+v", i)
}
if i.Fix == "code" && i.Details["what"] == nil {
t.Fatalf("fix code without what: %+v", i)
}
}
skipped := findIssue(t, list, "heading_skipped", "/cat/news")
if skipped.Fix != "code" || skipped.Tool != "" {
t.Fatalf("heading_skipped in pillar text = %+v", skipped)
}
findIssue(t, list, "missing_description", "/p/fixme")
// Ignore the skipped level, fix the description, crawl again.
var fp string
if err := env.db.Raw(`SELECT fingerprint FROM golem15_seo_issues WHERE id = ?`, skipped.ID).Scan(&fp).Error; err != nil || fp == "" {
t.Fatalf("fingerprint: %q %v", fp, err)
}
if err := env.db.Exec(`INSERT INTO golem15_seo_issue_ignores (fingerprint, ignored_at) VALUES (?, NOW())`, fp).Error; err != nil {
t.Fatal(err)
}
site.plant(doc{path: "/p/fixme", title: "Fix me"})
second := env.crawlOnce(t, token, ctx)
list = env.issues(t, token, "status=all")
again := findIssue(t, list, "heading_skipped", "/cat/news")
if !again.Ignored || again.CrawlID == nil || *again.CrawlID != second {
t.Fatalf("ignored issue in the next crawl = %+v", again)
}
for _, i := range list {
if i.URL == "/p/fixme" {
t.Fatalf("fixed page still has %+v", i)
}
}
}
// TestStagingCrawl checks RESEARCH Pitfall 12: on a non-production
// environment every page carries the staging lock's noindex, and no
// noindex-derived issue is reported; the pages still count as indexable.
func TestStagingCrawl(t *testing.T) {
env := bootCrawlEnv(t, "SUMMER_ENV=staging")
if !env.seoPlugin(t).Service().Staging() {
t.Fatal("the environment is not staging")
}
token := env.admin(t, true)
site := newFakeSite()
site.sitemap("/", "/a", "/b")
for _, d := range []doc{
{path: "/", title: "Home", links: []string{"/a", "/b", "/c"}},
{path: "/a", title: "Same"},
{path: "/b", title: "Same"},
{path: "/c", title: "Unlisted"},
} {
d.robots = "noindex, nofollow"
site.set(d.path, 200, d.render(), "X-Robots-Tag", "noindex")
}
env.crawlOnce(t, token, siteCtx(t, site))
list := env.issues(t, token, "")
for _, i := range list {
if i.Kind == "sitemap_noindex" || i.Kind == "canonical_noindex" || i.Kind == "noindex_blocked" {
t.Fatalf("staging noindex reported: %+v", i)
}
}
findIssue(t, list, "duplicate_title", "/a")
findIssue(t, list, "sitemap_missing", "/c")
}
// TestCrawlerNonASCII pins L3 WR-03 and WR-04 (P33-61) end to end: a
// percent-encoded non-ASCII address is fetched once under its decoded key,
// matches its CMS record (its content finding names the record type's
// tool), its breadcrumb parent link is recognised (no false content_js
// links finding), and the link graph reports its inbound link for the
// encoded absolute URL; a link to the site host on another port is neither
// fetched nor stored as a page.
func TestCrawlerNonASCII(t *testing.T) {
env := bootCrawlEnv(t)
token := env.admin(t, true)
const (
post = "/c/zażółć"
postEnc = "/c/za%C5%BC%C3%B3%C5%82%C4%87"
cat = "/cat/łąka"
catEnc = "/cat/%C5%82%C4%85ka"
otherURL = "https://www.example.test:8443/x"
)
setCrawlRecords(t, map[string][]crawlRecord{
crawlPostType: {{id: "1", path: post, title: "Zażółć", headings: []pact.ContentHeading{{Level: 1, Text: "W treści"}},
parents: []pact.ContentRef{{Type: crawlCategoryType, ID: "2"}}}},
crawlCategoryType: {{id: "2", path: cat, title: "Łąka"}},
})
site := newFakeSite()
site.sitemap("/", postEnc)
site.plant(
doc{path: "/", title: "Home", links: []string{postEnc, catEnc, otherURL}},
doc{path: postEnc, title: "Zażółć", h1: []string{"Zażółć", "W treści"}, links: []string{catEnc}},
doc{path: catEnc, title: "Łąka", links: []string{"/"}},
)
id := env.crawlOnce(t, token, siteCtx(t, site))
if row := env.crawlRow(t, id); row.Status != "finished" {
t.Fatalf("crawl = %+v", row)
}
pages := env.storedPages(t, id)
if pages[post] != 200 || pages[cat] != 200 {
t.Fatalf("stored pages = %v, want %s and %s with 200", pages, post, cat)
}
for url := range pages {
if strings.Contains(url, "%C5") || url == "/x" {
t.Fatalf("stored page %q: an encoded key or the other port's page", url)
}
}
if site.requested(":8443") {
t.Fatal("the crawler fetched the site host on another port")
}
n := 0
for _, u := range site.requests {
if strings.Contains(u, postEnc) {
n++
}
}
if n != 1 {
t.Fatalf("%s was requested %d times, want once: %v", postEnc, n, site.requests)
}
list := env.issues(t, token, "")
content := findIssue(t, list, "h1_multiple", post)
if content.Fix != "tool" || content.Tool != "acme_fix_headings" {
t.Fatalf("the record's h1_multiple = %+v, want the record type's tool", content)
}
for _, i := range list {
if i.Kind == "content_js" && i.URL == post {
t.Fatalf("false content_js on the non-ASCII page: %+v", i)
}
}
report := env.links(t, token, "https://www.example.test"+postEnc)
if report.Inbound == nil || *report.Inbound != 1 {
t.Fatalf("inbound links of the encoded URL = %v, want 1", report.Inbound)
}
if decoded := env.links(t, token, post); decoded.Inbound == nil || *decoded.Inbound != 1 {
t.Fatalf("inbound links of the decoded path = %v, want 1", decoded.Inbound)
}
}