golem15/sm-seo-plugin
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: unit tests for the schema builder, crawler and page helpers

Browse Source 
- schema: types per kind, @id links, every node kind, the publisher and
  local business, every validator rule, custom JSON-LD caps and merge
- crawl: HTML parse, each check's positive and negative case, fix routes
  with the fitting tool, the rendered comparison, allowRequest,
  resolverRules and NewRenderer
- pagination, alternates and x-default, primary parents, the filter's
  addresses, registry paths and fields, auto-link addresses
This commit is contained in:
Jakub Zych
2026-11-04 19:43:31 +01:00
parent dfc85bc805
commit 82b0dacff9
14 changed files with 1543 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
classes/crawl/checks_test.go Normal file
View File

@@ -0,0 +1,205 @@
package crawl
import (
"testing"
"git.golem15.com/golem15/sm-seo-plugin/models"
"git.golem15.com/golem15/summercms/modules/pact"
)
func okPage(key string, facts *PageFacts) *Page {
return &Page{Key: key, Resp: &Response{Status: 200, ContentType: "text/html"}, Facts: facts}
}
func kinds(fs []Finding) map[string]Finding {
out := map[string]Finding{}
for _, f := range fs {
out[f.Kind] = f
}
return out
}
// TestSeverityAndKinds pins the kinds table: every listed kind has a
// severity, an unknown kind is a warning, and each kind is listed once.
func TestSeverityAndKinds(t *testing.T) {
seen := map[string]bool{}
for _, k := range Kinds() {
if seen[k] {
t.Fatalf("%s is listed twice", k)
}
seen[k] = true
if _, ok := severities[k]; !ok {
t.Errorf("%s has no severity", k)
}
}
if len(seen) != len(severities) {
t.Fatalf("%d kinds listed, %d severities", len(seen), len(severities))
}
if Severity("nope") != models.SeverityWarning || Severity(KindPageStatus) != models.SeverityError || Severity(KindSitemapMissing) != models.SeverityNotice {
t.Fatal("Severity is wrong")
}
}
// TestCheckStatus pins page_status: a fetch error (not a cancel) and an
// error status other than 404 and 410; nothing for 200, 301, 404, 410.
func TestCheckStatus(t *testing.T) {
for _, tc := range []struct {
resp *Response
want bool
}{
{&Response{Err: "timeout"}, true}, {&Response{Err: "canceled"}, false}, {&Response{Status: 500}, true}, {&Response{Status: 403}, true},
{&Response{Status: 404}, false}, {&Response{Status: 410}, false}, {&Response{Status: 200}, false}, {&Response{Status: 301}, false}, {nil, false},
} {
got := checkStatus(&Result{}, &Page{Key: "/a", Resp: tc.resp}, nil)
if (len(got) == 1) != tc.want || (tc.want && (got[0].Kind != KindPageStatus || got[0].Tool != ToolCreateRedirect)) {
t.Errorf("%+v: %+v", tc.resp, got)
}
}
}
// TestCheckHead pins missing title, description and canonical (positive)
// and their absence on a complete head (negative), and the H1/title
// mismatch only when neither contains the other.
func TestCheckHead(t *testing.T) {
bad := kinds(checkHead(&Result{}, okPage("/a", &PageFacts{Headings: []Heading{{1, "Kanię"}}, Title: "Grzyby"}), nil))
for _, k := range []string{KindMissingDesc, KindCanonicalMissing, KindH1TitleMismatch} {
if _, ok := bad[k]; !ok {
t.Errorf("missing %s in %v", k, bad)
}
}
if bad[KindMissingDesc].Tool != ToolSetURLSEO {
t.Fatal("a missing description is fixed with seo_set_url_seo")
}
if _, ok := kinds(checkHead(&Result{}, okPage("/a", &PageFacts{}), nil))[KindMissingTitle]; !ok {
t.Fatal("no missing_title for an empty title")
}
good := checkHead(&Result{}, okPage("/a", &PageFacts{Title: "Grzyby lecznicze | Acme", Description: "d", Canonical: "https://example.test/a", Headings: []Heading{{1, "Grzyby LECZNICZE"}}}), nil)
if len(good) != 0 {
t.Fatalf("a complete head: %+v", good)
}
}
// TestCheckHeadings pins h1_missing, h1_multiple, heading_skipped (first
// skip with every skip counted) and heading_empty, and the fitting tool:
// an H1 in the record's content routes to the headings tool, a skipped
// level the record holds to the edit tool, a template heading to code.
func TestCheckHeadings(t *testing.T) {
rec := &Record{Headings: []pact.ContentHeading{{Level: 1, Text: "In body"}, {Level: 4, Text: "Deep"}}, Tools: map[string]string{pact.ContentToolHeadings: "journal_fix_headings", pact.ContentToolEdit: "journal_update_post"}}
got := kinds(checkHeadings(okPage("/a", &PageFacts{Headings: []Heading{{1, "Title"}, {1, "In body"}, {2, "A"}, {4, "Deep"}, {2, "B"}, {5, "Deeper"}, {3, ""}}}), rec))
multi, skip, empty := got[KindH1Multiple], got[KindHeadingSkipped], got[KindHeadingEmpty]
if multi.Details["count"] != 2 || len(multi.Content) != 2 || multi.Content[0] != pact.ContentToolHeadings {
t.Fatalf("h1_multiple = %+v", multi)
}
if fix, tool, _ := Route(multi); fix != models.FixTool || tool != "journal_fix_headings" {
t.Fatalf("h1_multiple routes to %s %s, want the headings tool", fix, tool)
}
if skip.Details["from"] != 2 || skip.Details["to"] != 4 || skip.Details["count"] != 2 || len(skip.Content) != 1 || skip.Content[0] != pact.ContentToolEdit {
t.Fatalf("heading_skipped = %+v", skip)
}
if fix, tool, _ := Route(skip); fix != models.FixTool || tool != "journal_update_post" {
t.Fatalf("heading_skipped routes to %s %s, want the edit tool", fix, tool)
}
if empty.Details["level"] != 3 || empty.Details["count"] != 1 || len(empty.Content) != 0 {
t.Fatalf("heading_empty = %+v", empty)
}
if fix, _, what := Route(empty); fix != models.FixCode || what == "" {
t.Fatalf("a template heading routes to code, got %s %q", fix, what)
}
// A category's skipped level the record does not hold is fixed in code.
cat := &Record{Tools: map[string]string{pact.ContentToolEdit: "journal_update_category"}}
skipCat := kinds(checkHeadings(okPage("/c", &PageFacts{Headings: []Heading{{1, "Cat"}, {3, "Post card"}}}), cat))[KindHeadingSkipped]
if fix, _, what := Route(skipCat); fix != models.FixCode || what != defaultWhat[KindHeadingSkipped] {
t.Fatalf("category skip routes to %s %q, want code", fix, what)
}
if _, ok := kinds(checkHeadings(okPage("/b", &PageFacts{}), nil))[KindH1Missing]; !ok {
t.Fatal("no h1_missing")
}
if clean := checkHeadings(okPage("/b", &PageFacts{Headings: []Heading{{1, "T"}, {2, "a"}, {3, "b"}, {2, "c"}}}), nil); len(clean) != 0 {
t.Fatalf("clean headings: %+v", clean)
}
}
// TestCheckJSONLD pins jsonld_invalid (the custom tool when the record
// stores custom JSON-LD), jsonld_multiple, jsonld_required and
// jsonld_recommended (the settings tool for a publisher node), and nothing
// for a valid single graph.
func TestCheckJSONLD(t *testing.T) {
got := kinds(checkJSONLD(okPage("/a", &PageFacts{JSONLD: []string{`{"@type":`, `{"@graph":[{"@type":"BlogPosting","headline":"h"},{"@type":"Organization"}]}`}}), &Record{CustomJSONLD: true}))
if got[KindJSONLDInvalid].Tool != ToolSetURLSEO || got[KindJSONLDMultiple].Details["count"] != 2 {
t.Fatalf("invalid/multiple = %+v", got)
}
if rec := got[KindJSONLDRecommended]; rec.Details["type"] != "BlogPosting" || rec.Tool != ToolSetURLSEO {
t.Fatalf("recommended = %+v", rec)
}
pub := kinds(checkJSONLD(okPage("/a", &PageFacts{JSONLD: []string{`[{"@type":"Person"}]`}}), nil))[KindJSONLDRequired]
if pub.Details["property"] != "name" || pub.Tool != ToolUpdateSettings {
t.Fatalf("publisher required = %+v", pub)
}
if none := checkJSONLD(okPage("/a", &PageFacts{JSONLD: []string{`{"@type":"WebSite","name":"n","url":"u"}`}}), nil); len(none) != 0 {
t.Fatalf("a valid graph: %+v", none)
}
if none := checkJSONLD(okPage("/a", &PageFacts{}), nil); none != nil {
t.Fatal("no scripts, no findings")
}
}
// TestCheckImages pins image_alt (an empty alt is decorative, not a
// finding; a record image routes to the image_alt tool), image_size and
// featured_lazy, and the list cap.
func TestCheckImages(t *testing.T) {
facts := &PageFacts{OGImage: "https://example.test/m/cover.jpg", Images: []Image{
{Src: "https://example.test/m/cover.jpg", Alt: "c", HasAlt: true, Width: "10", Height: "10", Loading: "lazy"},
{Src: "https://example.test/m/a.jpg", HasAlt: false, Width: "1", Height: "1"},
{Src: "https://example.test/m/deco.png", Alt: "", HasAlt: true},
}}
rec := &Record{Images: []pact.ContentImage{{URL: "/m/a.jpg"}}, Tools: map[string]string{pact.ContentToolImageAlt: "journal_set_image_alt"}}
got := kinds(checkImages(okPage("/a", facts), rec))
if alt := got[KindImageAlt]; alt.Details["count"] != 1 || len(alt.Content) != 2 {
t.Fatalf("image_alt = %+v", alt)
}
if _, tool, _ := Route(got[KindImageAlt]); tool != "journal_set_image_alt" {
t.Fatalf("image_alt routes to %q", tool)
}
if got[KindImageSize].Details["count"] != 1 || got[KindFeaturedLazy].Details["image"] != "https://example.test/m/cover.jpg" {
t.Fatalf("size/lazy = %+v", got)
}
many := &PageFacts{}
for i := 0; i < 15; i++ {
many.Images = append(many.Images, Image{Src: "/x.jpg", Width: "1", Height: "1"})
}
if alt := kinds(checkImages(okPage("/b", many), nil))[KindImageAlt]; alt.Details["count"] != 15 || len(alt.Details["images"].([]string)) != maxListed || len(alt.Content) != 0 {
t.Fatalf("capped image_alt = %+v", alt)
}
}
// TestCheckContentCMS pins content_js (check cms): the record title
// missing from the H1, under half the record's words; nothing when both
// are met or without a record.
func TestCheckContentCMS(t *testing.T) {
words := 100
rec := &Record{Title: "Chaga", Words: &words}
got := checkContentCMS(&Result{}, okPage("/a", &PageFacts{Headings: []Heading{{1, "Other"}}, Words: 40}), rec)
if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkCMS || got[0].Details["words_cms"] != 100 || got[0].Details["title"] != "Chaga" {
t.Fatalf("content_js = %+v", got)
}
if ok := checkContentCMS(&Result{}, okPage("/a", &PageFacts{Headings: []Heading{{1, "Chaga — grzyb"}}, Words: 50}), rec); ok != nil {
t.Fatalf("met: %+v", ok)
}
if none := checkContentCMS(&Result{}, okPage("/a", &PageFacts{}), nil); none != nil {
t.Fatal("no record, no finding")
}
}
// TestEnvNoindex pins the staging lock: on a non-production environment
// no page counts as noindex; otherwise meta robots and X-Robots-Tag do.
func TestEnvNoindex(t *testing.T) {
meta := okPage("/a", &PageFacts{Robots: "noindex"})
header := &Page{Key: "/b", Resp: &Response{Status: 200, XRobots: "noindex, nofollow"}, Facts: &PageFacts{}}
if !(&Env{}).noindex(meta) || !(&Env{}).noindex(header) || (&Env{Staging: true}).noindex(meta) || (&Env{}).noindex(okPage("/c", &PageFacts{})) {
t.Fatal("noindex is wrong")
}
var nilEnv *Env
if nilEnv.record("x") != nil || locale(nil) != "" || locale(&Record{Locale: "pl"}) != "pl" {
t.Fatal("nil env and record helpers")
}
}

74
classes/crawl/compare_test.go Normal file
View File

@@ -0,0 +1,74 @@
package crawl
import (
"net/url"
"strings"
"testing"
)
func linkFacts(words int, h1 string, hrefs ...string) *PageFacts {
f := &PageFacts{Words: words}
if h1 != "" {
f.Headings = []Heading{{Level: 1, Text: h1}}
}
for _, h := range hrefs {
u, _ := url.Parse(h)
f.Links = append(f.Links, Link{URL: u, Href: h})
}
return f
}
var siteHosts = []string{"example.test", "www.example.test"}
// TestCompareRendered pins the rendered comparison (P33-55): a same-host
// link (the www twin too), an H1 and growth of at least 50 words and 20 %
// present only after JavaScript are one content_js finding with check
// rendered; other-host links, smaller growth and an identical rendering
// find nothing.
func TestCompareRendered(t *testing.T) {
raw := linkFacts(100, "", "https://example.test/a")
rendered := linkFacts(160, "Grzyby", "https://example.test/a", "https://www.example.test/b?x=1", "https://other.test/c", "https://example.test/b?x=1")
got := compareRendered(raw, rendered, siteHosts)
if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkRendered {
t.Fatalf("findings = %+v", got)
}
d := got[0].Details
if links := d["links"].([]string); len(links) != 1 || links[0] != "/b?x=1" || d["h1"] != "Grzyby" || d["words_raw"] != 100 || d["words_rendered"] != 160 {
t.Fatalf("details = %v", d)
}
if !strings.Contains(got[0].What, "links /b?x=1") || !strings.Contains(got[0].What, "H1 “Grzyby”") || !strings.Contains(got[0].What, "60 words of text") {
t.Fatalf("what = %q", got[0].What)
}
for name, pair := range map[string][2]*PageFacts{
"identical": {linkFacts(100, "T", "https://example.test/a"), linkFacts(100, "T", "https://example.test/a")},
"other host only": {linkFacts(100, "T"), linkFacts(100, "T", "https://other.test/x")},
"49 words": {linkFacts(100, "T"), linkFacts(149, "T")},
"50 words under 20 %": {linkFacts(300, "T"), linkFacts(350, "T")},
"H1 already there": {linkFacts(10, "Raw"), linkFacts(10, "Rendered")},
} {
if got := compareRendered(pair[0], pair[1], siteHosts); got != nil {
t.Errorf("%s: %+v", name, got)
}
}
if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteHosts); len(got) != 1 {
t.Fatalf("50 words from an empty page: %+v", got)
}
if compareRendered(nil, rendered, siteHosts) != nil || compareRendered(raw, nil, siteHosts) != nil {
t.Fatal("a missing side compares to nothing")
}
}
// TestCompareRenderedCap pins the cap of the links a finding names, and
// that a path without a slash counts as /.
func TestCompareRenderedCap(t *testing.T) {
var hrefs []string
for _, p := range []string{"", "/1", "/2", "/3", "/4", "/5", "/6"} {
hrefs = append(hrefs, "https://example.test"+p)
}
got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteHosts)
links := got[0].Details["links"].([]string)
if len(links) != maxJSOnlyURLs || links[0] != "/" {
t.Fatalf("links = %v", links)
}
}

80
classes/crawl/fix_test.go Normal file
View File

@@ -0,0 +1,80 @@
package crawl
import (
"testing"
"git.golem15.com/golem15/sm-seo-plugin/models"
"git.golem15.com/golem15/summercms/modules/pact"
)
// TestRoute pins P33-38: a content finding names the record type's first
// fitting tool in the order given; without one a fixed CMS tool; else
// code with the finding's own what, the kind's default, or the page
// template.
func TestRoute(t *testing.T) {
both := &Record{Tools: map[string]string{pact.ContentToolHeadings: "acme_fix_headings", pact.ContentToolEdit: "acme_update"}}
editOnly := &Record{Tools: map[string]string{pact.ContentToolEdit: " acme_update "}}
for _, tc := range []struct {
name string
f Finding
fix, tool, what string
}{
{"headings first", Finding{Kind: KindH1Multiple, Content: []string{pact.ContentToolHeadings, pact.ContentToolEdit}, Record: both}, models.FixTool, "acme_fix_headings", ""},
{"edit fallback", Finding{Kind: KindH1Multiple, Content: []string{pact.ContentToolHeadings, pact.ContentToolEdit}, Record: editOnly}, models.FixTool, "acme_update", ""},
{"no record tool, fixed tool", Finding{Kind: KindJSONLDInvalid, Content: []string{pact.ContentToolEdit}, Record: &Record{}, Tool: ToolSetURLSEO}, models.FixTool, ToolSetURLSEO, ""},
{"nil record", Finding{Kind: KindImageAlt, Content: []string{pact.ContentToolImageAlt}}, models.FixCode, "", defaultWhat[KindImageAlt]},
{"fixed tool", Finding{Kind: KindMissingTitle, Tool: ToolSetURLSEO}, models.FixTool, ToolSetURLSEO, ""},
{"own what", Finding{Kind: KindContentJS, What: "links /a"}, models.FixCode, "", "links /a"},
{"default what", Finding{Kind: KindCanonicalMissing}, models.FixCode, "", defaultWhat[KindCanonicalMissing]},
{"page template", Finding{Kind: "unknown"}, models.FixCode, "", "the page template"},
} {
fix, tool, what := Route(tc.f)
if fix != tc.fix || tool != tc.tool || what != tc.what {
t.Errorf("%s: %s %q %q, want %s %q %q", tc.name, fix, tool, what, tc.fix, tc.tool, tc.what)
}
}
}
// TestRouteEveryKind pins that every kind gets a tool or a code fix.
func TestRouteEveryKind(t *testing.T) {
for _, k := range Kinds() {
fix, tool, what := Route(Finding{Kind: k})
if (fix == models.FixTool && tool == "") || (fix == models.FixCode && what == "") || (fix != models.FixTool && fix != models.FixCode) {
t.Errorf("%s: %s %q %q", k, fix, tool, what)
}
}
}
// TestFingerprint pins the identity of a finding across crawls: stable,
// 32 hex characters, and different for each of its four parts.
func TestFingerprint(t *testing.T) {
a := Fingerprint(KindContentJS, "/a", "pl", "cms")
if a != Fingerprint(KindContentJS, "/a", "pl", "cms") || len(a) != 32 {
t.Fatalf("fingerprint %q", a)
}
for _, other := range []string{
Fingerprint(KindH1Missing, "/a", "pl", "cms"), Fingerprint(KindContentJS, "/b", "pl", "cms"),
Fingerprint(KindContentJS, "/a", "en", "cms"), Fingerprint(KindContentJS, "/a", "pl", "rendered"),
Fingerprint(KindContentJS, "/a\x00pl", "", "cms"),
} {
if other == a {
t.Fatal("a different finding has the same fingerprint")
}
}
}
// TestRecordHeadingHelpers pins hasHeading (any level or text, compared
// without case and extra space) and hasEmptyHeading on a nil record too.
func TestRecordHeadingHelpers(t *testing.T) {
rec := &Record{Headings: []pact.ContentHeading{{Level: 2, Text: "Lwia Grzywa"}, {Level: 3, Text: " "}}}
if !rec.hasHeading(2, "lwia grzywa") || !rec.hasHeading(0, "") || rec.hasHeading(1, "") || rec.hasHeading(2, "Chaga") {
t.Fatal("hasHeading is wrong")
}
if !hasEmptyHeading(rec, 3) || hasEmptyHeading(rec, 2) || hasEmptyHeading(nil, 1) {
t.Fatal("hasEmptyHeading is wrong")
}
var none *Record
if none.hasHeading(0, "") || none.tool(pact.ContentToolEdit) != "" {
t.Fatal("a nil record has no headings and no tools")
}
}

128
classes/crawl/parse_test.go Normal file
View File

@@ -0,0 +1,128 @@
package crawl
import (
"net/url"
"strings"
"testing"
)
const fixtureHTML = `<!doctype html>
<html lang="pl-PL"><head>
<title> Grzyby lecznicze </title>
<title>Second</title>
<meta name="description" content=" O grzybach ">
<meta name="robots" content="index, follow">
<meta name="googlebot" content="noimageindex">
<meta property="og:image" content="/media/cover.jpg">
<link rel="canonical" href="/blog/grzyby#top">
<link rel="canonical" href="/other">
<link rel="alternate" hreflang="en" href="/en/blog/mushrooms">
<link rel="alternate" hreflang="x-default" href="https://example.test/blog/grzyby">
<link rel="alternate" type="application/rss+xml" href="/feed.xml">
<script type="application/ld+json">{"@type":"WebPage"}</script>
<script>var x = "<h1>not a heading</h1>";</script>
</head><body>
<header><a href="/">Start</a></header>
<main>
<h1>Grzyby <img src="/i/logo.png" alt="lecznicze"></h1>
<h2></h2>
<h4>Detail</h4>
<p>Raz dwa trzy.</p>
<a href="/blog/chaga?x=1#frag" rel="NoFollow">Chaga</a>
<a href="javascript:void(0)">js</a><a href="mailto:a@b.c">mail</a><a href="tel:1">tel</a><a href="#top">top</a>
<a href="https://other.test/x">Other</a>
<img src="/media/a.jpg" alt="" width="10" height="5" loading="LAZY">
<img src="/media/b.jpg">
<noscript>hidden text</noscript>
<style>.x{}</style>
</main>
<footer>Stopka</footer>
</body></html>`
// TestParseFacts pins Parse on an HTML fixture: language, the first title
// collapsed, description, robots joined from robots and googlebot,
// og:image and the first canonical resolved without the fragment,
// hreflang alternates only, JSON-LD scripts only, headings in order (an
// image alt counts as heading text, script content never), links without
// javascript:, mailto:, tel: and fragment-only ones, images with and
// without alt, and the main text without noscript and style.
func TestParseFacts(t *testing.T) {
base, _ := url.Parse("https://example.test/blog/grzyby")
f := Parse(base, []byte(fixtureHTML))
if f.Lang != "pl-PL" || !f.HasTitle || f.Title != "Grzyby lecznicze" || !f.HasDescription || f.Description != "O grzybach" {
t.Fatalf("head = %+v", f)
}
if f.Robots != "index, follow, noimageindex" || f.Noindex() {
t.Fatalf("robots = %q", f.Robots)
}
if f.OGImage != "https://example.test/media/cover.jpg" || f.Canonical != "https://example.test/blog/grzyby" {
t.Fatalf("og:image %q canonical %q", f.OGImage, f.Canonical)
}
if len(f.Alternates) != 2 || f.Alternates[0].Lang != "en" || f.Alternates[0].URL.String() != "https://example.test/en/blog/mushrooms" || f.Alternates[1].Lang != "x-default" {
t.Fatalf("alternates = %+v", f.Alternates)
}
if len(f.JSONLD) != 1 || f.JSONLD[0] != `{"@type":"WebPage"}` {
t.Fatalf("json-ld = %q", f.JSONLD)
}
if len(f.Headings) != 3 || f.Headings[0] != (Heading{1, "Grzyby lecznicze"}) || f.Headings[1] != (Heading{2, ""}) || f.Headings[2] != (Heading{4, "Detail"}) {
t.Fatalf("headings = %+v", f.Headings)
}
if h1 := f.H1(); len(h1) != 1 || h1[0] != "Grzyby lecznicze" {
t.Fatalf("H1 = %v", h1)
}
var links []string
for _, l := range f.Links {
links = append(links, l.URL.String())
}
if strings.Join(links, " ") != "https://example.test/ https://example.test/blog/chaga?x=1 https://other.test/x" || f.Links[1].Rel != "nofollow" || f.Links[1].Text != "Chaga" {
t.Fatalf("links = %v (%+v)", links, f.Links)
}
if len(f.Images) != 3 || !f.Images[1].HasAlt || f.Images[1].Alt != "" || f.Images[1].Loading != "lazy" || f.Images[2].HasAlt || f.Images[2].Src != "https://example.test/media/b.jpg" {
t.Fatalf("images = %+v", f.Images)
}
if strings.Contains(f.Text, "hidden text") || strings.Contains(f.Text, ".x{}") || strings.Contains(f.Text, "Stopka") || !strings.Contains(f.Text, "Raz dwa trzy.") || f.Words != len(strings.Fields(f.Text)) {
t.Fatalf("text = %q (%d words)", f.Text, f.Words)
}
}
// TestParseBodyText pins the text root: main when there is one, else the
// body; and an empty document parses to empty facts.
func TestParseBodyText(t *testing.T) {
f := Parse(nil, []byte(`<html><body><nav>Menu</nav><p>One two</p></body></html>`))
if f.Text != "Menu One two" || f.Words != 3 || f.HasTitle {
t.Fatalf("body text = %q", f.Text)
}
if empty := Parse(nil, nil); empty.Words != 0 || len(empty.Links) != 0 {
t.Fatalf("empty = %+v", empty)
}
}
// TestNoindexValues pins the robots values that mean noindex.
func TestNoindexValues(t *testing.T) {
for v, want := range map[string]bool{"noindex": true, "NOINDEX, follow": true, "none": true, "index;noindex": true, "index, follow": false, "": false, "noindexing": false} {
if got := hasNoindex(v); got != want {
t.Errorf("hasNoindex(%q) = %v, want %v", v, got, want)
}
}
}
// TestParseCaps pins the link cap and the text cap with a UTF-8 safe cut.
func TestParseCaps(t *testing.T) {
var b strings.Builder
b.WriteString("<main>")
for i := 0; i < MaxLinksPerPage+10; i++ {
b.WriteString(`<a href="/x">x</a>`)
}
b.WriteString("</main>")
if got := len(Parse(nil, []byte(b.String())).Links); got != MaxLinksPerPage {
t.Fatalf("%d links, want the %d cap", got, MaxLinksPerPage)
}
long := "<main>" + strings.Repeat("żółw ", maxText/4) + "</main>"
f := Parse(nil, []byte(long))
if len(f.Text) > maxText || !strings.HasPrefix(f.Text, "żółw") || strings.ContainsRune(f.Text, '<27>') {
t.Fatalf("text of %d bytes", len(f.Text))
}
if truncateUTF8("żółw", 2) != "ż" || truncateUTF8("ab", 5) != "ab" {
t.Fatal("truncateUTF8 is wrong")
}
}

126
classes/crawl/render_test.go Normal file
View File

@@ -0,0 +1,126 @@
package crawl
import (
"context"
"net/netip"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"git.golem15.com/golem15/summercms/modules/compass"
)
// TestAllowRequest pins what the headless browser may load: https on the
// site host and its www twin, data: and blob:; http only in trusted mode;
// other hosts, credentials in the URL and every other scheme refused.
func TestAllowRequest(t *testing.T) {
hosts := []string{"example.test", "www.example.test"}
for _, tc := range []struct {
url string
trusted bool
want bool
}{
{"https://example.test/a.js", false, true},
{"https://WWW.example.test/b.css", false, true},
{"https://example.test:8443/x", false, true},
{"data:image/png;base64,AAAA", false, true},
{"blob:https://example.test/uuid", false, true},
{"http://example.test/a.js", false, false},
{"http://example.test/a.js", true, true},
{"https://cdn.other.test/lib.js", false, false},
{"https://example.test.evil.test/x", false, false},
{"https://user:pw@example.test/x", false, false},
{"ws://example.test/socket", false, false},
{"file:///etc/passwd", false, false},
{"javascript:alert(1)", true, false},
{"://bad", false, false},
} {
if got := allowRequest(tc.url, hosts, tc.trusted); got != tc.want {
t.Errorf("allowRequest(%q, trusted=%v) = %v, want %v", tc.url, tc.trusted, got, tc.want)
}
}
if !allowRequest("https://[::1]/x", []string{"[::1]"}, false) {
t.Fatal("a bracketed IPv6 host must match")
}
}
// TestResolverRules pins Chrome's --host-resolver-rules: one MAP per
// allowed host, sorted, to its first IPv4 address (else the first one, in
// brackets for IPv6), a host without addresses left out, and MAP *
// ~NOTFOUND last, since Chrome takes the first matching rule.
func TestResolverRules(t *testing.T) {
got := resolverRules(map[string][]netip.Addr{
"www.example.test": {netip.MustParseAddr("2001:db8::1")},
"example.test": {netip.MustParseAddr("2001:db8::2"), netip.MustParseAddr("192.0.2.10"), netip.MustParseAddr("192.0.2.11")},
"empty.test": nil,
})
want := "MAP example.test 192.0.2.10, MAP www.example.test [2001:db8::1], MAP * ~NOTFOUND"
if got != want {
t.Fatalf("rules = %q\nwant %q", got, want)
}
if got := resolverRules(nil); got != "MAP * ~NOTFOUND" {
t.Fatalf("no hosts = %q", got)
}
if !strings.HasSuffix(got, "MAP * ~NOTFOUND") || strings.Count(got, "~NOTFOUND") != 1 {
t.Fatal("the catch-all must come once and last")
}
}
// TestNewRenderer pins the comparison's start: no renderer and no error
// without a Chrome path (skipped), an error for a relative path, a
// missing file, a directory or a file that is not executable (failed).
func TestNewRenderer(t *testing.T) {
site, _ := url.Parse("https://example.test")
cfg := func(path string) *compass.Config {
c, err := compass.Open(compass.Options{Dir: t.TempDir()})
if err != nil {
t.Fatal(err)
}
if path != "" {
if err := c.Set(ChromePathConfigKey, path); err != nil {
t.Fatal(err)
}
}
return c
}
if r, err := NewRenderer(context.Background(), cfg(""), site, false, "ua"); r != nil || err != nil {
t.Fatalf("no path: %v %v, want skipped", r, err)
}
if r, err := NewRenderer(context.Background(), nil, site, false, "ua"); r != nil || err != nil {
t.Fatal("no config: skipped")
}
dir := t.TempDir()
plain := filepath.Join(dir, "chrome.txt")
if err := os.WriteFile(plain, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
for name, path := range map[string]string{"relative": "bin/chrome", "missing": filepath.Join(dir, "nope"), "directory": dir, "not executable": plain} {
r, err := NewRenderer(context.Background(), cfg(path), site, false, "ua")
if r != nil || err == nil || !strings.Contains(err.Error(), "Chrome path") {
t.Errorf("%s: %v %v, want a Chrome path error", name, r, err)
}
}
}
// TestRenderMax pins the page cap: the default 200, a positive config
// value, and the default for 0 or less.
func TestRenderMax(t *testing.T) {
if RenderMax(nil) != DefaultRenderMax || DefaultRenderMax != 200 {
t.Fatal("default cap")
}
for v, want := range map[int]int{50: 50, 0: DefaultRenderMax, -1: DefaultRenderMax} {
c, _ := compass.Open(compass.Options{Dir: t.TempDir()})
_ = c.Set(RenderMaxConfigKey, v)
if got := RenderMax(c); got != want {
t.Errorf("RenderMax(%d) = %d, want %d", v, got, want)
}
}
if got := SiteHosts("WWW.Example.test"); len(got) != 2 || got[0] != "www.example.test" || got[1] != "example.test" {
t.Fatalf("SiteHosts = %v", got)
}
if got := SiteHosts("example.test"); got[1] != "www.example.test" || SiteHosts(" ") != nil {
t.Fatalf("SiteHosts = %v", got)
}
}