test: unit tests for the schema builder, crawler and page helpers

- schema: types per kind, @id links, every node kind, the publisher and
  local business, every validator rule, custom JSON-LD caps and merge
- crawl: HTML parse, each check's positive and negative case, fix routes
  with the fitting tool, the rendered comparison, allowRequest,
  resolverRules and NewRenderer
- pagination, alternates and x-default, primary parents, the filter's
  addresses, registry paths and fields, auto-link addresses

classes/crawl/parse_test.go

@@ -0,0 +1,128 @@
+package crawl
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+)
+
+const fixtureHTML = `<!doctype html>
+<html lang="pl-PL"><head>
+<title>  Grzyby   lecznicze </title>
+<title>Second</title>
+<meta name="description" content=" O grzybach ">
+<meta name="robots" content="index, follow">
+<meta name="googlebot" content="noimageindex">
+<meta property="og:image" content="/media/cover.jpg">
+<link rel="canonical" href="/blog/grzyby#top">
+<link rel="canonical" href="/other">
+<link rel="alternate" hreflang="en" href="/en/blog/mushrooms">
+<link rel="alternate" hreflang="x-default" href="https://example.test/blog/grzyby">
+<link rel="alternate" type="application/rss+xml" href="/feed.xml">
+<script type="application/ld+json">{"@type":"WebPage"}</script>
+<script>var x = "<h1>not a heading</h1>";</script>
+</head><body>
+<header><a href="/">Start</a></header>
+<main>
+<h1>Grzyby <img src="/i/logo.png" alt="lecznicze"></h1>
+<h2></h2>
+<h4>Detail</h4>
+<p>Raz dwa trzy.</p>
+<a href="/blog/chaga?x=1#frag" rel="NoFollow">Chaga</a>
+<a href="javascript:void(0)">js</a><a href="mailto:a@b.c">mail</a><a href="tel:1">tel</a><a href="#top">top</a>
+<a href="https://other.test/x">Other</a>
+<img src="/media/a.jpg" alt="" width="10" height="5" loading="LAZY">
+<img src="/media/b.jpg">
+<noscript>hidden text</noscript>
+<style>.x{}</style>
+</main>
+<footer>Stopka</footer>
+</body></html>`
+
+// TestParseFacts pins Parse on an HTML fixture: language, the first title
+// collapsed, description, robots joined from robots and googlebot,
+// og:image and the first canonical resolved without the fragment,
+// hreflang alternates only, JSON-LD scripts only, headings in order (an
+// image alt counts as heading text, script content never), links without
+// javascript:, mailto:, tel: and fragment-only ones, images with and
+// without alt, and the main text without noscript and style.
+func TestParseFacts(t *testing.T) {
+	base, _ := url.Parse("https://example.test/blog/grzyby")
+	f := Parse(base, []byte(fixtureHTML))
+	if f.Lang != "pl-PL" || !f.HasTitle || f.Title != "Grzyby lecznicze" || !f.HasDescription || f.Description != "O grzybach" {
+		t.Fatalf("head = %+v", f)
+	}
+	if f.Robots != "index, follow, noimageindex" || f.Noindex() {
+		t.Fatalf("robots = %q", f.Robots)
+	}
+	if f.OGImage != "https://example.test/media/cover.jpg" || f.Canonical != "https://example.test/blog/grzyby" {
+		t.Fatalf("og:image %q canonical %q", f.OGImage, f.Canonical)
+	}
+	if len(f.Alternates) != 2 || f.Alternates[0].Lang != "en" || f.Alternates[0].URL.String() != "https://example.test/en/blog/mushrooms" || f.Alternates[1].Lang != "x-default" {
+		t.Fatalf("alternates = %+v", f.Alternates)
+	}
+	if len(f.JSONLD) != 1 || f.JSONLD[0] != `{"@type":"WebPage"}` {
+		t.Fatalf("json-ld = %q", f.JSONLD)
+	}
+	if len(f.Headings) != 3 || f.Headings[0] != (Heading{1, "Grzyby lecznicze"}) || f.Headings[1] != (Heading{2, ""}) || f.Headings[2] != (Heading{4, "Detail"}) {
+		t.Fatalf("headings = %+v", f.Headings)
+	}
+	if h1 := f.H1(); len(h1) != 1 || h1[0] != "Grzyby lecznicze" {
+		t.Fatalf("H1 = %v", h1)
+	}
+	var links []string
+	for _, l := range f.Links {
+		links = append(links, l.URL.String())
+	}
+	if strings.Join(links, " ") != "https://example.test/ https://example.test/blog/chaga?x=1 https://other.test/x" || f.Links[1].Rel != "nofollow" || f.Links[1].Text != "Chaga" {
+		t.Fatalf("links = %v (%+v)", links, f.Links)
+	}
+	if len(f.Images) != 3 || !f.Images[1].HasAlt || f.Images[1].Alt != "" || f.Images[1].Loading != "lazy" || f.Images[2].HasAlt || f.Images[2].Src != "https://example.test/media/b.jpg" {
+		t.Fatalf("images = %+v", f.Images)
+	}
+	if strings.Contains(f.Text, "hidden text") || strings.Contains(f.Text, ".x{}") || strings.Contains(f.Text, "Stopka") || !strings.Contains(f.Text, "Raz dwa trzy.") || f.Words != len(strings.Fields(f.Text)) {
+		t.Fatalf("text = %q (%d words)", f.Text, f.Words)
+	}
+}
+
+// TestParseBodyText pins the text root: main when there is one, else the
+// body; and an empty document parses to empty facts.
+func TestParseBodyText(t *testing.T) {
+	f := Parse(nil, []byte(`<html><body><nav>Menu</nav><p>One two</p></body></html>`))
+	if f.Text != "Menu One two" || f.Words != 3 || f.HasTitle {
+		t.Fatalf("body text = %q", f.Text)
+	}
+	if empty := Parse(nil, nil); empty.Words != 0 || len(empty.Links) != 0 {
+		t.Fatalf("empty = %+v", empty)
+	}
+}
+
+// TestNoindexValues pins the robots values that mean noindex.
+func TestNoindexValues(t *testing.T) {
+	for v, want := range map[string]bool{"noindex": true, "NOINDEX, follow": true, "none": true, "index;noindex": true, "index, follow": false, "": false, "noindexing": false} {
+		if got := hasNoindex(v); got != want {
+			t.Errorf("hasNoindex(%q) = %v, want %v", v, got, want)
+		}
+	}
+}
+
+// TestParseCaps pins the link cap and the text cap with a UTF-8 safe cut.
+func TestParseCaps(t *testing.T) {
+	var b strings.Builder
+	b.WriteString("<main>")
+	for i := 0; i < MaxLinksPerPage+10; i++ {
+		b.WriteString(`<a href="/x">x</a>`)
+	}
+	b.WriteString("</main>")
+	if got := len(Parse(nil, []byte(b.String())).Links); got != MaxLinksPerPage {
+		t.Fatalf("%d links, want the %d cap", got, MaxLinksPerPage)
+	}
+	long := "<main>" + strings.Repeat("żółw ", maxText/4) + "</main>"
+	f := Parse(nil, []byte(long))
+	if len(f.Text) > maxText || !strings.HasPrefix(f.Text, "żółw") || strings.ContainsRune(f.Text, '<27>') {
+		t.Fatalf("text of %d bytes", len(f.Text))
+	}
+	if truncateUTF8("żółw", 2) != "ż" || truncateUTF8("ab", 5) != "ab" {
+		t.Fatal("truncateUTF8 is wrong")
+	}
+}