sm-seo-plugin/classes/crawl/parse_test.go

package crawl

import (
	"net/url"
	"strings"
	"testing"
)

const fixtureHTML = `<!doctype html>
<html lang="pl-PL"><head>
<title>  Grzyby   lecznicze </title>
<title>Second</title>
<meta name="description" content=" O grzybach ">
<meta name="robots" content="index, follow">
<meta name="googlebot" content="noimageindex">
<meta property="og:image" content="/media/cover.jpg">
<link rel="canonical" href="/blog/grzyby#top">
<link rel="canonical" href="/other">
<link rel="alternate" hreflang="en" href="/en/blog/mushrooms">
<link rel="alternate" hreflang="x-default" href="https://example.test/blog/grzyby">
<link rel="alternate" type="application/rss+xml" href="/feed.xml">
<script type="application/ld+json">{"@type":"WebPage"}</script>
<script>var x = "<h1>not a heading</h1>";</script>
</head><body>
<header><a href="/">Start</a></header>
<main>
<h1>Grzyby <img src="/i/logo.png" alt="lecznicze"></h1>
<h2></h2>
<h4>Detail</h4>
<p>Raz dwa trzy.</p>
<a href="/blog/chaga?x=1#frag" rel="NoFollow">Chaga</a>
<a href="javascript:void(0)">js</a><a href="mailto:a@b.c">mail</a><a href="tel:1">tel</a><a href="#top">top</a>
<a href="https://other.test/x">Other</a>
<img src="/media/a.jpg" alt="" width="10" height="5" loading="LAZY">
<img src="/media/b.jpg">
<noscript>hidden text</noscript>
<style>.x{}</style>
</main>
<footer>Stopka</footer>
</body></html>`

// TestParseFacts pins Parse on an HTML fixture: language, the first title
// collapsed, description, robots joined from robots and googlebot,
// og:image and the first canonical resolved without the fragment,
// hreflang alternates only, JSON-LD scripts only, headings in order (an
// image alt counts as heading text, script content never), links without
// javascript:, mailto:, tel: and fragment-only ones, images with and
// without alt, and the main text without noscript and style.
func TestParseFacts(t *testing.T) {
	base, _ := url.Parse("https://example.test/blog/grzyby")
	f := Parse(base, []byte(fixtureHTML))
	if f.Lang != "pl-PL" || !f.HasTitle || f.Title != "Grzyby lecznicze" || !f.HasDescription || f.Description != "O grzybach" {
		t.Fatalf("head = %+v", f)
	}
	if f.Robots != "index, follow, noimageindex" || f.Noindex() {
		t.Fatalf("robots = %q", f.Robots)
	}
	if f.OGImage != "https://example.test/media/cover.jpg" || f.Canonical != "https://example.test/blog/grzyby" {
		t.Fatalf("og:image %q canonical %q", f.OGImage, f.Canonical)
	}
	if len(f.Alternates) != 2 || f.Alternates[0].Lang != "en" || f.Alternates[0].URL.String() != "https://example.test/en/blog/mushrooms" || f.Alternates[1].Lang != "x-default" {
		t.Fatalf("alternates = %+v", f.Alternates)
	}
	if len(f.JSONLD) != 1 || f.JSONLD[0] != `{"@type":"WebPage"}` {
		t.Fatalf("json-ld = %q", f.JSONLD)
	}
	if len(f.Headings) != 3 || f.Headings[0] != (Heading{1, "Grzyby lecznicze"}) || f.Headings[1] != (Heading{2, ""}) || f.Headings[2] != (Heading{4, "Detail"}) {
		t.Fatalf("headings = %+v", f.Headings)
	}
	if h1 := f.H1(); len(h1) != 1 || h1[0] != "Grzyby lecznicze" {
		t.Fatalf("H1 = %v", h1)
	}
	var links []string
	for _, l := range f.Links {
		links = append(links, l.URL.String())
	}
	if strings.Join(links, " ") != "https://example.test/ https://example.test/blog/chaga?x=1 https://other.test/x" || f.Links[1].Rel != "nofollow" || f.Links[1].Text != "Chaga" {
		t.Fatalf("links = %v (%+v)", links, f.Links)
	}
	if len(f.Images) != 3 || !f.Images[1].HasAlt || f.Images[1].Alt != "" || f.Images[1].Loading != "lazy" || f.Images[2].HasAlt || f.Images[2].Src != "https://example.test/media/b.jpg" {
		t.Fatalf("images = %+v", f.Images)
	}
	if strings.Contains(f.Text, "hidden text") || strings.Contains(f.Text, ".x{}") || strings.Contains(f.Text, "Stopka") || !strings.Contains(f.Text, "Raz dwa trzy.") || f.Words != len(strings.Fields(f.Text)) {
		t.Fatalf("text = %q (%d words)", f.Text, f.Words)
	}
}

// TestParseBodyText pins the text root: main when there is one, else the
// body; and an empty document parses to empty facts.
func TestParseBodyText(t *testing.T) {
	f := Parse(nil, []byte(`<html><body><nav>Menu</nav><p>One two</p></body></html>`))
	if f.Text != "Menu One two" || f.Words != 3 || f.HasTitle {
		t.Fatalf("body text = %q", f.Text)
	}
	if empty := Parse(nil, nil); empty.Words != 0 || len(empty.Links) != 0 {
		t.Fatalf("empty = %+v", empty)
	}
}

// TestNoindexValues pins the robots values that mean noindex.
func TestNoindexValues(t *testing.T) {
	for v, want := range map[string]bool{"noindex": true, "NOINDEX, follow": true, "none": true, "index;noindex": true, "index, follow": false, "": false, "noindexing": false} {
		if got := hasNoindex(v); got != want {
			t.Errorf("hasNoindex(%q) = %v, want %v", v, got, want)
		}
	}
}

// TestParseCaps pins the link cap and the text cap with a UTF-8 safe cut.
func TestParseCaps(t *testing.T) {
	var b strings.Builder
	b.WriteString("<main>")
	for i := 0; i < MaxLinksPerPage+10; i++ {
		b.WriteString(`<a href="/x">x</a>`)
	}
	b.WriteString("</main>")
	if got := len(Parse(nil, []byte(b.String())).Links); got != MaxLinksPerPage {
		t.Fatalf("%d links, want the %d cap", got, MaxLinksPerPage)
	}
	long := "<main>" + strings.Repeat("żółw ", maxText/4) + "</main>"
	f := Parse(nil, []byte(long))
	if len(f.Text) > maxText || !strings.HasPrefix(f.Text, "żółw") || strings.ContainsRune(f.Text, '<27>') {
		t.Fatalf("text of %d bytes", len(f.Text))
	}
	if truncateUTF8("żółw", 2) != "ż" || truncateUTF8("ab", 5) != "ab" {
		t.Fatal("truncateUTF8 is wrong")
	}
}