package crawl
import (
"net/url"
"strings"
"testing"
)
const fixtureHTML = `
Grzyby lecznicze
Second
Grzyby 
Detail
Raz dwa trzy.
Chaga
jsmailteltop
Other
`
// TestParseFacts pins Parse on an HTML fixture: language, the first title
// collapsed, description, robots joined from robots and googlebot,
// og:image and the first canonical resolved without the fragment,
// hreflang alternates only, JSON-LD scripts only, headings in order (an
// image alt counts as heading text, script content never), links without
// javascript:, mailto:, tel: and fragment-only ones, images with and
// without alt, and the main text without noscript and style.
func TestParseFacts(t *testing.T) {
base, _ := url.Parse("https://example.test/blog/grzyby")
f := Parse(base, []byte(fixtureHTML))
if f.Lang != "pl-PL" || !f.HasTitle || f.Title != "Grzyby lecznicze" || !f.HasDescription || f.Description != "O grzybach" {
t.Fatalf("head = %+v", f)
}
if f.Robots != "index, follow, noimageindex" || f.Noindex() {
t.Fatalf("robots = %q", f.Robots)
}
if f.OGImage != "https://example.test/media/cover.jpg" || f.Canonical != "https://example.test/blog/grzyby" {
t.Fatalf("og:image %q canonical %q", f.OGImage, f.Canonical)
}
if len(f.Alternates) != 2 || f.Alternates[0].Lang != "en" || f.Alternates[0].URL.String() != "https://example.test/en/blog/mushrooms" || f.Alternates[1].Lang != "x-default" {
t.Fatalf("alternates = %+v", f.Alternates)
}
if len(f.JSONLD) != 1 || f.JSONLD[0] != `{"@type":"WebPage"}` {
t.Fatalf("json-ld = %q", f.JSONLD)
}
if len(f.Headings) != 3 || f.Headings[0] != (Heading{1, "Grzyby lecznicze"}) || f.Headings[1] != (Heading{2, ""}) || f.Headings[2] != (Heading{4, "Detail"}) {
t.Fatalf("headings = %+v", f.Headings)
}
if h1 := f.H1(); len(h1) != 1 || h1[0] != "Grzyby lecznicze" {
t.Fatalf("H1 = %v", h1)
}
var links []string
for _, l := range f.Links {
links = append(links, l.URL.String())
}
if strings.Join(links, " ") != "https://example.test/ https://example.test/blog/chaga?x=1 https://other.test/x" || f.Links[1].Rel != "nofollow" || f.Links[1].Text != "Chaga" {
t.Fatalf("links = %v (%+v)", links, f.Links)
}
if len(f.Images) != 3 || !f.Images[1].HasAlt || f.Images[1].Alt != "" || f.Images[1].Loading != "lazy" || f.Images[2].HasAlt || f.Images[2].Src != "https://example.test/media/b.jpg" {
t.Fatalf("images = %+v", f.Images)
}
if strings.Contains(f.Text, "hidden text") || strings.Contains(f.Text, ".x{}") || strings.Contains(f.Text, "Stopka") || !strings.Contains(f.Text, "Raz dwa trzy.") || f.Words != len(strings.Fields(f.Text)) {
t.Fatalf("text = %q (%d words)", f.Text, f.Words)
}
}
// TestParseBodyText pins the text root: main when there is one, else the
// body; and an empty document parses to empty facts.
func TestParseBodyText(t *testing.T) {
f := Parse(nil, []byte(`One two
`))
if f.Text != "Menu One two" || f.Words != 3 || f.HasTitle {
t.Fatalf("body text = %q", f.Text)
}
if empty := Parse(nil, nil); empty.Words != 0 || len(empty.Links) != 0 {
t.Fatalf("empty = %+v", empty)
}
}
// TestNoindexValues pins the robots values that mean noindex.
func TestNoindexValues(t *testing.T) {
for v, want := range map[string]bool{"noindex": true, "NOINDEX, follow": true, "none": true, "index;noindex": true, "index, follow": false, "": false, "noindexing": false} {
if got := hasNoindex(v); got != want {
t.Errorf("hasNoindex(%q) = %v, want %v", v, got, want)
}
}
}
// TestParseCaps pins the link cap and the text cap with a UTF-8 safe cut.
func TestParseCaps(t *testing.T) {
var b strings.Builder
b.WriteString("")
for i := 0; i < MaxLinksPerPage+10; i++ {
b.WriteString(`x`)
}
b.WriteString("")
if got := len(Parse(nil, []byte(b.String())).Links); got != MaxLinksPerPage {
t.Fatalf("%d links, want the %d cap", got, MaxLinksPerPage)
}
long := "" + strings.Repeat("żółw ", maxText/4) + ""
f := Parse(nil, []byte(long))
if len(f.Text) > maxText || !strings.HasPrefix(f.Text, "żółw") || strings.ContainsRune(f.Text, '�') {
t.Fatalf("text of %d bytes", len(f.Text))
}
if truncateUTF8("żółw", 2) != "ż" || truncateUTF8("ab", 5) != "ab" {
t.Fatal("truncateUTF8 is wrong")
}
}