package crawl import ( "net/url" "strings" "testing" ) const fixtureHTML = ` Grzyby lecznicze Second
Start

Grzyby lecznicze

Detail

Raz dwa trzy.

Chaga jsmailteltop Other
` // TestParseFacts pins Parse on an HTML fixture: language, the first title // collapsed, description, robots joined from robots and googlebot, // og:image and the first canonical resolved without the fragment, // hreflang alternates only, JSON-LD scripts only, headings in order (an // image alt counts as heading text, script content never), links without // javascript:, mailto:, tel: and fragment-only ones, images with and // without alt, and the main text without noscript and style. func TestParseFacts(t *testing.T) { base, _ := url.Parse("https://example.test/blog/grzyby") f := Parse(base, []byte(fixtureHTML)) if f.Lang != "pl-PL" || !f.HasTitle || f.Title != "Grzyby lecznicze" || !f.HasDescription || f.Description != "O grzybach" { t.Fatalf("head = %+v", f) } if f.Robots != "index, follow, noimageindex" || f.Noindex() { t.Fatalf("robots = %q", f.Robots) } if f.OGImage != "https://example.test/media/cover.jpg" || f.Canonical != "https://example.test/blog/grzyby" { t.Fatalf("og:image %q canonical %q", f.OGImage, f.Canonical) } if len(f.Alternates) != 2 || f.Alternates[0].Lang != "en" || f.Alternates[0].URL.String() != "https://example.test/en/blog/mushrooms" || f.Alternates[1].Lang != "x-default" { t.Fatalf("alternates = %+v", f.Alternates) } if len(f.JSONLD) != 1 || f.JSONLD[0] != `{"@type":"WebPage"}` { t.Fatalf("json-ld = %q", f.JSONLD) } if len(f.Headings) != 3 || f.Headings[0] != (Heading{1, "Grzyby lecznicze"}) || f.Headings[1] != (Heading{2, ""}) || f.Headings[2] != (Heading{4, "Detail"}) { t.Fatalf("headings = %+v", f.Headings) } if h1 := f.H1(); len(h1) != 1 || h1[0] != "Grzyby lecznicze" { t.Fatalf("H1 = %v", h1) } var links []string for _, l := range f.Links { links = append(links, l.URL.String()) } if strings.Join(links, " ") != "https://example.test/ https://example.test/blog/chaga?x=1 https://other.test/x" || f.Links[1].Rel != "nofollow" || f.Links[1].Text != "Chaga" { t.Fatalf("links = %v (%+v)", links, f.Links) } if len(f.Images) != 3 || !f.Images[1].HasAlt || f.Images[1].Alt != "" || f.Images[1].Loading != "lazy" || f.Images[2].HasAlt || f.Images[2].Src != "https://example.test/media/b.jpg" { t.Fatalf("images = %+v", f.Images) } if strings.Contains(f.Text, "hidden text") || strings.Contains(f.Text, ".x{}") || strings.Contains(f.Text, "Stopka") || !strings.Contains(f.Text, "Raz dwa trzy.") || f.Words != len(strings.Fields(f.Text)) { t.Fatalf("text = %q (%d words)", f.Text, f.Words) } } // TestParseBodyText pins the text root: main when there is one, else the // body; and an empty document parses to empty facts. func TestParseBodyText(t *testing.T) { f := Parse(nil, []byte(`

One two

`)) if f.Text != "Menu One two" || f.Words != 3 || f.HasTitle { t.Fatalf("body text = %q", f.Text) } if empty := Parse(nil, nil); empty.Words != 0 || len(empty.Links) != 0 { t.Fatalf("empty = %+v", empty) } } // TestNoindexValues pins the robots values that mean noindex. func TestNoindexValues(t *testing.T) { for v, want := range map[string]bool{"noindex": true, "NOINDEX, follow": true, "none": true, "index;noindex": true, "index, follow": false, "": false, "noindexing": false} { if got := hasNoindex(v); got != want { t.Errorf("hasNoindex(%q) = %v, want %v", v, got, want) } } } // TestParseCaps pins the link cap and the text cap with a UTF-8 safe cut. func TestParseCaps(t *testing.T) { var b strings.Builder b.WriteString("
") for i := 0; i < MaxLinksPerPage+10; i++ { b.WriteString(`x`) } b.WriteString("
") if got := len(Parse(nil, []byte(b.String())).Links); got != MaxLinksPerPage { t.Fatalf("%d links, want the %d cap", got, MaxLinksPerPage) } long := "
" + strings.Repeat("żółw ", maxText/4) + "
" f := Parse(nil, []byte(long)) if len(f.Text) > maxText || !strings.HasPrefix(f.Text, "żółw") || strings.ContainsRune(f.Text, '�') { t.Fatalf("text of %d bytes", len(f.Text)) } if truncateUTF8("żółw", 2) != "ż" || truncateUTF8("ab", 5) != "ab" { t.Fatal("truncateUTF8 is wrong") } }