package crawl import ( "net/url" "strings" "testing" ) func linkFacts(words int, h1 string, hrefs ...string) *PageFacts { f := &PageFacts{Words: words} if h1 != "" { f.Headings = []Heading{{Level: 1, Text: h1}} } for _, h := range hrefs { u, _ := url.Parse(h) f.Links = append(f.Links, Link{URL: u, Href: h}) } return f } var siteHosts = []string{"example.test", "www.example.test"} // TestCompareRendered pins the rendered comparison (P33-55): a same-host // link (the www twin too), an H1 and growth of at least 50 words and 20 % // present only after JavaScript are one content_js finding with check // rendered; other-host links, smaller growth and an identical rendering // find nothing. func TestCompareRendered(t *testing.T) { raw := linkFacts(100, "", "https://example.test/a") rendered := linkFacts(160, "Grzyby", "https://example.test/a", "https://www.example.test/b?x=1", "https://other.test/c", "https://example.test/b?x=1") got := compareRendered(raw, rendered, siteHosts) if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkRendered { t.Fatalf("findings = %+v", got) } d := got[0].Details if links := d["links"].([]string); len(links) != 1 || links[0] != "/b?x=1" || d["h1"] != "Grzyby" || d["words_raw"] != 100 || d["words_rendered"] != 160 { t.Fatalf("details = %v", d) } if !strings.Contains(got[0].What, "links /b?x=1") || !strings.Contains(got[0].What, "H1 “Grzyby”") || !strings.Contains(got[0].What, "60 words of text") { t.Fatalf("what = %q", got[0].What) } for name, pair := range map[string][2]*PageFacts{ "identical": {linkFacts(100, "T", "https://example.test/a"), linkFacts(100, "T", "https://example.test/a")}, "other host only": {linkFacts(100, "T"), linkFacts(100, "T", "https://other.test/x")}, "49 words": {linkFacts(100, "T"), linkFacts(149, "T")}, "50 words under 20 %": {linkFacts(300, "T"), linkFacts(350, "T")}, "H1 already there": {linkFacts(10, "Raw"), linkFacts(10, "Rendered")}, } { if got := compareRendered(pair[0], pair[1], siteHosts); got != nil { t.Errorf("%s: %+v", name, got) } } if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteHosts); len(got) != 1 { t.Fatalf("50 words from an empty page: %+v", got) } if compareRendered(nil, rendered, siteHosts) != nil || compareRendered(raw, nil, siteHosts) != nil { t.Fatal("a missing side compares to nothing") } } // TestCompareRenderedCap pins the cap of the links a finding names, and // that a path without a slash counts as /. func TestCompareRenderedCap(t *testing.T) { var hrefs []string for _, p := range []string{"", "/1", "/2", "/3", "/4", "/5", "/6"} { hrefs = append(hrefs, "https://example.test"+p) } got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteHosts) links := got[0].Details["links"].([]string) if len(links) != maxJSOnlyURLs || links[0] != "/" { t.Fatalf("links = %v", links) } }