3fdb44bce2
Fetcher.Allowed and Key, the rendered comparison and the Chrome request guard accept an address only on the site's effective port (explicit, else 443 for https and 80 for http), so page code cannot make headless Chrome reach other services on the site host's address and a link to another port is not reported as a page of the site.
83 lines
3.1 KiB
Go
83 lines
3.1 KiB
Go
package crawl
|
|
|
|
import (
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func linkFacts(words int, h1 string, hrefs ...string) *PageFacts {
|
|
f := &PageFacts{Words: words}
|
|
if h1 != "" {
|
|
f.Headings = []Heading{{Level: 1, Text: h1}}
|
|
}
|
|
for _, h := range hrefs {
|
|
u, _ := url.Parse(h)
|
|
f.Links = append(f.Links, Link{URL: u, Href: h})
|
|
}
|
|
return f
|
|
}
|
|
|
|
// siteKey is the crawl key of a link on https://example.test (its www
|
|
// twin too, port 443 only).
|
|
func siteKey(u *url.URL) string {
|
|
f, err := NewFetcher("https://example.test", nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return f.Key(u)
|
|
}
|
|
|
|
// TestCompareRendered pins the rendered comparison (P33-55): a same-host
|
|
// link (the www twin too), an H1 and growth of at least 50 words and 20 %
|
|
// present only after JavaScript are one content_js finding with check
|
|
// rendered; other-host links, smaller growth and an identical rendering
|
|
// find nothing.
|
|
func TestCompareRendered(t *testing.T) {
|
|
raw := linkFacts(100, "", "https://example.test/a")
|
|
rendered := linkFacts(160, "Grzyby", "https://example.test/a", "https://www.example.test/b?x=1", "https://other.test/c", "https://example.test/b?x=1")
|
|
got := compareRendered(raw, rendered, siteKey)
|
|
if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkRendered {
|
|
t.Fatalf("findings = %+v", got)
|
|
}
|
|
d := got[0].Details
|
|
if links := d["links"].([]string); len(links) != 1 || links[0] != "/b?x=1" || d["h1"] != "Grzyby" || d["words_raw"] != 100 || d["words_rendered"] != 160 {
|
|
t.Fatalf("details = %v", d)
|
|
}
|
|
if !strings.Contains(got[0].What, "links /b?x=1") || !strings.Contains(got[0].What, "H1 “Grzyby”") || !strings.Contains(got[0].What, "60 words of text") {
|
|
t.Fatalf("what = %q", got[0].What)
|
|
}
|
|
|
|
for name, pair := range map[string][2]*PageFacts{
|
|
"identical": {linkFacts(100, "T", "https://example.test/a"), linkFacts(100, "T", "https://example.test/a")},
|
|
"other host only": {linkFacts(100, "T"), linkFacts(100, "T", "https://other.test/x")},
|
|
"49 words": {linkFacts(100, "T"), linkFacts(149, "T")},
|
|
"50 words under 20 %": {linkFacts(300, "T"), linkFacts(350, "T")},
|
|
"H1 already there": {linkFacts(10, "Raw"), linkFacts(10, "Rendered")},
|
|
} {
|
|
if got := compareRendered(pair[0], pair[1], siteKey); got != nil {
|
|
t.Errorf("%s: %+v", name, got)
|
|
}
|
|
}
|
|
if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteKey); len(got) != 1 {
|
|
t.Fatalf("50 words from an empty page: %+v", got)
|
|
}
|
|
if compareRendered(nil, rendered, siteKey) != nil || compareRendered(raw, nil, siteKey) != nil {
|
|
t.Fatal("a missing side compares to nothing")
|
|
}
|
|
}
|
|
|
|
// TestCompareRenderedCap pins the cap of the links a finding names, and
|
|
// that a path without a slash counts as /.
|
|
func TestCompareRenderedCap(t *testing.T) {
|
|
var hrefs []string
|
|
for _, p := range []string{"", "/1", "/2", "/3", "/4", "/5", "/6"} {
|
|
hrefs = append(hrefs, "https://example.test"+p)
|
|
}
|
|
got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteKey)
|
|
links := got[0].Details["links"].([]string)
|
|
if len(links) != maxJSOnlyURLs || links[0] != "/" {
|
|
t.Fatalf("links = %v", links)
|
|
}
|
|
}
|