82b0dacff9
- schema: types per kind, @id links, every node kind, the publisher and local business, every validator rule, custom JSON-LD caps and merge - crawl: HTML parse, each check's positive and negative case, fix routes with the fitting tool, the rendered comparison, allowRequest, resolverRules and NewRenderer - pagination, alternates and x-default, primary parents, the filter's addresses, registry paths and fields, auto-link addresses
75 lines
2.9 KiB
Go
75 lines
2.9 KiB
Go
package crawl
|
|
|
|
import (
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func linkFacts(words int, h1 string, hrefs ...string) *PageFacts {
|
|
f := &PageFacts{Words: words}
|
|
if h1 != "" {
|
|
f.Headings = []Heading{{Level: 1, Text: h1}}
|
|
}
|
|
for _, h := range hrefs {
|
|
u, _ := url.Parse(h)
|
|
f.Links = append(f.Links, Link{URL: u, Href: h})
|
|
}
|
|
return f
|
|
}
|
|
|
|
var siteHosts = []string{"example.test", "www.example.test"}
|
|
|
|
// TestCompareRendered pins the rendered comparison (P33-55): a same-host
|
|
// link (the www twin too), an H1 and growth of at least 50 words and 20 %
|
|
// present only after JavaScript are one content_js finding with check
|
|
// rendered; other-host links, smaller growth and an identical rendering
|
|
// find nothing.
|
|
func TestCompareRendered(t *testing.T) {
|
|
raw := linkFacts(100, "", "https://example.test/a")
|
|
rendered := linkFacts(160, "Grzyby", "https://example.test/a", "https://www.example.test/b?x=1", "https://other.test/c", "https://example.test/b?x=1")
|
|
got := compareRendered(raw, rendered, siteHosts)
|
|
if len(got) != 1 || got[0].Kind != KindContentJS || got[0].Check != checkRendered {
|
|
t.Fatalf("findings = %+v", got)
|
|
}
|
|
d := got[0].Details
|
|
if links := d["links"].([]string); len(links) != 1 || links[0] != "/b?x=1" || d["h1"] != "Grzyby" || d["words_raw"] != 100 || d["words_rendered"] != 160 {
|
|
t.Fatalf("details = %v", d)
|
|
}
|
|
if !strings.Contains(got[0].What, "links /b?x=1") || !strings.Contains(got[0].What, "H1 “Grzyby”") || !strings.Contains(got[0].What, "60 words of text") {
|
|
t.Fatalf("what = %q", got[0].What)
|
|
}
|
|
|
|
for name, pair := range map[string][2]*PageFacts{
|
|
"identical": {linkFacts(100, "T", "https://example.test/a"), linkFacts(100, "T", "https://example.test/a")},
|
|
"other host only": {linkFacts(100, "T"), linkFacts(100, "T", "https://other.test/x")},
|
|
"49 words": {linkFacts(100, "T"), linkFacts(149, "T")},
|
|
"50 words under 20 %": {linkFacts(300, "T"), linkFacts(350, "T")},
|
|
"H1 already there": {linkFacts(10, "Raw"), linkFacts(10, "Rendered")},
|
|
} {
|
|
if got := compareRendered(pair[0], pair[1], siteHosts); got != nil {
|
|
t.Errorf("%s: %+v", name, got)
|
|
}
|
|
}
|
|
if got := compareRendered(linkFacts(0, "T"), linkFacts(50, "T"), siteHosts); len(got) != 1 {
|
|
t.Fatalf("50 words from an empty page: %+v", got)
|
|
}
|
|
if compareRendered(nil, rendered, siteHosts) != nil || compareRendered(raw, nil, siteHosts) != nil {
|
|
t.Fatal("a missing side compares to nothing")
|
|
}
|
|
}
|
|
|
|
// TestCompareRenderedCap pins the cap of the links a finding names, and
|
|
// that a path without a slash counts as /.
|
|
func TestCompareRenderedCap(t *testing.T) {
|
|
var hrefs []string
|
|
for _, p := range []string{"", "/1", "/2", "/3", "/4", "/5", "/6"} {
|
|
hrefs = append(hrefs, "https://example.test"+p)
|
|
}
|
|
got := compareRendered(linkFacts(10, "T"), linkFacts(10, "T", hrefs...), siteHosts)
|
|
links := got[0].Details["links"].([]string)
|
|
if len(links) != maxJSOnlyURLs || links[0] != "/" {
|
|
t.Fatalf("links = %v", links)
|
|
}
|
|
}
|