- schema: types per kind, @id links, every node kind, the publisher and local business, every validator rule, custom JSON-LD caps and merge - crawl: HTML parse, each check's positive and negative case, fix routes with the fitting tool, the rendered comparison, allowRequest, resolverRules and NewRenderer - pagination, alternates and x-default, primary parents, the filter's addresses, registry paths and fields, auto-link addresses
127 lines
4.6 KiB
Go
127 lines
4.6 KiB
Go
package crawl
|
|
|
|
import (
|
|
"context"
|
|
"net/netip"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/compass"
|
|
)
|
|
|
|
// TestAllowRequest pins what the headless browser may load: https on the
|
|
// site host and its www twin, data: and blob:; http only in trusted mode;
|
|
// other hosts, credentials in the URL and every other scheme refused.
|
|
func TestAllowRequest(t *testing.T) {
|
|
hosts := []string{"example.test", "www.example.test"}
|
|
for _, tc := range []struct {
|
|
url string
|
|
trusted bool
|
|
want bool
|
|
}{
|
|
{"https://example.test/a.js", false, true},
|
|
{"https://WWW.example.test/b.css", false, true},
|
|
{"https://example.test:8443/x", false, true},
|
|
{"data:image/png;base64,AAAA", false, true},
|
|
{"blob:https://example.test/uuid", false, true},
|
|
{"http://example.test/a.js", false, false},
|
|
{"http://example.test/a.js", true, true},
|
|
{"https://cdn.other.test/lib.js", false, false},
|
|
{"https://example.test.evil.test/x", false, false},
|
|
{"https://user:pw@example.test/x", false, false},
|
|
{"ws://example.test/socket", false, false},
|
|
{"file:///etc/passwd", false, false},
|
|
{"javascript:alert(1)", true, false},
|
|
{"://bad", false, false},
|
|
} {
|
|
if got := allowRequest(tc.url, hosts, tc.trusted); got != tc.want {
|
|
t.Errorf("allowRequest(%q, trusted=%v) = %v, want %v", tc.url, tc.trusted, got, tc.want)
|
|
}
|
|
}
|
|
if !allowRequest("https://[::1]/x", []string{"[::1]"}, false) {
|
|
t.Fatal("a bracketed IPv6 host must match")
|
|
}
|
|
}
|
|
|
|
// TestResolverRules pins Chrome's --host-resolver-rules: one MAP per
|
|
// allowed host, sorted, to its first IPv4 address (else the first one, in
|
|
// brackets for IPv6), a host without addresses left out, and MAP *
|
|
// ~NOTFOUND last, since Chrome takes the first matching rule.
|
|
func TestResolverRules(t *testing.T) {
|
|
got := resolverRules(map[string][]netip.Addr{
|
|
"www.example.test": {netip.MustParseAddr("2001:db8::1")},
|
|
"example.test": {netip.MustParseAddr("2001:db8::2"), netip.MustParseAddr("192.0.2.10"), netip.MustParseAddr("192.0.2.11")},
|
|
"empty.test": nil,
|
|
})
|
|
want := "MAP example.test 192.0.2.10, MAP www.example.test [2001:db8::1], MAP * ~NOTFOUND"
|
|
if got != want {
|
|
t.Fatalf("rules = %q\nwant %q", got, want)
|
|
}
|
|
if got := resolverRules(nil); got != "MAP * ~NOTFOUND" {
|
|
t.Fatalf("no hosts = %q", got)
|
|
}
|
|
if !strings.HasSuffix(got, "MAP * ~NOTFOUND") || strings.Count(got, "~NOTFOUND") != 1 {
|
|
t.Fatal("the catch-all must come once and last")
|
|
}
|
|
}
|
|
|
|
// TestNewRenderer pins the comparison's start: no renderer and no error
|
|
// without a Chrome path (skipped), an error for a relative path, a
|
|
// missing file, a directory or a file that is not executable (failed).
|
|
func TestNewRenderer(t *testing.T) {
|
|
site, _ := url.Parse("https://example.test")
|
|
cfg := func(path string) *compass.Config {
|
|
c, err := compass.Open(compass.Options{Dir: t.TempDir()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if path != "" {
|
|
if err := c.Set(ChromePathConfigKey, path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return c
|
|
}
|
|
if r, err := NewRenderer(context.Background(), cfg(""), site, false, "ua"); r != nil || err != nil {
|
|
t.Fatalf("no path: %v %v, want skipped", r, err)
|
|
}
|
|
if r, err := NewRenderer(context.Background(), nil, site, false, "ua"); r != nil || err != nil {
|
|
t.Fatal("no config: skipped")
|
|
}
|
|
dir := t.TempDir()
|
|
plain := filepath.Join(dir, "chrome.txt")
|
|
if err := os.WriteFile(plain, []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, path := range map[string]string{"relative": "bin/chrome", "missing": filepath.Join(dir, "nope"), "directory": dir, "not executable": plain} {
|
|
r, err := NewRenderer(context.Background(), cfg(path), site, false, "ua")
|
|
if r != nil || err == nil || !strings.Contains(err.Error(), "Chrome path") {
|
|
t.Errorf("%s: %v %v, want a Chrome path error", name, r, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRenderMax pins the page cap: the default 200, a positive config
|
|
// value, and the default for 0 or less.
|
|
func TestRenderMax(t *testing.T) {
|
|
if RenderMax(nil) != DefaultRenderMax || DefaultRenderMax != 200 {
|
|
t.Fatal("default cap")
|
|
}
|
|
for v, want := range map[int]int{50: 50, 0: DefaultRenderMax, -1: DefaultRenderMax} {
|
|
c, _ := compass.Open(compass.Options{Dir: t.TempDir()})
|
|
_ = c.Set(RenderMaxConfigKey, v)
|
|
if got := RenderMax(c); got != want {
|
|
t.Errorf("RenderMax(%d) = %d, want %d", v, got, want)
|
|
}
|
|
}
|
|
if got := SiteHosts("WWW.Example.test"); len(got) != 2 || got[0] != "www.example.test" || got[1] != "example.test" {
|
|
t.Fatalf("SiteHosts = %v", got)
|
|
}
|
|
if got := SiteHosts("example.test"); got[1] != "www.example.test" || SiteHosts(" ") != nil {
|
|
t.Fatalf("SiteHosts = %v", got)
|
|
}
|
|
}
|