package crawl import ( "net/url" "testing" ) // TestFetcherPortAndKey pins L3 WR-03 (and WR-04, P33-61): the fetcher // accepts an address only on the site's effective port, so a link to // another port is neither fetched nor reported as a page of the site. func TestFetcherPortAndKey(t *testing.T) { f, err := NewFetcher("https://site.test", nil) if err != nil { t.Fatal(err) } if f.Port() != "443" { t.Fatalf("Port = %q", f.Port()) } parse := func(raw string) *url.URL { u, err := url.Parse(raw) if err != nil { t.Fatal(err) } return u } for raw, want := range map[string]string{ "https://site.test:8443/x": "", "https://site.test:443/x": "/x", "https://www.site.test/x": "/x", "https://site.test:80/x": "", "http://site.test/x": "", "https://other.test/x": "", "https://site.test": "/", "https://site.test/a?b=1&c=": "/a?b=1&c=", } { u := parse(raw) if got := f.Key(u); got != want { t.Errorf("Key(%s) = %q, want %q", raw, got, want) } if f.Allowed(u) != (want != "") { t.Errorf("Allowed(%s) = %v", raw, f.Allowed(u)) } } // One decoded path form (WR-04, P33-61): the key is decoded, the // request URL encoded; a path that does not decode to text keeps its // encoded form, and so does one holding "%" or "?". for raw, want := range map[string]string{ "https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1": "/blog/zażółć?a=1", "https://site.test/blog/zażółć": "/blog/zażółć", "https://site.test/%FF": "/%FF", "https://site.test/a%00b": "/a%00b", "https://site.test/100%25": "/100%25", "https://site.test/a%3Fb?c=1": "/a%3Fb?c=1", } { if got := f.Key(parse(raw)); got != want { t.Errorf("Key(%s) = %q, want %q", raw, got, want) } } for key, want := range map[string]string{ "/blog/zażółć?a=1": "https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1", "/%FF": "https://site.test/%FF", "/100%25": "https://site.test/100%25", "/a b": "https://site.test/a%20b", "/": "https://site.test/", } { if got := f.Absolute(key); got != want { t.Errorf("Absolute(%q) = %q, want %q", key, got, want) } } local, err := NewFetcher("http://localhost:3000", nil) if err != nil { t.Fatal(err) } if local.Port() != "3000" || effectivePort(parse("http://x")) != "80" || effectivePort(parse("ftp://x")) != "" || effectivePort(nil) != "" { t.Fatal("effectivePort is wrong") } }