package crawl import ( "net/url" "testing" ) // TestFetcherPortAndKey pins L3 WR-03 (and WR-04, P33-61): the fetcher // accepts an address only on the site's effective port, so a link to // another port is neither fetched nor reported as a page of the site. func TestFetcherPortAndKey(t *testing.T) { f, err := NewFetcher("https://site.test", nil) if err != nil { t.Fatal(err) } if f.Port() != "443" { t.Fatalf("Port = %q", f.Port()) } parse := func(raw string) *url.URL { u, err := url.Parse(raw) if err != nil { t.Fatal(err) } return u } for raw, want := range map[string]string{ "https://site.test:8443/x": "", "https://site.test:443/x": "/x", "https://www.site.test/x": "/x", "https://site.test:80/x": "", "http://site.test/x": "", "https://other.test/x": "", "https://site.test": "/", "https://site.test/a?b=1&c=": "/a?b=1&c=", } { u := parse(raw) if got := f.Key(u); got != want { t.Errorf("Key(%s) = %q, want %q", raw, got, want) } if f.Allowed(u) != (want != "") { t.Errorf("Allowed(%s) = %v", raw, f.Allowed(u)) } } local, err := NewFetcher("http://localhost:3000", nil) if err != nil { t.Fatal(err) } if local.Port() != "3000" || effectivePort(parse("http://x")) != "80" || effectivePort(parse("ftp://x")) != "" || effectivePort(nil) != "" { t.Fatal("effectivePort is wrong") } }