e2af5b44c8
Crawl keys, the rendered comparison and the link graph use the decoded path (classes.PathKey), the form the resolver, the descriptors and the SEO entries use; the request URL is percent-encoded again (classes.EscapeKeyPath). A non-ASCII address now matches its CMS record and its stored links. A path that does not decode to text keeps its encoded form.
78 lines
2.6 KiB
Go
78 lines
2.6 KiB
Go
package crawl
|
|
|
|
import (
|
|
"net/url"
|
|
"testing"
|
|
)
|
|
|
|
// TestFetcherPortAndKey pins L3 WR-03 (and WR-04, P33-61): the fetcher
|
|
// accepts an address only on the site's effective port, so a link to
|
|
// another port is neither fetched nor reported as a page of the site.
|
|
func TestFetcherPortAndKey(t *testing.T) {
|
|
f, err := NewFetcher("https://site.test", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.Port() != "443" {
|
|
t.Fatalf("Port = %q", f.Port())
|
|
}
|
|
parse := func(raw string) *url.URL {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return u
|
|
}
|
|
for raw, want := range map[string]string{
|
|
"https://site.test:8443/x": "",
|
|
"https://site.test:443/x": "/x",
|
|
"https://www.site.test/x": "/x",
|
|
"https://site.test:80/x": "",
|
|
"http://site.test/x": "",
|
|
"https://other.test/x": "",
|
|
"https://site.test": "/",
|
|
"https://site.test/a?b=1&c=": "/a?b=1&c=",
|
|
} {
|
|
u := parse(raw)
|
|
if got := f.Key(u); got != want {
|
|
t.Errorf("Key(%s) = %q, want %q", raw, got, want)
|
|
}
|
|
if f.Allowed(u) != (want != "") {
|
|
t.Errorf("Allowed(%s) = %v", raw, f.Allowed(u))
|
|
}
|
|
}
|
|
// One decoded path form (WR-04, P33-61): the key is decoded, the
|
|
// request URL encoded; a path that does not decode to text keeps its
|
|
// encoded form, and so does one holding "%" or "?".
|
|
for raw, want := range map[string]string{
|
|
"https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1": "/blog/zażółć?a=1",
|
|
"https://site.test/blog/zażółć": "/blog/zażółć",
|
|
"https://site.test/%FF": "/%FF",
|
|
"https://site.test/a%00b": "/a%00b",
|
|
"https://site.test/100%25": "/100%25",
|
|
"https://site.test/a%3Fb?c=1": "/a%3Fb?c=1",
|
|
} {
|
|
if got := f.Key(parse(raw)); got != want {
|
|
t.Errorf("Key(%s) = %q, want %q", raw, got, want)
|
|
}
|
|
}
|
|
for key, want := range map[string]string{
|
|
"/blog/zażółć?a=1": "https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1",
|
|
"/%FF": "https://site.test/%FF",
|
|
"/100%25": "https://site.test/100%25",
|
|
"/a b": "https://site.test/a%20b",
|
|
"/": "https://site.test/",
|
|
} {
|
|
if got := f.Absolute(key); got != want {
|
|
t.Errorf("Absolute(%q) = %q, want %q", key, got, want)
|
|
}
|
|
}
|
|
local, err := NewFetcher("http://localhost:3000", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if local.Port() != "3000" || effectivePort(parse("http://x")) != "80" || effectivePort(parse("ftp://x")) != "" || effectivePort(nil) != "" {
|
|
t.Fatal("effectivePort is wrong")
|
|
}
|
|
}
|