fix: one decoded path form for crawl keys and CMS lookups

Crawl keys, the rendered comparison and the link graph use the decoded
path (classes.PathKey), the form the resolver, the descriptors and the
SEO entries use; the request URL is percent-encoded again
(classes.EscapeKeyPath). A non-ASCII address now matches its CMS record
and its stored links. A path that does not decode to text keeps its
encoded form.

classes/crawl/fetch.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"git.golem15.com/golem15/sm-seo-plugin/classes"
 	"git.golem15.com/golem15/summercms/modules/compass"
 	"git.golem15.com/golem15/summercms/modules/fetchguard"
 )
@@ -168,25 +169,30 @@ func (f *Fetcher) Allowed(u *url.URL) bool {
 	return false
 }
 
-// Key is the address of u on the site: its path with the query, or ""
-// when u is not on the site (another host, scheme or port).
+// Key is the address of u on the site: its path in the decoded form of
+// classes.PathKey with the raw query, or "" when u is not on the site
+// (another host, scheme or port). The crawl's keys, the rendered
+// comparison, the CMS lookups and the link graph use this one form.
 func (f *Fetcher) Key(u *url.URL) string {
 	if !f.Allowed(u) {
 		return ""
 	}
-	p := u.EscapedPath()
-	if p == "" {
-		p = "/"
-	}
+	p := classes.PathKey(u)
 	if u.RawQuery != "" {
 		p += "?" + u.RawQuery
 	}
 	return p
 }
 
-// Absolute returns the site URL of an address key.
+// Absolute returns the site URL of an address key, its path
+// percent-encoded for the request (classes.EscapeKeyPath).
 func (f *Fetcher) Absolute(key string) string {
-	return f.site.Scheme + "://" + f.site.Host + key
+	p, q, hasQuery := strings.Cut(key, "?")
+	out := f.site.Scheme + "://" + f.site.Host + classes.EscapeKeyPath(p)
+	if hasQuery {
+		out += "?" + q
+	}
+	return out
 }
 
 // Get fetches one address key of the site.

classes/crawl/fetch_test.go

@@ -41,6 +41,32 @@ func TestFetcherPortAndKey(t *testing.T) {
 			t.Errorf("Allowed(%s) = %v", raw, f.Allowed(u))
 		}
 	}
+	// One decoded path form (WR-04, P33-61): the key is decoded, the
+	// request URL encoded; a path that does not decode to text keeps its
+	// encoded form, and so does one holding "%" or "?".
+	for raw, want := range map[string]string{
+		"https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1": "/blog/zażółć?a=1",
+		"https://site.test/blog/zażółć":                         "/blog/zażółć",
+		"https://site.test/%FF":                                 "/%FF",
+		"https://site.test/a%00b":                               "/a%00b",
+		"https://site.test/100%25":                              "/100%25",
+		"https://site.test/a%3Fb?c=1":                           "/a%3Fb?c=1",
+	} {
+		if got := f.Key(parse(raw)); got != want {
+			t.Errorf("Key(%s) = %q, want %q", raw, got, want)
+		}
+	}
+	for key, want := range map[string]string{
+		"/blog/zażółć?a=1": "https://site.test/blog/za%C5%BC%C3%B3%C5%82%C4%87?a=1",
+		"/%FF":             "https://site.test/%FF",
+		"/100%25":          "https://site.test/100%25",
+		"/a b":             "https://site.test/a%20b",
+		"/":                "https://site.test/",
+	} {
+		if got := f.Absolute(key); got != want {
+			t.Errorf("Absolute(%q) = %q, want %q", key, got, want)
+		}
+	}
 	local, err := NewFetcher("http://localhost:3000", nil)
 	if err != nil {
 		t.Fatal(err)

classes/linkgraph.go

@@ -239,25 +239,51 @@ func (s *Service) LinkSuggestions(ctx context.Context, db *gorm.DB, path string,
 	return out, nil
 }
 
-// sitePath is the path of a path or an absolute URL; ErrBadURL otherwise.
+// sitePath is the crawl key of a path or an absolute URL (PathKey with
+// the raw query); ErrBadURL otherwise. A path and the absolute URL of the
+// same page give the same answer, written decoded or percent-encoded.
 func sitePath(raw string) (string, error) {
 	raw = strings.TrimSpace(raw)
 	if strings.HasPrefix(raw, "/") && !strings.HasPrefix(raw, "//") {
-		path, _, _ := strings.Cut(raw, "#")
-		return path, nil
+		u, err := url.Parse(raw)
+		if err != nil {
+			return "", ErrBadURL
+		}
+		return joinQuery(PathKey(u), u.RawQuery), nil
 	}
 	u, err := url.Parse(raw)
 	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
 		return "", ErrBadURL
 	}
-	path := u.EscapedPath()
-	if path == "" {
-		path = "/"
+	return joinQuery(PathKey(u), u.RawQuery), nil
+}
+
+// PathKey is the path form of the crawl's addresses (P33-61), the form
+// the resolver, the descriptors and the SEO entries use: u's decoded path
+// ("/" when empty). A path whose decoded form is not valid UTF-8 without
+// NUL, or holds "%" or "?" (which would read as an escape or a query),
+// keeps its percent-encoded form.
+func PathKey(u *url.URL) string {
+	if u == nil {
+		return "/"
 	}
-	if u.RawQuery != "" {
-		path += "?" + u.RawQuery
+	p := u.Path
+	if !safeText(p) || strings.ContainsAny(p, "%?") {
+		p = u.EscapedPath()
 	}
-	return path, nil
+	if p == "" {
+		p = "/"
+	}
+	return p
+}
+
+// EscapeKeyPath is the request form of a PathKey path: a decoded path is
+// percent-encoded, an encoded one (it holds "%") is kept.
+func EscapeKeyPath(p string) string {
+	if strings.Contains(p, "%") {
+		return p
+	}
+	return (&url.URL{Path: p}).EscapedPath()
 }
 
 // LinksOfURL is the internal links of a page addressed by URL (a path or