Jakub Zych
c42bc265c8
A url item stored as javascript://%0aalert(1)%0a//x passed Url::to and reached <loc>, which sitemap.xsl renders as a link on the site origin. - The definitions form refuses, per row, a url that is not a path or an http(s) URL (classes.AllowedItemURL) and a urlPattern that is not a path (signpost.ValidItemPattern), keyed data.<i>.url / .urlPattern. - Generator.addItem drops a loc or alternate that is not http(s). - sitemap:import-winter drops such url items and clears such patterns with a report line; sitemap:import refuses such an entry.
567 lines
15 KiB
Go
567 lines
15 KiB
Go
package classes
|
|
|
|
import (
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// This file ports the parts of WinterCMS's URL generator the PHP sitemap
|
|
// called, so the Go sitemap writes the same bytes: Laravel's Url::to plus
|
|
// Winter's UrlGenerator::buildUrl, PHP's parse_url, rawurlencode,
|
|
// rawurldecode and urldecode, and FILTER_VALIDATE_URL. It is deliberately
|
|
// hand written: net/url escapes differently. The golden table
|
|
// testdata/urlto.golden, generated from the vendored Winter router, pins
|
|
// it.
|
|
|
|
// validURLPrefix is Laravel UrlGenerator::isValidUrl's fast path.
|
|
var validURLPrefix = regexp.MustCompile(`^(#|//|https?://|(mailto|tel|sms):)`)
|
|
|
|
// URLTo is Winter's Url::to(path) for a request whose root URL is root
|
|
// (scheme and host, plus a base path if any, without a trailing slash). A
|
|
// valid URL is kept; anything else becomes root plus "/" plus the path
|
|
// trimmed of slashes, the query string appended as it is. The result then
|
|
// goes through BuildURL, which re-encodes every path segment, so "/" is the
|
|
// root with its slash, "/blog/" loses its trailing slash and ":" becomes
|
|
// "%3A".
|
|
func URLTo(root, path string) string {
|
|
if isValidURL(path) {
|
|
return BuildURL(path)
|
|
}
|
|
p, query := path, ""
|
|
if i := strings.IndexByte(path, '?'); i >= 0 {
|
|
p, query = path[:i], path[i:]
|
|
}
|
|
inner := strings.Trim(p+"/", "/")
|
|
return BuildURL(strings.Trim(root+"/"+inner, "/") + query)
|
|
}
|
|
|
|
// isValidURL is Laravel UrlGenerator::isValidUrl.
|
|
func isValidURL(path string) bool {
|
|
if validURLPrefix.MatchString(path) {
|
|
return true
|
|
}
|
|
return filterValidateURL(path)
|
|
}
|
|
|
|
// filterURLChars is the character set PHP's FILTER_VALIDATE_URL accepts;
|
|
// a value with any other byte fails.
|
|
const filterURLChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789$-_.+!*'(),{}|\\^~[]`<>#%\";/?:@&="
|
|
|
|
// filterValidateURL approximates PHP filter_var(FILTER_VALIDATE_URL): only
|
|
// URL characters, a scheme, a host unless the scheme is mailto, news or
|
|
// file, and a valid host name for http and https.
|
|
func filterValidateURL(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for i := 0; i < len(s); i++ {
|
|
if !strings.ContainsRune(filterURLChars, rune(s[i])) {
|
|
return false
|
|
}
|
|
}
|
|
u := parseURL(s)
|
|
if !u.has(partScheme) || u.scheme == "" {
|
|
return false
|
|
}
|
|
scheme := strings.ToLower(u.scheme)
|
|
if !u.has(partHost) || u.host == "" {
|
|
return scheme == "mailto" || scheme == "news" || scheme == "file"
|
|
}
|
|
if scheme == "http" || scheme == "https" {
|
|
return validHostName(u.host)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// validHostName reports whether host is dot-separated labels of letters,
|
|
// digits and inner hyphens.
|
|
func validHostName(host string) bool {
|
|
host = strings.TrimSuffix(host, ".")
|
|
if host == "" || len(host) > 253 {
|
|
return false
|
|
}
|
|
for _, label := range strings.Split(host, ".") {
|
|
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
|
return false
|
|
}
|
|
for i := 0; i < len(label); i++ {
|
|
c := label[i]
|
|
if !(isAlnum(c) || c == '-') {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// The parts a parsed URL may hold, as PHP parse_url's array keys.
|
|
const (
|
|
partScheme = 1 << iota
|
|
partHost
|
|
partUser
|
|
partPass
|
|
partPort
|
|
partPath
|
|
partQuery
|
|
partFragment
|
|
)
|
|
|
|
// phpURL is the array PHP parse_url returns; set records which keys exist.
|
|
type phpURL struct {
|
|
set int
|
|
scheme, host, user, pass, port, path, query, frag string
|
|
}
|
|
|
|
func (u phpURL) has(part int) bool { return u.set&part != 0 }
|
|
|
|
// parseURL ports PHP parse_url for the URL shapes a sitemap meets: an
|
|
// optional scheme, an authority after "//", a path, a query and a fragment.
|
|
// Control characters in a part are replaced by "_", as PHP does.
|
|
func parseURL(s string) phpURL {
|
|
var u phpURL
|
|
rest := s
|
|
if i := strings.IndexByte(rest, ':'); i > 0 && validScheme(rest[:i]) {
|
|
after := rest[i+1:]
|
|
if !strings.HasPrefix(after, "//") && portOnly(after) {
|
|
// "host:port/path" without a scheme.
|
|
} else {
|
|
u.scheme, u.set = rest[:i], u.set|partScheme
|
|
rest = after
|
|
}
|
|
}
|
|
if strings.HasPrefix(rest, "//") {
|
|
rest = rest[2:]
|
|
end := strings.IndexAny(rest, "/?#")
|
|
authority := rest
|
|
if end >= 0 {
|
|
authority, rest = rest[:end], rest[end:]
|
|
} else {
|
|
rest = ""
|
|
}
|
|
u.parseAuthority(authority)
|
|
}
|
|
if i := strings.IndexByte(rest, '#'); i >= 0 {
|
|
u.frag, u.set = rest[i+1:], u.set|partFragment
|
|
rest = rest[:i]
|
|
}
|
|
if i := strings.IndexByte(rest, '?'); i >= 0 {
|
|
u.query, u.set = rest[i+1:], u.set|partQuery
|
|
rest = rest[:i]
|
|
}
|
|
if rest != "" {
|
|
u.path, u.set = rest, u.set|partPath
|
|
}
|
|
u.scheme = replaceControl(u.scheme)
|
|
u.host = replaceControl(u.host)
|
|
u.user = replaceControl(u.user)
|
|
u.pass = replaceControl(u.pass)
|
|
u.path = replaceControl(u.path)
|
|
u.query = replaceControl(u.query)
|
|
u.frag = replaceControl(u.frag)
|
|
return u
|
|
}
|
|
|
|
func (u *phpURL) parseAuthority(a string) {
|
|
if i := strings.LastIndexByte(a, '@'); i >= 0 {
|
|
userinfo := a[:i]
|
|
a = a[i+1:]
|
|
if j := strings.IndexByte(userinfo, ':'); j >= 0 {
|
|
u.user, u.pass = userinfo[:j], userinfo[j+1:]
|
|
u.set |= partUser | partPass
|
|
} else {
|
|
u.user = userinfo
|
|
u.set |= partUser
|
|
}
|
|
}
|
|
if i := strings.LastIndexByte(a, ':'); i >= 0 && !strings.HasSuffix(a, "]") {
|
|
port := a[i+1:]
|
|
if port != "" && allDigits(port) {
|
|
u.port, u.set = port, u.set|partPort
|
|
}
|
|
a = a[:i]
|
|
}
|
|
if a != "" {
|
|
u.host, u.set = a, u.set|partHost
|
|
}
|
|
}
|
|
|
|
func validScheme(s string) bool {
|
|
for i := 0; i < len(s); i++ {
|
|
c := s[i]
|
|
if !(isAlnum(c) || c == '+' || c == '-' || c == '.') {
|
|
return false
|
|
}
|
|
}
|
|
return s != ""
|
|
}
|
|
|
|
// portOnly reports whether s is a port number up to the first "/", the
|
|
// case PHP parse_url reads as host:port rather than scheme:path.
|
|
func portOnly(s string) bool {
|
|
end := strings.IndexByte(s, '/')
|
|
if end < 0 {
|
|
end = len(s)
|
|
}
|
|
return end > 0 && end <= 5 && allDigits(s[:end])
|
|
}
|
|
|
|
func allDigits(s string) bool {
|
|
for i := 0; i < len(s); i++ {
|
|
if s[i] < '0' || s[i] > '9' {
|
|
return false
|
|
}
|
|
}
|
|
return s != ""
|
|
}
|
|
|
|
func isAlnum(c byte) bool {
|
|
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9'
|
|
}
|
|
|
|
func replaceControl(s string) string {
|
|
if strings.IndexFunc(s, func(r rune) bool { return r < 0x20 || r == 0x7f }) < 0 {
|
|
return s
|
|
}
|
|
b := []byte(s)
|
|
for i, c := range b {
|
|
if c < 0x20 || c == 0x7f {
|
|
b[i] = '_'
|
|
}
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// phpTrim is PHP trim() with its default character list.
|
|
func phpTrim(s string) string {
|
|
return strings.Trim(s, " \t\n\r\x00\x0b")
|
|
}
|
|
|
|
// singleColonSchemes are written "scheme:" instead of "scheme://".
|
|
var singleColonSchemes = map[string]bool{"mailto": true, "tel": true, "sms": true}
|
|
|
|
// defaultPorts are Winter UrlGenerator::isDefaultPort's known ports.
|
|
var defaultPorts = map[string]int{
|
|
"ftp": 21, "ssh": 22, "sftp": 22, "telnet": 23, "smtp": 25, "dns": 53,
|
|
"gopher": 70, "http": 80, "pop3": 110, "imap": 143, "ldap": 389,
|
|
"https": 443, "smtps": 465, "ldaps": 636, "ftps": 990, "imaps": 993,
|
|
"pop3s": 995,
|
|
}
|
|
|
|
// BuildURL is Winter UrlGenerator::buildUrl($url) with no replacements: the
|
|
// query is URL-decoded, the URL parsed, every path segment re-encoded with
|
|
// rawurlencode(rawurldecode()) after "." and ".." are resolved, the query
|
|
// rebuilt with buildStr and the fragment re-encoded.
|
|
func BuildURL(raw string) string {
|
|
return buildParts(parseURL(decodeQueryParams(raw)))
|
|
}
|
|
|
|
// replacePath is buildUrl($url, ['path' => path]).
|
|
func replacePath(raw, path string) string {
|
|
u := parseURL(decodeQueryParams(raw))
|
|
u.path, u.set = path, u.set|partPath
|
|
return buildParts(u)
|
|
}
|
|
|
|
// decodeQueryParams URL-decodes everything after the first "?".
|
|
func decodeQueryParams(s string) string {
|
|
if i := strings.IndexByte(s, '?'); i >= 0 {
|
|
return s[:i] + "?" + urldecode(s[i+1:])
|
|
}
|
|
return s
|
|
}
|
|
|
|
func buildParts(u phpURL) string {
|
|
u.scheme, u.host, u.user, u.pass = phpTrim(u.scheme), phpTrim(u.host), phpTrim(u.user), phpTrim(u.pass)
|
|
u.path, u.query, u.frag = phpTrim(u.path), phpTrim(u.query), phpTrim(u.frag)
|
|
port := 0
|
|
if u.has(partPort) {
|
|
if n, err := strconv.Atoi(phpTrim(u.port)); err == nil && n >= 1 && n <= 65535 {
|
|
port = n
|
|
}
|
|
}
|
|
var b strings.Builder
|
|
hasScheme := u.has(partScheme) && u.scheme != ""
|
|
if hasScheme {
|
|
b.WriteString(u.scheme)
|
|
if singleColonSchemes[u.scheme] {
|
|
b.WriteString(":")
|
|
} else {
|
|
b.WriteString("://")
|
|
}
|
|
}
|
|
if u.has(partUser) && u.user != "" {
|
|
b.WriteString(u.user)
|
|
if u.has(partPass) && u.pass != "" {
|
|
b.WriteString(":" + u.pass)
|
|
}
|
|
b.WriteString("@")
|
|
}
|
|
if u.has(partHost) && u.host != "" {
|
|
b.WriteString(u.host)
|
|
}
|
|
if port != 0 {
|
|
if def, ok := defaultPorts[strings.ToLower(u.scheme)]; !ok || def != port {
|
|
b.WriteString(":" + strconv.Itoa(port))
|
|
}
|
|
}
|
|
path := ""
|
|
if u.has(partPath) && u.path != "" {
|
|
var kept []string
|
|
for _, seg := range strings.Split(u.path, "/") {
|
|
switch {
|
|
case seg == ".":
|
|
case seg == ".." && len(kept) > 0:
|
|
kept = kept[:len(kept)-1]
|
|
default:
|
|
kept = append(kept, seg)
|
|
}
|
|
}
|
|
for i, seg := range kept {
|
|
kept[i] = rawurlencode(rawurldecode(seg))
|
|
}
|
|
path = strings.Join(kept, "/")
|
|
}
|
|
if hasScheme && !singleColonSchemes[u.scheme] {
|
|
if !strings.HasPrefix(path, "/") {
|
|
b.WriteString("/")
|
|
}
|
|
b.WriteString(path)
|
|
} else {
|
|
b.WriteString(path)
|
|
}
|
|
if u.has(partQuery) && u.query != "" {
|
|
b.WriteString("?" + buildStr(parseQueryPairs(u.query), ""))
|
|
}
|
|
if u.has(partFragment) && u.frag != "" {
|
|
b.WriteString("#" + rawurlencode(rawurldecode(u.frag)))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// phpKey is a PHP array key: an int or a string.
|
|
type phpKey struct {
|
|
isInt bool
|
|
i int64
|
|
s string
|
|
}
|
|
|
|
func (k phpKey) String() string {
|
|
if k.isInt {
|
|
return strconv.FormatInt(k.i, 10)
|
|
}
|
|
return k.s
|
|
}
|
|
|
|
// keyOf is PHP's array key cast of a string: a canonical decimal integer
|
|
// becomes an int key.
|
|
func keyOf(s string) phpKey {
|
|
if n, err := strconv.ParseInt(s, 10, 64); err == nil && strconv.FormatInt(n, 10) == s {
|
|
return phpKey{isInt: true, i: n}
|
|
}
|
|
return phpKey{s: s}
|
|
}
|
|
|
|
// phpArray is an ordered PHP array of strings, nested arrays or nulls.
|
|
type phpArray struct {
|
|
keys []phpKey
|
|
vals []any
|
|
next int64
|
|
}
|
|
|
|
func (a *phpArray) index(k phpKey) int {
|
|
for i, have := range a.keys {
|
|
if have == k {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (a *phpArray) set(k phpKey, v any) {
|
|
if i := a.index(k); i >= 0 {
|
|
a.vals[i] = v
|
|
return
|
|
}
|
|
a.keys = append(a.keys, k)
|
|
a.vals = append(a.vals, v)
|
|
if k.isInt && k.i >= a.next {
|
|
a.next = k.i + 1
|
|
}
|
|
}
|
|
|
|
func (a *phpArray) push(v any) {
|
|
a.set(phpKey{isInt: true, i: a.next}, v)
|
|
}
|
|
|
|
// parseQueryPairs is buildUrl's split of a string query into
|
|
// $queryParams.
|
|
func parseQueryPairs(query string) *phpArray {
|
|
params := &phpArray{}
|
|
for _, pair := range strings.Split(query, "&") {
|
|
key, value := pair, pair
|
|
if i := strings.IndexByte(pair, '='); i >= 0 {
|
|
key, value = pair[:i], pair[i+1:]
|
|
}
|
|
if !strings.Contains(pair, "=") {
|
|
params.push(key)
|
|
continue
|
|
}
|
|
if strings.Contains(urldecode(pair), "[") {
|
|
k := keyOf(key)
|
|
var inner *phpArray
|
|
if i := params.index(k); i >= 0 {
|
|
inner, _ = params.vals[i].(*phpArray)
|
|
}
|
|
if inner == nil {
|
|
inner = &phpArray{}
|
|
}
|
|
inner.push(value)
|
|
params.set(k, inner)
|
|
continue
|
|
}
|
|
params.set(keyOf(key), value)
|
|
}
|
|
return params
|
|
}
|
|
|
|
// buildStr is Winter UrlGenerator::buildStr.
|
|
func buildStr(query *phpArray, prefix string) string {
|
|
var out []string
|
|
for i, k := range query.keys {
|
|
v := query.vals[i]
|
|
var key string
|
|
if k.isInt && k.i == int64(i) {
|
|
if prefix != "" {
|
|
if strings.HasSuffix(prefix, "]") {
|
|
key = prefix
|
|
} else {
|
|
key = prefix + "[]"
|
|
}
|
|
} else {
|
|
key, _ = v.(string)
|
|
v = nil
|
|
}
|
|
} else if prefix != "" {
|
|
key = prefix + "[" + k.String() + "]"
|
|
} else {
|
|
key = k.String()
|
|
}
|
|
if nested, ok := v.(*phpArray); ok {
|
|
out = append(out, buildStr(nested, key))
|
|
continue
|
|
}
|
|
part := rawurlencode(rawurldecode(key))
|
|
if s, ok := v.(string); ok {
|
|
part += "=" + rawurlencode(rawurldecode(s))
|
|
}
|
|
out = append(out, part)
|
|
}
|
|
return strings.Join(out, "&")
|
|
}
|
|
|
|
// rawurlencode is PHP rawurlencode: every byte except A-Z a-z 0-9 - _ . ~
|
|
// becomes %XX with upper-case hex.
|
|
func rawurlencode(s string) string {
|
|
const hex = "0123456789ABCDEF"
|
|
var b strings.Builder
|
|
for i := 0; i < len(s); i++ {
|
|
c := s[i]
|
|
if isAlnum(c) || c == '-' || c == '_' || c == '.' || c == '~' {
|
|
b.WriteByte(c)
|
|
continue
|
|
}
|
|
b.WriteByte('%')
|
|
b.WriteByte(hex[c>>4])
|
|
b.WriteByte(hex[c&15])
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// rawurldecode is PHP rawurldecode: %XX becomes its byte, anything else,
|
|
// including a malformed escape, stays.
|
|
func rawurldecode(s string) string { return percentDecode(s, false) }
|
|
|
|
// urldecode is PHP urldecode: rawurldecode that also turns "+" into a
|
|
// space.
|
|
func urldecode(s string) string { return percentDecode(s, true) }
|
|
|
|
func percentDecode(s string, plus bool) string {
|
|
if !strings.ContainsAny(s, "%+") {
|
|
return s
|
|
}
|
|
var b strings.Builder
|
|
for i := 0; i < len(s); i++ {
|
|
c := s[i]
|
|
if c == '+' && plus {
|
|
b.WriteByte(' ')
|
|
continue
|
|
}
|
|
if c == '%' && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]) {
|
|
b.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2]))
|
|
i += 2
|
|
continue
|
|
}
|
|
b.WriteByte(c)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func isHex(c byte) bool {
|
|
return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F'
|
|
}
|
|
|
|
func unhex(c byte) byte {
|
|
switch {
|
|
case '0' <= c && c <= '9':
|
|
return c - '0'
|
|
case 'a' <= c && c <= 'f':
|
|
return c - 'a' + 10
|
|
default:
|
|
return c - 'A' + 10
|
|
}
|
|
}
|
|
|
|
// URLPath is the path PHP parse_url($url, PHP_URL_PATH) returns, "" when
|
|
// the URL has none.
|
|
func URLPath(raw string) string {
|
|
u := parseURL(raw)
|
|
if !u.has(partPath) {
|
|
return ""
|
|
}
|
|
return u.path
|
|
}
|
|
|
|
// schemePrefix matches a URL that starts with a scheme and its colon.
|
|
var schemePrefix = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`)
|
|
|
|
// AllowedItemURL reports whether a url item's address may be stored: empty,
|
|
// a path (relative or starting with a single "/"), or an absolute http or
|
|
// https URL. Any other scheme (javascript:, data:, mailto: and the like), a
|
|
// protocol-relative "//host" or "/\host", a bare fragment, surrounding
|
|
// whitespace and control characters are refused. Url::to keeps any URL
|
|
// FILTER_VALIDATE_URL accepts, so without this a javascript: URL reached
|
|
// <loc>, which sitemap.xsl renders as a link.
|
|
func AllowedItemURL(u string) bool {
|
|
if u == "" {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(u) != u || strings.ContainsFunc(u, func(r rune) bool { return r < 0x20 || r == 0x7f }) {
|
|
return false
|
|
}
|
|
if httpURL(u) {
|
|
return true
|
|
}
|
|
if schemePrefix.MatchString(u) || u[0] == '#' || u[0] == '\\' {
|
|
return false
|
|
}
|
|
return !strings.HasPrefix(u, "//") && !strings.HasPrefix(u, `/\`)
|
|
}
|
|
|
|
// httpURL reports whether u starts with http:// or https://
|
|
// (case-insensitive).
|
|
func httpURL(u string) bool {
|
|
lower := strings.ToLower(u)
|
|
return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
|
|
}
|