package classes import ( "regexp" "strconv" "strings" ) // This file ports the parts of WinterCMS's URL generator the PHP sitemap // called, so the Go sitemap writes the same bytes: Laravel's Url::to plus // Winter's UrlGenerator::buildUrl, PHP's parse_url, rawurlencode, // rawurldecode and urldecode, and FILTER_VALIDATE_URL. It is deliberately // hand written: net/url escapes differently. The golden table // testdata/urlto.golden, generated from the vendored Winter router, pins // it. // validURLPrefix is Laravel UrlGenerator::isValidUrl's fast path. var validURLPrefix = regexp.MustCompile(`^(#|//|https?://|(mailto|tel|sms):)`) // URLTo is Winter's Url::to(path) for a request whose root URL is root // (scheme and host, plus a base path if any, without a trailing slash). A // valid URL is kept; anything else becomes root plus "/" plus the path // trimmed of slashes, the query string appended as it is. The result then // goes through BuildURL, which re-encodes every path segment, so "/" is the // root with its slash, "/blog/" loses its trailing slash and ":" becomes // "%3A". func URLTo(root, path string) string { if isValidURL(path) { return BuildURL(path) } p, query := path, "" if i := strings.IndexByte(path, '?'); i >= 0 { p, query = path[:i], path[i:] } inner := strings.Trim(p+"/", "/") return BuildURL(strings.Trim(root+"/"+inner, "/") + query) } // isValidURL is Laravel UrlGenerator::isValidUrl. func isValidURL(path string) bool { if validURLPrefix.MatchString(path) { return true } return filterValidateURL(path) } // filterURLChars is the character set PHP's FILTER_VALIDATE_URL accepts; // a value with any other byte fails. const filterURLChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789$-_.+!*'(),{}|\\^~[]`<>#%\";/?:@&=" // filterValidateURL approximates PHP filter_var(FILTER_VALIDATE_URL): only // URL characters, a scheme, a host unless the scheme is mailto, news or // file, and a valid host name for http and https. func filterValidateURL(s string) bool { if s == "" { return false } for i := 0; i < len(s); i++ { if !strings.ContainsRune(filterURLChars, rune(s[i])) { return false } } u := parseURL(s) if !u.has(partScheme) || u.scheme == "" { return false } scheme := strings.ToLower(u.scheme) if !u.has(partHost) || u.host == "" { return scheme == "mailto" || scheme == "news" || scheme == "file" } if scheme == "http" || scheme == "https" { return validHostName(u.host) } return true } // validHostName reports whether host is dot-separated labels of letters, // digits and inner hyphens. func validHostName(host string) bool { host = strings.TrimSuffix(host, ".") if host == "" || len(host) > 253 { return false } for _, label := range strings.Split(host, ".") { if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { return false } for i := 0; i < len(label); i++ { c := label[i] if !(isAlnum(c) || c == '-') { return false } } } return true } // The parts a parsed URL may hold, as PHP parse_url's array keys. const ( partScheme = 1 << iota partHost partUser partPass partPort partPath partQuery partFragment ) // phpURL is the array PHP parse_url returns; set records which keys exist. type phpURL struct { set int scheme, host, user, pass, port, path, query, frag string } func (u phpURL) has(part int) bool { return u.set&part != 0 } // parseURL ports PHP parse_url for the URL shapes a sitemap meets: an // optional scheme, an authority after "//", a path, a query and a fragment. // Control characters in a part are replaced by "_", as PHP does. func parseURL(s string) phpURL { var u phpURL rest := s if i := strings.IndexByte(rest, ':'); i > 0 && validScheme(rest[:i]) { after := rest[i+1:] if !strings.HasPrefix(after, "//") && portOnly(after) { // "host:port/path" without a scheme. } else { u.scheme, u.set = rest[:i], u.set|partScheme rest = after } } if strings.HasPrefix(rest, "//") { rest = rest[2:] end := strings.IndexAny(rest, "/?#") authority := rest if end >= 0 { authority, rest = rest[:end], rest[end:] } else { rest = "" } u.parseAuthority(authority) } if i := strings.IndexByte(rest, '#'); i >= 0 { u.frag, u.set = rest[i+1:], u.set|partFragment rest = rest[:i] } if i := strings.IndexByte(rest, '?'); i >= 0 { u.query, u.set = rest[i+1:], u.set|partQuery rest = rest[:i] } if rest != "" { u.path, u.set = rest, u.set|partPath } u.scheme = replaceControl(u.scheme) u.host = replaceControl(u.host) u.user = replaceControl(u.user) u.pass = replaceControl(u.pass) u.path = replaceControl(u.path) u.query = replaceControl(u.query) u.frag = replaceControl(u.frag) return u } func (u *phpURL) parseAuthority(a string) { if i := strings.LastIndexByte(a, '@'); i >= 0 { userinfo := a[:i] a = a[i+1:] if j := strings.IndexByte(userinfo, ':'); j >= 0 { u.user, u.pass = userinfo[:j], userinfo[j+1:] u.set |= partUser | partPass } else { u.user = userinfo u.set |= partUser } } if i := strings.LastIndexByte(a, ':'); i >= 0 && !strings.HasSuffix(a, "]") { port := a[i+1:] if port != "" && allDigits(port) { u.port, u.set = port, u.set|partPort } a = a[:i] } if a != "" { u.host, u.set = a, u.set|partHost } } func validScheme(s string) bool { for i := 0; i < len(s); i++ { c := s[i] if !(isAlnum(c) || c == '+' || c == '-' || c == '.') { return false } } return s != "" } // portOnly reports whether s is a port number up to the first "/", the // case PHP parse_url reads as host:port rather than scheme:path. func portOnly(s string) bool { end := strings.IndexByte(s, '/') if end < 0 { end = len(s) } return end > 0 && end <= 5 && allDigits(s[:end]) } func allDigits(s string) bool { for i := 0; i < len(s); i++ { if s[i] < '0' || s[i] > '9' { return false } } return s != "" } func isAlnum(c byte) bool { return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' } func replaceControl(s string) string { if strings.IndexFunc(s, func(r rune) bool { return r < 0x20 || r == 0x7f }) < 0 { return s } b := []byte(s) for i, c := range b { if c < 0x20 || c == 0x7f { b[i] = '_' } } return string(b) } // phpTrim is PHP trim() with its default character list. func phpTrim(s string) string { return strings.Trim(s, " \t\n\r\x00\x0b") } // singleColonSchemes are written "scheme:" instead of "scheme://". var singleColonSchemes = map[string]bool{"mailto": true, "tel": true, "sms": true} // defaultPorts are Winter UrlGenerator::isDefaultPort's known ports. var defaultPorts = map[string]int{ "ftp": 21, "ssh": 22, "sftp": 22, "telnet": 23, "smtp": 25, "dns": 53, "gopher": 70, "http": 80, "pop3": 110, "imap": 143, "ldap": 389, "https": 443, "smtps": 465, "ldaps": 636, "ftps": 990, "imaps": 993, "pop3s": 995, } // BuildURL is Winter UrlGenerator::buildUrl($url) with no replacements: the // query is URL-decoded, the URL parsed, every path segment re-encoded with // rawurlencode(rawurldecode()) after "." and ".." are resolved, the query // rebuilt with buildStr and the fragment re-encoded. func BuildURL(raw string) string { return buildParts(parseURL(decodeQueryParams(raw))) } // replacePath is buildUrl($url, ['path' => path]). func replacePath(raw, path string) string { u := parseURL(decodeQueryParams(raw)) u.path, u.set = path, u.set|partPath return buildParts(u) } // decodeQueryParams URL-decodes everything after the first "?". func decodeQueryParams(s string) string { if i := strings.IndexByte(s, '?'); i >= 0 { return s[:i] + "?" + urldecode(s[i+1:]) } return s } func buildParts(u phpURL) string { u.scheme, u.host, u.user, u.pass = phpTrim(u.scheme), phpTrim(u.host), phpTrim(u.user), phpTrim(u.pass) u.path, u.query, u.frag = phpTrim(u.path), phpTrim(u.query), phpTrim(u.frag) port := 0 if u.has(partPort) { if n, err := strconv.Atoi(phpTrim(u.port)); err == nil && n >= 1 && n <= 65535 { port = n } } var b strings.Builder hasScheme := u.has(partScheme) && u.scheme != "" if hasScheme { b.WriteString(u.scheme) if singleColonSchemes[u.scheme] { b.WriteString(":") } else { b.WriteString("://") } } if u.has(partUser) && u.user != "" { b.WriteString(u.user) if u.has(partPass) && u.pass != "" { b.WriteString(":" + u.pass) } b.WriteString("@") } if u.has(partHost) && u.host != "" { b.WriteString(u.host) } if port != 0 { if def, ok := defaultPorts[strings.ToLower(u.scheme)]; !ok || def != port { b.WriteString(":" + strconv.Itoa(port)) } } path := "" if u.has(partPath) && u.path != "" { var kept []string for _, seg := range strings.Split(u.path, "/") { switch { case seg == ".": case seg == ".." && len(kept) > 0: kept = kept[:len(kept)-1] default: kept = append(kept, seg) } } for i, seg := range kept { kept[i] = rawurlencode(rawurldecode(seg)) } path = strings.Join(kept, "/") } if hasScheme && !singleColonSchemes[u.scheme] { if !strings.HasPrefix(path, "/") { b.WriteString("/") } b.WriteString(path) } else { b.WriteString(path) } if u.has(partQuery) && u.query != "" { b.WriteString("?" + buildStr(parseQueryPairs(u.query), "")) } if u.has(partFragment) && u.frag != "" { b.WriteString("#" + rawurlencode(rawurldecode(u.frag))) } return b.String() } // phpKey is a PHP array key: an int or a string. type phpKey struct { isInt bool i int64 s string } func (k phpKey) String() string { if k.isInt { return strconv.FormatInt(k.i, 10) } return k.s } // keyOf is PHP's array key cast of a string: a canonical decimal integer // becomes an int key. func keyOf(s string) phpKey { if n, err := strconv.ParseInt(s, 10, 64); err == nil && strconv.FormatInt(n, 10) == s { return phpKey{isInt: true, i: n} } return phpKey{s: s} } // phpArray is an ordered PHP array of strings, nested arrays or nulls. type phpArray struct { keys []phpKey vals []any next int64 } func (a *phpArray) index(k phpKey) int { for i, have := range a.keys { if have == k { return i } } return -1 } func (a *phpArray) set(k phpKey, v any) { if i := a.index(k); i >= 0 { a.vals[i] = v return } a.keys = append(a.keys, k) a.vals = append(a.vals, v) if k.isInt && k.i >= a.next { a.next = k.i + 1 } } func (a *phpArray) push(v any) { a.set(phpKey{isInt: true, i: a.next}, v) } // parseQueryPairs is buildUrl's split of a string query into // $queryParams. func parseQueryPairs(query string) *phpArray { params := &phpArray{} for _, pair := range strings.Split(query, "&") { key, value := pair, pair if i := strings.IndexByte(pair, '='); i >= 0 { key, value = pair[:i], pair[i+1:] } if !strings.Contains(pair, "=") { params.push(key) continue } if strings.Contains(urldecode(pair), "[") { k := keyOf(key) var inner *phpArray if i := params.index(k); i >= 0 { inner, _ = params.vals[i].(*phpArray) } if inner == nil { inner = &phpArray{} } inner.push(value) params.set(k, inner) continue } params.set(keyOf(key), value) } return params } // buildStr is Winter UrlGenerator::buildStr. func buildStr(query *phpArray, prefix string) string { var out []string for i, k := range query.keys { v := query.vals[i] var key string if k.isInt && k.i == int64(i) { if prefix != "" { if strings.HasSuffix(prefix, "]") { key = prefix } else { key = prefix + "[]" } } else { key, _ = v.(string) v = nil } } else if prefix != "" { key = prefix + "[" + k.String() + "]" } else { key = k.String() } if nested, ok := v.(*phpArray); ok { out = append(out, buildStr(nested, key)) continue } part := rawurlencode(rawurldecode(key)) if s, ok := v.(string); ok { part += "=" + rawurlencode(rawurldecode(s)) } out = append(out, part) } return strings.Join(out, "&") } // rawurlencode is PHP rawurlencode: every byte except A-Z a-z 0-9 - _ . ~ // becomes %XX with upper-case hex. func rawurlencode(s string) string { const hex = "0123456789ABCDEF" var b strings.Builder for i := 0; i < len(s); i++ { c := s[i] if isAlnum(c) || c == '-' || c == '_' || c == '.' || c == '~' { b.WriteByte(c) continue } b.WriteByte('%') b.WriteByte(hex[c>>4]) b.WriteByte(hex[c&15]) } return b.String() } // rawurldecode is PHP rawurldecode: %XX becomes its byte, anything else, // including a malformed escape, stays. func rawurldecode(s string) string { return percentDecode(s, false) } // urldecode is PHP urldecode: rawurldecode that also turns "+" into a // space. func urldecode(s string) string { return percentDecode(s, true) } func percentDecode(s string, plus bool) string { if !strings.ContainsAny(s, "%+") { return s } var b strings.Builder for i := 0; i < len(s); i++ { c := s[i] if c == '+' && plus { b.WriteByte(' ') continue } if c == '%' && i+2 < len(s) && isHex(s[i+1]) && isHex(s[i+2]) { b.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2])) i += 2 continue } b.WriteByte(c) } return b.String() } func isHex(c byte) bool { return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' } func unhex(c byte) byte { switch { case '0' <= c && c <= '9': return c - '0' case 'a' <= c && c <= 'f': return c - 'a' + 10 default: return c - 'A' + 10 } } // URLPath is the path PHP parse_url($url, PHP_URL_PATH) returns, "" when // the URL has none. func URLPath(raw string) string { u := parseURL(raw) if !u.has(partPath) { return "" } return u.path } // schemePrefix matches a URL that starts with a scheme and its colon. var schemePrefix = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`) // AllowedItemURL reports whether a url item's address may be stored: empty, // a path (relative or starting with a single "/"), or an absolute http or // https URL. Any other scheme (javascript:, data:, mailto: and the like), a // protocol-relative "//host" or "/\host", a bare fragment, surrounding // whitespace and control characters are refused. Url::to keeps any URL // FILTER_VALIDATE_URL accepts, so without this a javascript: URL reached // , which sitemap.xsl renders as a link. func AllowedItemURL(u string) bool { if u == "" { return true } if strings.TrimSpace(u) != u || strings.ContainsFunc(u, func(r rune) bool { return r < 0x20 || r == 0x7f }) { return false } if httpURL(u) { return true } if schemePrefix.MatchString(u) || u[0] == '#' || u[0] == '\\' { return false } return !strings.HasPrefix(u, "//") && !strings.HasPrefix(u, `/\`) } // httpURL reports whether u starts with http:// or https:// // (case-insensitive). func httpURL(u string) bool { lower := strings.ToLower(u) return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") }