fix: the definitions migration keeps and logs every url item

Migration 202610280002 converts the Phase 17 url items through
models.ConvertLegacyItems: a relative path gets its leading slash,
spaces and the bytes " < > \ are percent-encoded, and a malformed item
no longer drops the whole definition. Each url item that cannot be
copied is logged at warn level with its label, value and reason.

models/legacy.go

@@ -4,8 +4,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 )
 
 // Text returns *p, "" for nil.
@@ -175,3 +178,101 @@ func phpNumberString(n json.Number) string {
 	}
 	return strconv.FormatFloat(f, 'f', -1, 64)
 }
+
+// DroppedItem is a url item of a Phase 17 definition that ConvertLegacyItems
+// could not copy: its PHP label (index or key; "" for a definition that is
+// not a list of items), the value as stored and the reason.
+type DroppedItem struct {
+	Label  string
+	URL    string
+	Reason string
+}
+
+// legacyScheme matches a value that starts with a URL scheme.
+var legacyScheme = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`)
+
+// ConvertLegacyItems turns the url items of a Phase 17 definition into
+// additional addresses (D-32, P33-66; pure). The entries are parsed one by
+// one: a malformed item is dropped with its parse error and the others are
+// kept; items of another type (records the content types list) are neither
+// kept nor reported. A value without a scheme and without a leading / gets
+// one, as the Phase 17 sitemap resolved it against the site root; spaces
+// and the bytes " < > \ are percent-encoded. The result must pass
+// AllowedAddress; one that does not is dropped with the reason (an
+// absolute URL with & among them). A duplicate is kept once and not
+// reported.
+func ConvertLegacyItems(data []byte) ([]ExtraURL, []DroppedItem) {
+	rows := []ExtraURL{}
+	var dropped []DroppedItem
+	data = LegacyJSON(data)
+	if len(data) == 0 || bytes.Equal(data, []byte("null")) {
+		return rows, nil
+	}
+	raws, labels, ok := LegacyEntries(data)
+	if !ok {
+		return rows, []DroppedItem{{Reason: "the definition data is not a list of items"}}
+	}
+	seen := map[string]bool{}
+	for i, raw := range raws {
+		item, err := ParseLegacyItem(raw)
+		if err != nil {
+			dropped = append(dropped, DroppedItem{Label: labels[i], Reason: err.Error()})
+			continue
+		}
+		if item.Type != LegacyItemURL {
+			continue
+		}
+		u, reason := convertLegacyURL(item.URL)
+		if reason != "" {
+			dropped = append(dropped, DroppedItem{Label: labels[i], URL: item.URL, Reason: reason})
+			continue
+		}
+		if seen[u] {
+			continue
+		}
+		seen[u] = true
+		rows = append(rows, ExtraURL{URL: u})
+	}
+	return rows, dropped
+}
+
+// convertLegacyURL is one url item's value as an additional address, or
+// the reason it cannot be one.
+func convertLegacyURL(raw string) (string, string) {
+	u := strings.TrimSpace(raw)
+	if u == "" {
+		return "", "the item has no address"
+	}
+	if !utf8.ValidString(u) {
+		return "", "the address is not valid UTF-8"
+	}
+	if !legacyScheme.MatchString(u) && !strings.HasPrefix(u, "/") {
+		u = "/" + u
+	}
+	var b strings.Builder
+	for _, r := range u {
+		switch {
+		case r == '"' || r == '<' || r == '>' || r == '\\' || unicode.IsSpace(r):
+			for _, c := range []byte(string(r)) {
+				fmt.Fprintf(&b, "%%%02X", c)
+			}
+		default:
+			b.WriteRune(r)
+		}
+	}
+	u = b.String()
+	if AllowedAddress(u) {
+		return u, ""
+	}
+	switch {
+	case strings.HasPrefix(u, "//"):
+		return "", "an address starting with // has no scheme"
+	case strings.HasPrefix(u, "/"):
+		return "", "the path holds a control character"
+	case strings.Contains(u, "&"):
+		return "", "an absolute address may not hold &"
+	case strings.ContainsFunc(u, unicode.IsControl):
+		return "", "the address holds a control character"
+	}
+	return "", "only a path or an absolute http(s) address can be an additional address"
+}