package models import ( "bytes" "encoding/json" "fmt" "regexp" "strconv" "strings" "unicode" "unicode/utf8" ) // Text returns *p, "" for nil. func Text(p *string) string { if p == nil { return "" } return *p } // LegacyItemURL is the type of a Phase 17 (and WinterCMS) sitemap // definition item that is an explicit address. const LegacyItemURL = "url" // LegacyItem is one item of a Phase 17 or WinterCMS sitemap definition, as // golem15_sitemap_definitions.data and the WinterCMS export store it. Only // the fields the per-type format still reads are kept: the type (a missing // type is url, as in PHP), the address of a url item and the reference of a // registry item. type LegacyItem struct { Type string URL string Reference string CMSPage string } // LegacyEntries splits a JSON array, or a JSON object in key order (a PHP // array with keys), into its values with their PHP labels (index or key). // ok is false for any other value. func LegacyEntries(raw []byte) ([]json.RawMessage, []string, bool) { raw = bytes.TrimSpace(raw) if len(raw) == 0 { return nil, nil, false } switch raw[0] { case '[': var list []json.RawMessage if err := json.Unmarshal(raw, &list); err != nil { return nil, nil, false } labels := make([]string, len(list)) for i := range list { labels[i] = strconv.Itoa(i) } return list, labels, true case '{': dec := json.NewDecoder(bytes.NewReader(raw)) if _, err := dec.Token(); err != nil { return nil, nil, false } var list []json.RawMessage var labels []string for dec.More() { tok, err := dec.Token() if err != nil { return nil, nil, false } key, _ := tok.(string) var v json.RawMessage if err := dec.Decode(&v); err != nil { return nil, nil, false } list = append(list, v) labels = append(labels, key) } return list, labels, true } return nil, nil, false } // ParseLegacyItem decodes one stored item tolerantly: text fields may be // strings, numbers, booleans or null; a missing type is url; unknown keys // are ignored. func ParseLegacyItem(raw json.RawMessage) (LegacyItem, error) { var fields map[string]json.RawMessage if err := json.Unmarshal(raw, &fields); err != nil { return LegacyItem{}, fmt.Errorf("an item must be an object: %w", err) } item := LegacyItem{Type: LegacyItemURL} for key, dest := range map[string]*string{"type": &item.Type, "url": &item.URL, "reference": &item.Reference, "cmsPage": &item.CMSPage} { v, ok := fields[key] if !ok { continue } text, err := DecodeText(v) if err != nil { return LegacyItem{}, fmt.Errorf("%s %w", key, err) } *dest = strings.TrimSpace(Text(text)) } return item, nil } // ParseLegacyItems decodes the data of a definition: a JSON array or a PHP // keyed object of items, itself possibly stored as a JSON string. Empty or // null data has no items. func ParseLegacyItems(data []byte) ([]LegacyItem, []string, error) { data = LegacyJSON(data) if len(data) == 0 || bytes.Equal(data, []byte("null")) { return nil, nil, nil } raws, labels, ok := LegacyEntries(data) if !ok { return nil, nil, fmt.Errorf("the definition data is not a list of items") } items := make([]LegacyItem, 0, len(raws)) for i, raw := range raws { item, err := ParseLegacyItem(raw) if err != nil { return nil, nil, fmt.Errorf("definition item %s: %w", labels[i], err) } items = append(items, item) } return items, labels, nil } // LegacyJSON is a JSON value that may be held as text (a TEXT column // selected as it is) or as JSON: a string is decoded once more. func LegacyJSON(raw []byte) []byte { raw = bytes.TrimSpace(raw) if len(raw) > 0 && raw[0] == '"' { var text string if json.Unmarshal(raw, &text) == nil { return []byte(strings.TrimSpace(text)) } } return raw } // DecodeText decodes a stored text value with PHP's string conversion: // null is nil, a number its PHP text, true "1" and false "". func DecodeText(raw json.RawMessage) (*string, error) { var v any dec := json.NewDecoder(bytes.NewReader(raw)) dec.UseNumber() if err := dec.Decode(&v); err != nil { return nil, err } var s string switch t := v.(type) { case nil: return nil, nil case string: s = t case json.Number: s = phpNumberString(t) case bool: if t { s = "1" } default: return nil, fmt.Errorf("must be text, got %T", v) } return &s, nil } func phpNumberString(n json.Number) string { text := n.String() if !strings.ContainsAny(text, ".eE") { if i, err := strconv.ParseInt(text, 10, 64); err == nil { return strconv.FormatInt(i, 10) } } f, err := n.Float64() if err != nil { return text } return strconv.FormatFloat(f, 'f', -1, 64) } // DroppedItem is a url item of a Phase 17 definition that ConvertLegacyItems // could not copy: its PHP label (index or key; "" for a definition that is // not a list of items), the value as stored and the reason. type DroppedItem struct { Label string URL string Reason string } // legacyScheme matches a value that starts with a URL scheme. var legacyScheme = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`) // ConvertLegacyItems turns the url items of a Phase 17 definition into // additional addresses (D-32, P33-66; pure). The entries are parsed one by // one: a malformed item is dropped with its parse error and the others are // kept; items of another type (records the content types list) are neither // kept nor reported. A value without a scheme and without a leading / gets // one, as the Phase 17 sitemap resolved it against the site root; spaces // and the bytes " < > \ are percent-encoded. The result must pass // AllowedAddress; one that does not is dropped with the reason (an // absolute URL with & among them). A duplicate is kept once and not // reported. func ConvertLegacyItems(data []byte) ([]ExtraURL, []DroppedItem) { rows := []ExtraURL{} var dropped []DroppedItem data = LegacyJSON(data) if len(data) == 0 || bytes.Equal(data, []byte("null")) { return rows, nil } raws, labels, ok := LegacyEntries(data) if !ok { return rows, []DroppedItem{{Reason: "the definition data is not a list of items"}} } seen := map[string]bool{} for i, raw := range raws { item, err := ParseLegacyItem(raw) if err != nil { dropped = append(dropped, DroppedItem{Label: labels[i], Reason: err.Error()}) continue } if item.Type != LegacyItemURL { continue } u, reason := convertLegacyURL(item.URL) if reason != "" { dropped = append(dropped, DroppedItem{Label: labels[i], URL: item.URL, Reason: reason}) continue } if seen[u] { continue } seen[u] = true rows = append(rows, ExtraURL{URL: u}) } return rows, dropped } // convertLegacyURL is one url item's value as an additional address, or // the reason it cannot be one. func convertLegacyURL(raw string) (string, string) { u := strings.TrimSpace(raw) if u == "" { return "", "the item has no address" } if !utf8.ValidString(u) { return "", "the address is not valid UTF-8" } if !legacyScheme.MatchString(u) && !strings.HasPrefix(u, "/") { u = "/" + u } var b strings.Builder for _, r := range u { switch { case r == '"' || r == '<' || r == '>' || r == '\\' || unicode.IsSpace(r): for _, c := range []byte(string(r)) { fmt.Fprintf(&b, "%%%02X", c) } default: b.WriteRune(r) } } u = b.String() if AllowedAddress(u) { return u, "" } switch { case strings.HasPrefix(u, "//"): return "", "an address starting with // has no scheme" case strings.HasPrefix(u, "/"): return "", "the path holds a control character" case strings.Contains(u, "&"): return "", "an absolute address may not hold &" case strings.ContainsFunc(u, unicode.IsControl): return "", "the address holds a control character" } return "", "only a path or an absolute http(s) address can be an additional address" }