sm-sitemap-plugin/models/legacy.go

package models

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

// Text returns *p, "" for nil.
func Text(p *string) string {
	if p == nil {
		return ""
	}
	return *p
}

// LegacyItemURL is the type of a Phase 17 (and WinterCMS) sitemap
// definition item that is an explicit address.
const LegacyItemURL = "url"

// LegacyItem is one item of a Phase 17 or WinterCMS sitemap definition, as
// golem15_sitemap_definitions.data and the WinterCMS export store it. Only
// the fields the per-type format still reads are kept: the type (a missing
// type is url, as in PHP), the address of a url item and the reference of a
// registry item.
type LegacyItem struct {
	Type      string
	URL       string
	Reference string
	CMSPage   string
}

// LegacyEntries splits a JSON array, or a JSON object in key order (a PHP
// array with keys), into its values with their PHP labels (index or key).
// ok is false for any other value.
func LegacyEntries(raw []byte) ([]json.RawMessage, []string, bool) {
	raw = bytes.TrimSpace(raw)
	if len(raw) == 0 {
		return nil, nil, false
	}
	switch raw[0] {
	case '[':
		var list []json.RawMessage
		if err := json.Unmarshal(raw, &list); err != nil {
			return nil, nil, false
		}
		labels := make([]string, len(list))
		for i := range list {
			labels[i] = strconv.Itoa(i)
		}
		return list, labels, true
	case '{':
		dec := json.NewDecoder(bytes.NewReader(raw))
		if _, err := dec.Token(); err != nil {
			return nil, nil, false
		}
		var list []json.RawMessage
		var labels []string
		for dec.More() {
			tok, err := dec.Token()
			if err != nil {
				return nil, nil, false
			}
			key, _ := tok.(string)
			var v json.RawMessage
			if err := dec.Decode(&v); err != nil {
				return nil, nil, false
			}
			list = append(list, v)
			labels = append(labels, key)
		}
		return list, labels, true
	}
	return nil, nil, false
}

// ParseLegacyItem decodes one stored item tolerantly: text fields may be
// strings, numbers, booleans or null; a missing type is url; unknown keys
// are ignored.
func ParseLegacyItem(raw json.RawMessage) (LegacyItem, error) {
	var fields map[string]json.RawMessage
	if err := json.Unmarshal(raw, &fields); err != nil {
		return LegacyItem{}, fmt.Errorf("an item must be an object: %w", err)
	}
	item := LegacyItem{Type: LegacyItemURL}
	for key, dest := range map[string]*string{"type": &item.Type, "url": &item.URL, "reference": &item.Reference, "cmsPage": &item.CMSPage} {
		v, ok := fields[key]
		if !ok {
			continue
		}
		text, err := DecodeText(v)
		if err != nil {
			return LegacyItem{}, fmt.Errorf("%s %w", key, err)
		}
		*dest = strings.TrimSpace(Text(text))
	}
	return item, nil
}

// ParseLegacyItems decodes the data of a definition: a JSON array or a PHP
// keyed object of items, itself possibly stored as a JSON string. Empty or
// null data has no items.
func ParseLegacyItems(data []byte) ([]LegacyItem, []string, error) {
	data = LegacyJSON(data)
	if len(data) == 0 || bytes.Equal(data, []byte("null")) {
		return nil, nil, nil
	}
	raws, labels, ok := LegacyEntries(data)
	if !ok {
		return nil, nil, fmt.Errorf("the definition data is not a list of items")
	}
	items := make([]LegacyItem, 0, len(raws))
	for i, raw := range raws {
		item, err := ParseLegacyItem(raw)
		if err != nil {
			return nil, nil, fmt.Errorf("definition item %s: %w", labels[i], err)
		}
		items = append(items, item)
	}
	return items, labels, nil
}

// LegacyJSON is a JSON value that may be held as text (a TEXT column
// selected as it is) or as JSON: a string is decoded once more.
func LegacyJSON(raw []byte) []byte {
	raw = bytes.TrimSpace(raw)
	if len(raw) > 0 && raw[0] == '"' {
		var text string
		if json.Unmarshal(raw, &text) == nil {
			return []byte(strings.TrimSpace(text))
		}
	}
	return raw
}

// DecodeText decodes a stored text value with PHP's string conversion:
// null is nil, a number its PHP text, true "1" and false "".
func DecodeText(raw json.RawMessage) (*string, error) {
	var v any
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	if err := dec.Decode(&v); err != nil {
		return nil, err
	}
	var s string
	switch t := v.(type) {
	case nil:
		return nil, nil
	case string:
		s = t
	case json.Number:
		s = phpNumberString(t)
	case bool:
		if t {
			s = "1"
		}
	default:
		return nil, fmt.Errorf("must be text, got %T", v)
	}
	return &s, nil
}

func phpNumberString(n json.Number) string {
	text := n.String()
	if !strings.ContainsAny(text, ".eE") {
		if i, err := strconv.ParseInt(text, 10, 64); err == nil {
			return strconv.FormatInt(i, 10)
		}
	}
	f, err := n.Float64()
	if err != nil {
		return text
	}
	return strconv.FormatFloat(f, 'f', -1, 64)
}

// DroppedItem is a url item of a Phase 17 definition that ConvertLegacyItems
// could not copy: its PHP label (index or key; "" for a definition that is
// not a list of items), the value as stored and the reason.
type DroppedItem struct {
	Label  string
	URL    string
	Reason string
}

// legacyScheme matches a value that starts with a URL scheme.
var legacyScheme = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`)

// ConvertLegacyItems turns the url items of a Phase 17 definition into
// additional addresses (D-32, P33-66; pure). The entries are parsed one by
// one: a malformed item is dropped with its parse error and the others are
// kept; items of another type (records the content types list) are neither
// kept nor reported. A value without a scheme and without a leading / gets
// one, as the Phase 17 sitemap resolved it against the site root; spaces
// and the bytes " < > \ are percent-encoded. The result must pass
// AllowedAddress; one that does not is dropped with the reason (an
// absolute URL with & among them). A duplicate is kept once and not
// reported.
func ConvertLegacyItems(data []byte) ([]ExtraURL, []DroppedItem) {
	rows := []ExtraURL{}
	var dropped []DroppedItem
	data = LegacyJSON(data)
	if len(data) == 0 || bytes.Equal(data, []byte("null")) {
		return rows, nil
	}
	raws, labels, ok := LegacyEntries(data)
	if !ok {
		return rows, []DroppedItem{{Reason: "the definition data is not a list of items"}}
	}
	seen := map[string]bool{}
	for i, raw := range raws {
		item, err := ParseLegacyItem(raw)
		if err != nil {
			dropped = append(dropped, DroppedItem{Label: labels[i], Reason: err.Error()})
			continue
		}
		if item.Type != LegacyItemURL {
			continue
		}
		u, reason := convertLegacyURL(item.URL)
		if reason != "" {
			dropped = append(dropped, DroppedItem{Label: labels[i], URL: item.URL, Reason: reason})
			continue
		}
		if seen[u] {
			continue
		}
		seen[u] = true
		rows = append(rows, ExtraURL{URL: u})
	}
	return rows, dropped
}

// convertLegacyURL is one url item's value as an additional address, or
// the reason it cannot be one.
func convertLegacyURL(raw string) (string, string) {
	u := strings.TrimSpace(raw)
	if u == "" {
		return "", "the item has no address"
	}
	if !utf8.ValidString(u) {
		return "", "the address is not valid UTF-8"
	}
	if !legacyScheme.MatchString(u) && !strings.HasPrefix(u, "/") {
		u = "/" + u
	}
	var b strings.Builder
	for _, r := range u {
		switch {
		case r == '"' || r == '<' || r == '>' || r == '\\' || unicode.IsSpace(r):
			for _, c := range []byte(string(r)) {
				fmt.Fprintf(&b, "%%%02X", c)
			}
		default:
			b.WriteRune(r)
		}
	}
	u = b.String()
	if AllowedAddress(u) {
		return u, ""
	}
	switch {
	case strings.HasPrefix(u, "//"):
		return "", "an address starting with // has no scheme"
	case strings.HasPrefix(u, "/"):
		return "", "the path holds a control character"
	case strings.Contains(u, "&"):
		return "", "an absolute address may not hold &"
	case strings.ContainsFunc(u, unicode.IsControl):
		return "", "the address holds a control character"
	}
	return "", "only a path or an absolute http(s) address can be an additional address"
}