summercms/modules/tide/centrifugo_golden.go

package tide

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/goccy/go-yaml"
)

// BroadcastGolden is a versioned file of normalised Centrifugo requests a
// reference backend sent while a flow ran. Pending, when set, says why the
// golden is recorded but not yet asserted.
type BroadcastGolden struct {
	Version      int
	Name         string
	Flow         string
	Pending      string
	Publications []Publication
}

type goldenFile struct {
	Version      int             `yaml:"version"`
	Name         string          `yaml:"name"`
	Flow         string          `yaml:"flow,omitempty"`
	Pending      string          `yaml:"pending,omitempty"`
	Publications []goldenPubFile `yaml:"publications"`
}

type goldenPubFile struct {
	Method        string `yaml:"method"`
	Path          string `yaml:"path"`
	Authorization bool   `yaml:"authorization"`
	Body          Body   `yaml:"body"`
}

// LoadBroadcastGolden reads a golden strictly (unknown fields rejected) and
// checks that every body parses.
func LoadBroadcastGolden(path string) (BroadcastGolden, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return BroadcastGolden{}, fmt.Errorf("tide: read %s: %w", path, err)
	}
	var f goldenFile
	dec := yaml.NewDecoder(bytes.NewReader(raw), yaml.DisallowUnknownField())
	if err := dec.Decode(&f); err != nil {
		return BroadcastGolden{}, fmt.Errorf("tide: parse golden %s: %w", path, err)
	}
	g := BroadcastGolden{Version: f.Version, Name: f.Name, Flow: f.Flow, Pending: f.Pending}
	for _, p := range f.Publications {
		g.Publications = append(g.Publications, Publication{
			Method:        p.Method,
			Path:          p.Path,
			Authorization: p.Authorization,
			Body:          json.RawMessage(p.Body),
		})
	}
	if err := validateGolden(g); err != nil {
		return BroadcastGolden{}, fmt.Errorf("tide: golden %s: %w", path, err)
	}
	return g, nil
}

// WriteBroadcastGolden writes g atomically. It refuses a golden whose
// bodies still hold a token-, secret- or password-shaped value.
func WriteBroadcastGolden(path string, g BroadcastGolden) error {
	if err := validateGolden(g); err != nil {
		return err
	}
	for i, p := range g.Publications {
		if hit := remainingCredential(string(p.Body)); hit != "" {
			return fmt.Errorf("tide: golden publication %d holds an unclassified credential-shaped value (%s)", i, hit)
		}
	}
	var b strings.Builder
	fmt.Fprintf(&b, "version: %d\n", g.Version)
	writeKV(&b, 0, "name", g.Name)
	if g.Flow != "" {
		writeKV(&b, 0, "flow", g.Flow)
	}
	if g.Pending != "" {
		writeKV(&b, 0, "pending", g.Pending)
	}
	if len(g.Publications) == 0 {
		b.WriteString("publications: []\n")
	} else {
		b.WriteString("publications:\n")
	}
	for _, p := range g.Publications {
		fmt.Fprintf(&b, "  - method: %s\n", encodeScalar(p.Method))
		writeKV(&b, 4, "path", p.Path)
		fmt.Fprintf(&b, "    authorization: %t\n", p.Authorization)
		writeBody(&b, 4, string(p.Body))
	}
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("tide: create golden dir: %w", err)
	}
	tmp, err := os.CreateTemp(dir, ".tide-golden-*.tmp")
	if err != nil {
		return fmt.Errorf("tide: create temp golden: %w", err)
	}
	name := tmp.Name()
	if _, err := tmp.WriteString(b.String()); err != nil {
		_ = tmp.Close()
		_ = os.Remove(name)
		return fmt.Errorf("tide: write golden: %w", err)
	}
	if err := tmp.Sync(); err != nil {
		_ = tmp.Close()
		_ = os.Remove(name)
		return fmt.Errorf("tide: sync golden: %w", err)
	}
	if err := tmp.Close(); err != nil {
		_ = os.Remove(name)
		return fmt.Errorf("tide: close golden: %w", err)
	}
	if err := os.Chmod(name, 0o644); err != nil {
		_ = os.Remove(name)
		return err
	}
	if err := os.Rename(name, path); err != nil {
		_ = os.Remove(name)
		return fmt.Errorf("tide: commit golden: %w", err)
	}
	return nil
}

func validateGolden(g BroadcastGolden) error {
	if g.Version != CurrentVersion {
		return fmt.Errorf("tide: unsupported golden version %d (want %d)", g.Version, CurrentVersion)
	}
	if strings.TrimSpace(g.Name) == "" {
		return fmt.Errorf("tide: golden name is required")
	}
	for i, p := range g.Publications {
		if strings.TrimSpace(p.Method) == "" || strings.TrimSpace(p.Path) == "" {
			return fmt.Errorf("tide: publication %d is missing method or path", i)
		}
		if _, err := decodePlaceholderJSON(p.Body); err != nil {
			return fmt.Errorf("tide: publication %d body: %w", i, err)
		}
	}
	return nil
}

// isoOffsetRe is the ISO 8601 shape of Carbon's toIso8601String, which
// Centrifugo payload timestamps use. Other shapes are left unmasked so a
// format change shows up as a diff.
var isoOffsetRe = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$`)

// channelIDRe matches a channel name whose last segment is numeric, such as
// collection:12, acme:collection:12 or presence:room:7.
var channelIDRe = regexp.MustCompile(`^((?:[a-z][a-z0-9_.-]*:)+)([0-9]+)$`)

// NormalizePublications masks the values that legitimately differ between
// two backends and nothing else:
//   - $.data.timestamp and $.data.payload.timestamp become "{{timestamp}}"
//     when they have the ISO 8601 offset shape;
//   - $.data.payload.actor becomes "{{actor}}" when it is an object with
//     exactly user_id and name;
//   - a *_at value anywhere under $.data.payload.album becomes
//     "{{datetime}}" when it has Carbon's +00:00 shape; any other shape is
//     left as it is, so a format change shows up as a diff;
//   - a number or string under an id, *_id or *_ids key that equals an id:*
//     variable of store becomes that {{id:name}} placeholder (bare for a
//     number, quoted for a string), and so does the numeric last segment of
//     a channel name such as collection:12.
//
// A value that matches two id variables is an error. Bodies are re-encoded
// with object keys in their original order.
func NormalizePublications(pubs []Publication, store *Store) ([]Publication, error) {
	ids := idValues(store)
	out := make([]Publication, len(pubs))
	for i, p := range pubs {
		root, err := decodeOrdered(p.Body)
		if err != nil {
			return nil, fmt.Errorf("tide: publication %d body: %w", i, err)
		}
		n := &normalizer{ids: ids}
		root = n.walk("$", "", root)
		if n.err != nil {
			return nil, fmt.Errorf("tide: publication %d: %w", i, n.err)
		}
		var buf bytes.Buffer
		if err := encodeOrdered(&buf, root); err != nil {
			return nil, err
		}
		p.Body = json.RawMessage(buf.Bytes())
		out[i] = p
	}
	return out, nil
}

// DiffPublications compares expected with actual: the count, then per
// publication the method, path, authorization flag and the body with the
// structural JSON diff (key order ignored). Paths look like $[0].body.data.
func DiffPublications(expected, actual []Publication) []Diff {
	var diffs []Diff
	if len(expected) != len(actual) {
		diffs = append(diffs, Diff{
			Path:     "$",
			Expected: fmt.Sprintf("%d publications", len(expected)),
			Actual:   fmt.Sprintf("%d publications", len(actual)),
		})
	}
	for i := 0; i < min(len(expected), len(actual)); i++ {
		w, g := expected[i], actual[i]
		base := fmt.Sprintf("$[%d]", i)
		if w.Method != g.Method {
			diffs = append(diffs, Diff{Path: base + ".method", Expected: w.Method, Actual: g.Method})
		}
		if w.Path != g.Path {
			diffs = append(diffs, Diff{Path: base + ".path", Expected: w.Path, Actual: g.Path})
		}
		if w.Authorization != g.Authorization {
			diffs = append(diffs, Diff{Path: base + ".authorization", Expected: fmt.Sprint(w.Authorization), Actual: fmt.Sprint(g.Authorization)})
		}
		wv, err := decodePlaceholderJSON(w.Body)
		if err != nil {
			diffs = append(diffs, Diff{Path: base + ".body", Expected: "valid JSON", Actual: err.Error()})
			continue
		}
		gv, err := decodePlaceholderJSON(g.Body)
		if err != nil {
			diffs = append(diffs, Diff{Path: base + ".body", Expected: formatValue(wv), Actual: err.Error()})
			continue
		}
		compareValue(base+".body", wv, gv, &diffs)
	}
	sort.SliceStable(diffs, func(i, j int) bool { return diffs[i].Path < diffs[j].Path })
	return diffs
}

func idValues(store *Store) map[string][]string {
	out := map[string][]string{}
	if store == nil {
		return out
	}
	store.mu.Lock()
	defer store.mu.Unlock()
	for k, v := range store.vals {
		if strings.HasPrefix(k, "id:") && v != "" {
			out[v] = append(out[v], k)
		}
	}
	for v := range out {
		sort.Strings(out[v])
	}
	return out
}

type normalizer struct {
	ids map[string][]string
	err error
}

func (n *normalizer) lookup(v string) (string, bool) {
	names := n.ids[v]
	switch len(names) {
	case 0:
		return "", false
	case 1:
		return names[0], true
	default:
		if n.err == nil {
			n.err = fmt.Errorf("id value %s is ambiguous: %s", v, strings.Join(names, ", "))
		}
		return "", false
	}
}

func (n *normalizer) walk(path, key string, v *onode) *onode {
	switch path {
	case "$.data.timestamp", "$.data.payload.timestamp":
		if v.kind == kindString && isoOffsetRe.MatchString(v.str) {
			return &onode{kind: kindString, str: "{{timestamp}}"}
		}
		return v
	case "$.data.payload.actor":
		if v.kind == kindObject && len(v.keys) == 2 && hasKeys(v, "user_id", "name") {
			return &onode{kind: kindString, str: "{{actor}}"}
		}
		return v
	}
	switch v.kind {
	case kindObject:
		for i, k := range v.keys {
			v.vals[i] = n.walk(pathJoin(path, k), k, v.vals[i])
		}
		return v
	case kindArray:
		for i := range v.vals {
			v.vals[i] = n.walk(fmt.Sprintf("%s[%d]", path, i), key, v.vals[i])
		}
		return v
	case kindNumber:
		if key != "" && isIDKey(key) {
			if name, ok := n.lookup(v.str); ok {
				return &onode{kind: kindPlaceholder, str: name}
			}
		}
		return v
	case kindString:
		if strings.HasPrefix(path, "$.data.payload.album.") && strings.HasSuffix(key, "_at") && carbonOffsetRe.MatchString(v.str) {
			return &onode{kind: kindString, str: "{{datetime}}"}
		}
		if key != "" && isIDKey(key) {
			if name, ok := n.lookup(v.str); ok {
				return &onode{kind: kindString, str: "{{" + name + "}}"}
			}
		}
		if m := channelIDRe.FindStringSubmatch(v.str); m != nil {
			if name, ok := n.lookup(m[2]); ok {
				return &onode{kind: kindString, str: m[1] + "{{" + name + "}}"}
			}
		}
		return v
	default:
		return v
	}
}

func hasKeys(v *onode, keys ...string) bool {
	for _, want := range keys {
		found := false
		for _, k := range v.keys {
			if k == want {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

const (
	kindNull = iota
	kindBool
	kindNumber
	kindString
	kindArray
	kindObject
	// kindPlaceholder is a bare {{name}} standing for a number.
	kindPlaceholder
)

// onode is a JSON value that keeps object keys in document order.
type onode struct {
	kind int
	str  string // string value, number literal or placeholder name
	b    bool
	keys []string
	vals []*onode
}

func decodeOrdered(raw []byte) (*onode, error) {
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.UseNumber()
	v, err := decodeOrderedValue(dec)
	if err != nil {
		return nil, err
	}
	if dec.More() {
		return nil, fmt.Errorf("trailing JSON after first value")
	}
	if _, err := dec.Token(); err == nil {
		return nil, fmt.Errorf("trailing JSON after first value")
	}
	return v, nil
}

func decodeOrderedValue(dec *json.Decoder) (*onode, error) {
	tok, err := dec.Token()
	if err != nil {
		return nil, err
	}
	switch t := tok.(type) {
	case json.Delim:
		switch t {
		case '{':
			n := &onode{kind: kindObject}
			for dec.More() {
				kt, err := dec.Token()
				if err != nil {
					return nil, err
				}
				k, ok := kt.(string)
				if !ok {
					return nil, fmt.Errorf("object key %v is not a string", kt)
				}
				v, err := decodeOrderedValue(dec)
				if err != nil {
					return nil, err
				}
				n.keys = append(n.keys, k)
				n.vals = append(n.vals, v)
			}
			if _, err := dec.Token(); err != nil {
				return nil, err
			}
			return n, nil
		case '[':
			n := &onode{kind: kindArray}
			for dec.More() {
				v, err := decodeOrderedValue(dec)
				if err != nil {
					return nil, err
				}
				n.vals = append(n.vals, v)
			}
			if _, err := dec.Token(); err != nil {
				return nil, err
			}
			return n, nil
		default:
			return nil, fmt.Errorf("unexpected delimiter %v", t)
		}
	case string:
		return &onode{kind: kindString, str: t}, nil
	case json.Number:
		return &onode{kind: kindNumber, str: string(t)}, nil
	case bool:
		return &onode{kind: kindBool, b: t}, nil
	case nil:
		return &onode{kind: kindNull}, nil
	default:
		return nil, fmt.Errorf("unexpected token %v", tok)
	}
}

func encodeOrdered(buf *bytes.Buffer, n *onode) error {
	switch n.kind {
	case kindNull:
		buf.WriteString("null")
	case kindBool:
		if n.b {
			buf.WriteString("true")
		} else {
			buf.WriteString("false")
		}
	case kindNumber:
		buf.WriteString(n.str)
	case kindPlaceholder:
		buf.WriteString("{{" + n.str + "}}")
	case kindString:
		return encodeJSONString(buf, n.str)
	case kindArray:
		buf.WriteByte('[')
		for i, v := range n.vals {
			if i > 0 {
				buf.WriteByte(',')
			}
			if err := encodeOrdered(buf, v); err != nil {
				return err
			}
		}
		buf.WriteByte(']')
	case kindObject:
		buf.WriteByte('{')
		for i, k := range n.keys {
			if i > 0 {
				buf.WriteByte(',')
			}
			if err := encodeJSONString(buf, k); err != nil {
				return err
			}
			buf.WriteByte(':')
			if err := encodeOrdered(buf, n.vals[i]); err != nil {
				return err
			}
		}
		buf.WriteByte('}')
	default:
		return fmt.Errorf("tide: unknown JSON node kind %d", n.kind)
	}
	return nil
}

func encodeJSONString(buf *bytes.Buffer, s string) error {
	var tmp bytes.Buffer
	enc := json.NewEncoder(&tmp)
	enc.SetEscapeHTML(false)
	if err := enc.Encode(s); err != nil {
		return err
	}
	buf.Write(bytes.TrimSuffix(tmp.Bytes(), []byte("\n")))
	return nil
}

// numberPlaceholderSuffix marks a bare {{name}} (a masked number) when a
// normalised body is parsed for diffing, so a number that turned into a
// string, or the reverse, still shows up as a difference.
const numberPlaceholderSuffix = " (number)"

// decodePlaceholderJSON parses a normalised body: JSON in which a bare
// {{name}} may stand where a number was.
func decodePlaceholderJSON(raw []byte) (any, error) {
	var out bytes.Buffer
	inString := false
	escaped := false
	for i := 0; i < len(raw); i++ {
		c := raw[i]
		if inString {
			out.WriteByte(c)
			switch {
			case escaped:
				escaped = false
			case c == '\\':
				escaped = true
			case c == '"':
				inString = false
			}
			continue
		}
		if c == '"' {
			inString = true
			out.WriteByte(c)
			continue
		}
		if c == '{' && i+1 < len(raw) && raw[i+1] == '{' {
			end := bytes.Index(raw[i:], []byte("}}"))
			if end < 0 {
				return nil, fmt.Errorf("unterminated placeholder at byte %d", i)
			}
			name := string(raw[i+2 : i+end])
			if name == "" || strings.ContainsAny(name, "{}\"\\") {
				return nil, fmt.Errorf("invalid placeholder at byte %d", i)
			}
			if err := encodeJSONString(&out, "{{"+name+"}}"+numberPlaceholderSuffix); err != nil {
				return nil, err
			}
			i += end + 1
			continue
		}
		out.WriteByte(c)
	}
	return decodeJSON(out.Bytes())
}