package tide import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "regexp" "sort" "strings" "github.com/goccy/go-yaml" ) // BroadcastGolden is a versioned file of normalised Centrifugo requests a // reference backend sent while a flow ran. Pending, when set, says why the // golden is recorded but not yet asserted. type BroadcastGolden struct { Version int Name string Flow string Pending string Publications []Publication } type goldenFile struct { Version int `yaml:"version"` Name string `yaml:"name"` Flow string `yaml:"flow,omitempty"` Pending string `yaml:"pending,omitempty"` Publications []goldenPubFile `yaml:"publications"` } type goldenPubFile struct { Method string `yaml:"method"` Path string `yaml:"path"` Authorization bool `yaml:"authorization"` Body Body `yaml:"body"` } // LoadBroadcastGolden reads a golden strictly (unknown fields rejected) and // checks that every body parses. func LoadBroadcastGolden(path string) (BroadcastGolden, error) { raw, err := os.ReadFile(path) if err != nil { return BroadcastGolden{}, fmt.Errorf("tide: read %s: %w", path, err) } var f goldenFile dec := yaml.NewDecoder(bytes.NewReader(raw), yaml.DisallowUnknownField()) if err := dec.Decode(&f); err != nil { return BroadcastGolden{}, fmt.Errorf("tide: parse golden %s: %w", path, err) } g := BroadcastGolden{Version: f.Version, Name: f.Name, Flow: f.Flow, Pending: f.Pending} for _, p := range f.Publications { g.Publications = append(g.Publications, Publication{ Method: p.Method, Path: p.Path, Authorization: p.Authorization, Body: json.RawMessage(p.Body), }) } if err := validateGolden(g); err != nil { return BroadcastGolden{}, fmt.Errorf("tide: golden %s: %w", path, err) } return g, nil } // WriteBroadcastGolden writes g atomically. It refuses a golden whose // bodies still hold a token-, secret- or password-shaped value. func WriteBroadcastGolden(path string, g BroadcastGolden) error { if err := validateGolden(g); err != nil { return err } for i, p := range g.Publications { if hit := remainingCredential(string(p.Body)); hit != "" { return fmt.Errorf("tide: golden publication %d holds an unclassified credential-shaped value (%s)", i, hit) } } var b strings.Builder fmt.Fprintf(&b, "version: %d\n", g.Version) writeKV(&b, 0, "name", g.Name) if g.Flow != "" { writeKV(&b, 0, "flow", g.Flow) } if g.Pending != "" { writeKV(&b, 0, "pending", g.Pending) } if len(g.Publications) == 0 { b.WriteString("publications: []\n") } else { b.WriteString("publications:\n") } for _, p := range g.Publications { fmt.Fprintf(&b, " - method: %s\n", encodeScalar(p.Method)) writeKV(&b, 4, "path", p.Path) fmt.Fprintf(&b, " authorization: %t\n", p.Authorization) writeBody(&b, 4, string(p.Body)) } dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("tide: create golden dir: %w", err) } tmp, err := os.CreateTemp(dir, ".tide-golden-*.tmp") if err != nil { return fmt.Errorf("tide: create temp golden: %w", err) } name := tmp.Name() if _, err := tmp.WriteString(b.String()); err != nil { _ = tmp.Close() _ = os.Remove(name) return fmt.Errorf("tide: write golden: %w", err) } if err := tmp.Sync(); err != nil { _ = tmp.Close() _ = os.Remove(name) return fmt.Errorf("tide: sync golden: %w", err) } if err := tmp.Close(); err != nil { _ = os.Remove(name) return fmt.Errorf("tide: close golden: %w", err) } if err := os.Chmod(name, 0o644); err != nil { _ = os.Remove(name) return err } if err := os.Rename(name, path); err != nil { _ = os.Remove(name) return fmt.Errorf("tide: commit golden: %w", err) } return nil } func validateGolden(g BroadcastGolden) error { if g.Version != CurrentVersion { return fmt.Errorf("tide: unsupported golden version %d (want %d)", g.Version, CurrentVersion) } if strings.TrimSpace(g.Name) == "" { return fmt.Errorf("tide: golden name is required") } for i, p := range g.Publications { if strings.TrimSpace(p.Method) == "" || strings.TrimSpace(p.Path) == "" { return fmt.Errorf("tide: publication %d is missing method or path", i) } if _, err := decodePlaceholderJSON(p.Body); err != nil { return fmt.Errorf("tide: publication %d body: %w", i, err) } } return nil } // isoOffsetRe is the ISO 8601 shape of Carbon's toIso8601String, which // Centrifugo payload timestamps use. Other shapes are left unmasked so a // format change shows up as a diff. var isoOffsetRe = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}$`) // channelIDRe matches a channel name whose last segment is numeric, such as // collection:12, acme:collection:12 or presence:room:7. var channelIDRe = regexp.MustCompile(`^((?:[a-z][a-z0-9_.-]*:)+)([0-9]+)$`) // NormalizePublications masks the values that legitimately differ between // two backends and nothing else: // - $.data.timestamp and $.data.payload.timestamp become "{{timestamp}}" // when they have the ISO 8601 offset shape; // - $.data.payload.actor becomes "{{actor}}" when it is an object with // exactly user_id and name; // - a *_at value anywhere under $.data.payload.album becomes // "{{datetime}}" when it has Carbon's +00:00 shape; any other shape is // left as it is, so a format change shows up as a diff; // - a number or string under an id, *_id or *_ids key that equals an id:* // variable of store becomes that {{id:name}} placeholder (bare for a // number, quoted for a string), and so does the numeric last segment of // a channel name such as collection:12. // // A value that matches two id variables is an error. Bodies are re-encoded // with object keys in their original order. func NormalizePublications(pubs []Publication, store *Store) ([]Publication, error) { ids := idValues(store) out := make([]Publication, len(pubs)) for i, p := range pubs { root, err := decodeOrdered(p.Body) if err != nil { return nil, fmt.Errorf("tide: publication %d body: %w", i, err) } n := &normalizer{ids: ids} root = n.walk("$", "", root) if n.err != nil { return nil, fmt.Errorf("tide: publication %d: %w", i, n.err) } var buf bytes.Buffer if err := encodeOrdered(&buf, root); err != nil { return nil, err } p.Body = json.RawMessage(buf.Bytes()) out[i] = p } return out, nil } // DiffPublications compares expected with actual: the count, then per // publication the method, path, authorization flag and the body with the // structural JSON diff (key order ignored). Paths look like $[0].body.data. func DiffPublications(expected, actual []Publication) []Diff { var diffs []Diff if len(expected) != len(actual) { diffs = append(diffs, Diff{ Path: "$", Expected: fmt.Sprintf("%d publications", len(expected)), Actual: fmt.Sprintf("%d publications", len(actual)), }) } for i := 0; i < min(len(expected), len(actual)); i++ { w, g := expected[i], actual[i] base := fmt.Sprintf("$[%d]", i) if w.Method != g.Method { diffs = append(diffs, Diff{Path: base + ".method", Expected: w.Method, Actual: g.Method}) } if w.Path != g.Path { diffs = append(diffs, Diff{Path: base + ".path", Expected: w.Path, Actual: g.Path}) } if w.Authorization != g.Authorization { diffs = append(diffs, Diff{Path: base + ".authorization", Expected: fmt.Sprint(w.Authorization), Actual: fmt.Sprint(g.Authorization)}) } wv, err := decodePlaceholderJSON(w.Body) if err != nil { diffs = append(diffs, Diff{Path: base + ".body", Expected: "valid JSON", Actual: err.Error()}) continue } gv, err := decodePlaceholderJSON(g.Body) if err != nil { diffs = append(diffs, Diff{Path: base + ".body", Expected: formatValue(wv), Actual: err.Error()}) continue } compareValue(base+".body", wv, gv, &diffs) } sort.SliceStable(diffs, func(i, j int) bool { return diffs[i].Path < diffs[j].Path }) return diffs } func idValues(store *Store) map[string][]string { out := map[string][]string{} if store == nil { return out } store.mu.Lock() defer store.mu.Unlock() for k, v := range store.vals { if strings.HasPrefix(k, "id:") && v != "" { out[v] = append(out[v], k) } } for v := range out { sort.Strings(out[v]) } return out } type normalizer struct { ids map[string][]string err error } func (n *normalizer) lookup(v string) (string, bool) { names := n.ids[v] switch len(names) { case 0: return "", false case 1: return names[0], true default: if n.err == nil { n.err = fmt.Errorf("id value %s is ambiguous: %s", v, strings.Join(names, ", ")) } return "", false } } func (n *normalizer) walk(path, key string, v *onode) *onode { switch path { case "$.data.timestamp", "$.data.payload.timestamp": if v.kind == kindString && isoOffsetRe.MatchString(v.str) { return &onode{kind: kindString, str: "{{timestamp}}"} } return v case "$.data.payload.actor": if v.kind == kindObject && len(v.keys) == 2 && hasKeys(v, "user_id", "name") { return &onode{kind: kindString, str: "{{actor}}"} } return v } switch v.kind { case kindObject: for i, k := range v.keys { v.vals[i] = n.walk(pathJoin(path, k), k, v.vals[i]) } return v case kindArray: for i := range v.vals { v.vals[i] = n.walk(fmt.Sprintf("%s[%d]", path, i), key, v.vals[i]) } return v case kindNumber: if key != "" && isIDKey(key) { if name, ok := n.lookup(v.str); ok { return &onode{kind: kindPlaceholder, str: name} } } return v case kindString: if strings.HasPrefix(path, "$.data.payload.album.") && strings.HasSuffix(key, "_at") && carbonOffsetRe.MatchString(v.str) { return &onode{kind: kindString, str: "{{datetime}}"} } if key != "" && isIDKey(key) { if name, ok := n.lookup(v.str); ok { return &onode{kind: kindString, str: "{{" + name + "}}"} } } if m := channelIDRe.FindStringSubmatch(v.str); m != nil { if name, ok := n.lookup(m[2]); ok { return &onode{kind: kindString, str: m[1] + "{{" + name + "}}"} } } return v default: return v } } func hasKeys(v *onode, keys ...string) bool { for _, want := range keys { found := false for _, k := range v.keys { if k == want { found = true break } } if !found { return false } } return true } const ( kindNull = iota kindBool kindNumber kindString kindArray kindObject // kindPlaceholder is a bare {{name}} standing for a number. kindPlaceholder ) // onode is a JSON value that keeps object keys in document order. type onode struct { kind int str string // string value, number literal or placeholder name b bool keys []string vals []*onode } func decodeOrdered(raw []byte) (*onode, error) { dec := json.NewDecoder(bytes.NewReader(raw)) dec.UseNumber() v, err := decodeOrderedValue(dec) if err != nil { return nil, err } if dec.More() { return nil, fmt.Errorf("trailing JSON after first value") } if _, err := dec.Token(); err == nil { return nil, fmt.Errorf("trailing JSON after first value") } return v, nil } func decodeOrderedValue(dec *json.Decoder) (*onode, error) { tok, err := dec.Token() if err != nil { return nil, err } switch t := tok.(type) { case json.Delim: switch t { case '{': n := &onode{kind: kindObject} for dec.More() { kt, err := dec.Token() if err != nil { return nil, err } k, ok := kt.(string) if !ok { return nil, fmt.Errorf("object key %v is not a string", kt) } v, err := decodeOrderedValue(dec) if err != nil { return nil, err } n.keys = append(n.keys, k) n.vals = append(n.vals, v) } if _, err := dec.Token(); err != nil { return nil, err } return n, nil case '[': n := &onode{kind: kindArray} for dec.More() { v, err := decodeOrderedValue(dec) if err != nil { return nil, err } n.vals = append(n.vals, v) } if _, err := dec.Token(); err != nil { return nil, err } return n, nil default: return nil, fmt.Errorf("unexpected delimiter %v", t) } case string: return &onode{kind: kindString, str: t}, nil case json.Number: return &onode{kind: kindNumber, str: string(t)}, nil case bool: return &onode{kind: kindBool, b: t}, nil case nil: return &onode{kind: kindNull}, nil default: return nil, fmt.Errorf("unexpected token %v", tok) } } func encodeOrdered(buf *bytes.Buffer, n *onode) error { switch n.kind { case kindNull: buf.WriteString("null") case kindBool: if n.b { buf.WriteString("true") } else { buf.WriteString("false") } case kindNumber: buf.WriteString(n.str) case kindPlaceholder: buf.WriteString("{{" + n.str + "}}") case kindString: return encodeJSONString(buf, n.str) case kindArray: buf.WriteByte('[') for i, v := range n.vals { if i > 0 { buf.WriteByte(',') } if err := encodeOrdered(buf, v); err != nil { return err } } buf.WriteByte(']') case kindObject: buf.WriteByte('{') for i, k := range n.keys { if i > 0 { buf.WriteByte(',') } if err := encodeJSONString(buf, k); err != nil { return err } buf.WriteByte(':') if err := encodeOrdered(buf, n.vals[i]); err != nil { return err } } buf.WriteByte('}') default: return fmt.Errorf("tide: unknown JSON node kind %d", n.kind) } return nil } func encodeJSONString(buf *bytes.Buffer, s string) error { var tmp bytes.Buffer enc := json.NewEncoder(&tmp) enc.SetEscapeHTML(false) if err := enc.Encode(s); err != nil { return err } buf.Write(bytes.TrimSuffix(tmp.Bytes(), []byte("\n"))) return nil } // numberPlaceholderSuffix marks a bare {{name}} (a masked number) when a // normalised body is parsed for diffing, so a number that turned into a // string, or the reverse, still shows up as a difference. const numberPlaceholderSuffix = " (number)" // decodePlaceholderJSON parses a normalised body: JSON in which a bare // {{name}} may stand where a number was. func decodePlaceholderJSON(raw []byte) (any, error) { var out bytes.Buffer inString := false escaped := false for i := 0; i < len(raw); i++ { c := raw[i] if inString { out.WriteByte(c) switch { case escaped: escaped = false case c == '\\': escaped = true case c == '"': inString = false } continue } if c == '"' { inString = true out.WriteByte(c) continue } if c == '{' && i+1 < len(raw) && raw[i+1] == '{' { end := bytes.Index(raw[i:], []byte("}}")) if end < 0 { return nil, fmt.Errorf("unterminated placeholder at byte %d", i) } name := string(raw[i+2 : i+end]) if name == "" || strings.ContainsAny(name, "{}\"\\") { return nil, fmt.Errorf("invalid placeholder at byte %d", i) } if err := encodeJSONString(&out, "{{"+name+"}}"+numberPlaceholderSuffix); err != nil { return nil, err } i += end + 1 continue } out.WriteByte(c) } return decodeJSON(out.Bytes()) }