From 5101ecb49cae4e1bcceecc91491c198175ce71c1 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Sat, 3 Oct 2026 21:34:52 +0200 Subject: [PATCH] feat(14-03): tide keeps binary upstream bodies and compares every *_url upload by shape - A response body that is not valid UTF-8 (a cover image) is written as a YAML !!binary scalar, never masked and replayed byte for byte - The upload URL normalizer covers every key ending in _url (cover_url), not only url and thumb_url - README and parity-testing docs updated --- docs/services/parity-testing.md | 4 +- modules/tide/README.md | 4 +- modules/tide/normalize.go | 10 ++++- modules/tide/normalize_upload_test.go | 4 ++ modules/tide/upstream.go | 55 ++++++++++++++++++++++++--- modules/tide/upstream_test.go | 48 +++++++++++++++++++++++ 6 files changed, 115 insertions(+), 10 deletions(-) diff --git a/docs/services/parity-testing.md b/docs/services/parity-testing.md index 364c6b3..86fcfdc 100644 --- a/docs/services/parity-testing.md +++ b/docs/services/parity-testing.md @@ -96,7 +96,7 @@ A step that uploads a file describes its multipart body as `parts` instead of a Recording and replaying encode the parts in order with one fixed boundary, `tide.MultipartBoundary`, so the reference backend and the port receive the same bytes. Set `BaseDir` on `tide.RecordConfig` and `tide.ReplayConfig` to the fixture directory the files are read from. `tide.LoadFlow` refuses a fixture whose part file is missing or no longer matches its `sha256`, and the file bytes are never copied into the YAML. -Upload responses carry URLs with random parts: the partition and disk name of the original, and the file id in a thumbnail name. Under `url` and `thumb_url` keys, the normalizer checks the WinterCMS shape below the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, `tide.DefaultUploadPrefix` by default) and masks only the random parts. The prefix, the thumbnail size and mode and the extension are still compared, so a port that serves `/storage/uploads/...` or makes 100 by 100 thumbnails instead of 200 by 200 fails the diff. +Upload responses carry URLs with random parts: the partition and disk name of the original, and the file id in a thumbnail name. Under the `url` key and every key ending in `_url` (`thumb_url`, `cover_url`), the normalizer checks the WinterCMS shape below the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, `tide.DefaultUploadPrefix` by default) and masks only the random parts. The prefix, the thumbnail size and mode and the extension are still compared, so a port that serves `/storage/uploads/...` or makes 100 by 100 thumbnails instead of 200 by 200 fails the diff. ## The parity commands @@ -149,7 +149,7 @@ exchanges: body: '{"id":7}' ``` -Only the compared request headers are kept (`tide.UpstreamCompareHeaders`: User-Agent, Accept, Content-Type, Authorization, X-Api-Key and the model vendors' version headers). A multipart request keeps an ordered list of `parts` instead of a body, with the SHA-256 of each file instead of its bytes, and a JSON string longer than 1024 characters that is base64 (an uploaded photo, for example) is stored as `{{sha256:}}`. +Only the compared request headers are kept (`tide.UpstreamCompareHeaders`: User-Agent, Accept, Content-Type, Authorization, X-Api-Key and the model vendors' version headers). A multipart request keeps an ordered list of `parts` instead of a body, with the SHA-256 of each file instead of its bytes, and a JSON string longer than 1024 characters that is base64 (an uploaded photo, for example) is stored as `{{sha256:}}`. A response body that is not valid UTF-8, such as a cover image, is written as a YAML `!!binary` scalar, left unmasked and replayed byte for byte; a script file can answer with such a body the same way (`body: !!binary `). ### Recording through the proxy diff --git a/modules/tide/README.md b/modules/tide/README.md index f5b4c7e..10c2ab6 100644 --- a/modules/tide/README.md +++ b/modules/tide/README.md @@ -16,11 +16,11 @@ HTTP parity toolkit that records request and response fixtures from a reference - Recording proxy: `tide.NewProxy` builds a reverse proxy that only binds to and forwards to loopback addresses, groups traffic into named sessions (from the `tide.SessionHeader` request header or a default session) and writes one fixture per complete session on `tide.Proxy.Flush`. - Capture rules: `tide.Rules` (loaded with `tide.LoadRules`) decide which request and response headers are kept per route and which values are captured into variables, from response JSON paths, headers, redirect query strings or form fields. - Variables: `tide.Store` holds captured values such as tokens and IDs in a mode-0600 file, `tide.Store.Expand` substitutes `{{name}}` placeholders before a request is sent, and `tide.ScrubStep` puts placeholders back into fixtures. Scrubbing fails when a step still holds an unclassified token- or password-shaped value, so credentials do not leak into committed fixtures. -- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. A `Content-Disposition` header is compared after masking its `YYYY-MM-DD` dates on both sides, so a download named after the day it was made, such as `attachment; filename=export-2026-09-17.csv`, replays on a later day; each masked token must be a real calendar date, and a different name, an invalid date or a date on one side only is still a difference. Uploaded-file URLs under `url` and `thumb_url` keys are compared by shape: under the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, default `tide.DefaultUploadPrefix`, `/storage/app/uploads/public`) an original must be `/xxx/yyy/zzz/` with the partition taken from the disk name, and a thumbnail `/xxx/yyy/zzz/thumb______.`. The partition, disk name and file id are masked; the prefix, size, offsets, mode and extension stay, so a thumbnail of another size or an upload URL under another prefix is still a difference. +- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. A `Content-Disposition` header is compared after masking its `YYYY-MM-DD` dates on both sides, so a download named after the day it was made, such as `attachment; filename=export-2026-09-17.csv`, replays on a later day; each masked token must be a real calendar date, and a different name, an invalid date or a date on one side only is still a difference. Uploaded-file URLs under the `url` key and every key ending in `_url` (`thumb_url`, `cover_url`) are compared by shape: under the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, default `tide.DefaultUploadPrefix`, `/storage/app/uploads/public`) an original must be `/xxx/yyy/zzz/` with the partition taken from the disk name, and a thumbnail `/xxx/yyy/zzz/thumb______.`. The partition, disk name and file id are masked; the prefix, size, offsets, mode and extension stay, so a thumbnail of another size or an upload URL under another prefix is still a difference. - Fake Centrifugo: `tide.NewCentrifugoRecorder` returns an `http.Handler` that records every POST to a path ending in `/publish` or `/broadcast` as a `tide.Publication` (method, path, whether `Authorization: apikey ` carried the configured key, JSON body) and answers `{"result":{}}`. Paths ending in `/presence` answer `{"result":{"presence":{}}}`, `/unsubscribe` and `/info` answer `{"result":{}}`, anything else is 404. Bodies are capped at `tide.MaxPublicationBody` (1 MiB). The API key is only compared, never stored. `tide.CentrifugoRecorder.ListenAndServe` binds loopback addresses only, like the recording proxy. - Broadcast goldens: `tide.RecordBroadcasts` runs a flow against a loopback reference backend whose Centrifugo API URL points at a recorder on `tide.DefaultCentrifugoListen` (`127.0.0.1:8424`). With `tide.BroadcastConfig` `Step` set, earlier steps run as setup and only that step's publications are kept. The result is a `tide.BroadcastGolden`, written with `tide.WriteBroadcastGolden` (which refuses token-shaped bodies) and read strictly with `tide.LoadBroadcastGolden`. A golden with `pending` set is recorded but not yet asserted. - Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. Carbon `+00:00` values of `*_at` keys anywhere under `$.data.payload.album` become `"{{datetime}}"`. A notification publication's own row fields are masked too: a Carbon `+00:00` `$.data.payload.created_at` becomes `"{{datetime}}"`, and a positive integer `$.data.payload.id` that no `id:*` variable names becomes a bare `{{id}}` (a captured one keeps `{{id:name}}`); a date of another shape is left as it is, so a format change shows as a difference. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`. -- Upstream sidecars: `tide.UpstreamSidecar` (version 1) holds the ordered `tide.UpstreamExchange` values a backend sent to outside services during one fixture, stored as `.upstream.yaml` (`tide.UpstreamPath`). `tide.LoadUpstream` reads one strictly (a missing file wraps `fs.ErrNotExist`). `tide.WriteUpstream` masks every variable value as `{{name}}`, replaces a JSON string longer than 1024 characters that decodes as base64 (also behind a `data:` URL prefix) with `{{sha256:}}`, and refuses an Authorization or X-Api-Key value that no variable masks. A multipart request keeps ordered `tide.UpstreamPart` values: plain fields by value, files by SHA-256. Bodies are capped at `tide.MaxUpstreamBody` (32 MiB). +- Upstream sidecars: `tide.UpstreamSidecar` (version 1) holds the ordered `tide.UpstreamExchange` values a backend sent to outside services during one fixture, stored as `.upstream.yaml` (`tide.UpstreamPath`). `tide.LoadUpstream` reads one strictly (a missing file wraps `fs.ErrNotExist`). `tide.WriteUpstream` masks every variable value as `{{name}}`, replaces a JSON string longer than 1024 characters that decodes as base64 (also behind a `data:` URL prefix) with `{{sha256:}}`, and refuses an Authorization or X-Api-Key value that no variable masks. A multipart request keeps ordered `tide.UpstreamPart` values: plain fields by value, files by SHA-256. A response body that is not valid UTF-8, such as an image, is written as a YAML `!!binary` scalar, never masked and replayed byte for byte. Bodies are capped at `tide.MaxUpstreamBody` (32 MiB). - Upstream fake: `tide.NewUpstreamFake` returns an `http.RoundTripper` that answers each request from the next recorded exchange without dialing and asserts it: method, scheme, host, path, query (order-insensitive), the `tide.UpstreamCompareHeaders` with placeholders expanded from a `tide.Store`, and the body (JSON semantically, multipart part by part, hashed base64 by digest). Mismatch messages never print credential header values. `tide.UpstreamFake.Verify` joins every mismatch, extra request and unconsumed exchange. Hand the fake to code under test with `fetchguard.WithTransport`. - Upstream recording proxy: `tide.NewUpstreamProxy` builds a loopback-only CONNECT proxy (`tide.DefaultUpstreamProxyListen`, `127.0.0.1:8425`) that terminates TLS with per-host certificates signed by a local parity CA (`tide.EnsureParityCA`: ECDSA P-256, certificate mode 0644, key mode 0600, standard library crypto only). In script mode it answers from a `tide.UpstreamScript` (the first unused entry whose method, host and path match; no match answers 599 and fails the recording); in forward mode it sends each request once to the vendor through a `fetchguard.Client` in `fetchguard.PublicOnlyMode`. `tide.UpstreamProxy.Flush` writes the sidecar with `tide.WriteUpstream` and refuses when any request failed. The CA directory and the vars file must be outside the sidecar's directory. - Manifests: `tide.Manifest` lists routes with auth groups, a pending or ported status, cases and fixture paths; `tide.RecordManifest` records missing cases in batches of at most `tide.MaxBatch`, and `tide.ReplayManifest` replays every recorded case into a `tide.Coverage` table. diff --git a/modules/tide/normalize.go b/modules/tide/normalize.go index 7691086..21741b5 100644 --- a/modules/tide/normalize.go +++ b/modules/tide/normalize.go @@ -73,7 +73,7 @@ func maskLeaf(path string, val any, step Step, opts maskOptions, diffs *[]Diff) if key == "slug" || disabledPath(step, path, key) { return val } - if key == "url" || key == "thumb_url" { + if isUploadURLKey(key) { if s, ok := val.(string); ok { return maskUploadURL(path, s, opts.prefix(), diffs) } @@ -101,6 +101,14 @@ func maskLeaf(path string, val any, step Step, opts maskOptions, diffs *[]Diff) return val } +// isUploadURLKey reports the keys whose string values are compared as +// uploaded-file URLs: url and every key ending in _url (thumb_url, +// cover_url). A value outside the uploads prefix that does not look like an +// upload is compared as it is. +func isUploadURLKey(key string) bool { + return key == "url" || strings.HasSuffix(key, "_url") +} + func maskDate(path string, val any, diffs *[]Diff) any { if val == nil { return nil diff --git a/modules/tide/normalize_upload_test.go b/modules/tide/normalize_upload_test.go index 4d22962..b01e8e0 100644 --- a/modules/tide/normalize_upload_test.go +++ b/modules/tide/normalize_upload_test.go @@ -30,6 +30,10 @@ func TestNormalizeMaskEdges(t *testing.T) { `{"url":"/files//.jpg"}`, 0}, {"upload under another prefix", `{"url":"/elsewhere/6ab/f82/1c3/6abf821c3d4e5f6a7b8c9d.png"}`, "", Step{}, "", 1}, {"plain url kept", `{"url":"https://example.com/a.png"}`, "", Step{}, `{"url":"https://example.com/a.png"}`, 0}, + {"any _url key", `{"cover_url":"` + orig + `","master_url":"https://api.example.test/masters/1"}`, "", Step{}, + `{"cover_url":"` + DefaultUploadPrefix + `//.png","master_url":"https://api.example.test/masters/1"}`, 0}, + {"_url key under another prefix", `{"cover_url":"/elsewhere/6ab/f82/1c3/6abf821c3d4e5f6a7b8c9d.png"}`, "", Step{}, "", 1}, + {"_urls list not masked", `{"cover_urls":["` + orig + `"]}`, "", Step{}, `{"cover_urls":["` + orig + `"]}`, 0}, {"non-string url kept", `{"url":5,"thumb_url":null}`, "", Step{}, `{"thumb_url":null,"url":5}`, 0}, {"slug never masked", `{"slug":"abc_id","id":7}`, "", Step{}, `{"id":"","slug":"abc_id"}`, 0}, {"collection key", `{"collection_key":"e53f1bd22f01bbe8268079f016301ad5","client_id":null}`, "", Step{}, `{"client_id":null,"collection_key":""}`, 0}, diff --git a/modules/tide/upstream.go b/modules/tide/upstream.go index eee535f..4271c37 100644 --- a/modules/tide/upstream.go +++ b/modules/tide/upstream.go @@ -19,6 +19,7 @@ import ( "slices" "strings" "sync" + "unicode/utf8" "github.com/goccy/go-yaml" ) @@ -71,13 +72,49 @@ const MaxUpstreamBody = 32 << 20 // stored as a {{sha256:}} placeholder. const upstreamHashMin = 1024 -// UpstreamResponse is the recorded vendor answer the fake replays. +// UpstreamResponse is the recorded vendor answer the fake replays. A body +// that is not valid UTF-8 (an image, for example) is written as a YAML +// !!binary scalar (base64) and read back byte for byte. type UpstreamResponse struct { Status int `yaml:"status"` Headers map[string]string `yaml:"headers,omitempty"` Body string `yaml:"body,omitempty"` } +// upstreamResponseYAML is UpstreamResponse as written: the body is a plain +// string, a binaryScalar, or nil when empty. +type upstreamResponseYAML struct { + Status int `yaml:"status"` + Headers map[string]string `yaml:"headers,omitempty"` + Body any `yaml:"body,omitempty"` +} + +// MarshalYAML writes a non-UTF-8 body as !!binary so the bytes survive the +// round trip. +func (r UpstreamResponse) MarshalYAML() (any, error) { + return upstreamResponseYAML{Status: r.Status, Headers: r.Headers, Body: yamlBody(r.Body)}, nil +} + +// binaryScalar is a byte string written as a YAML !!binary scalar. +type binaryScalar string + +// MarshalYAML implements yaml.BytesMarshaler. +func (b binaryScalar) MarshalYAML() ([]byte, error) { + return []byte("!!binary " + base64.StdEncoding.EncodeToString([]byte(b))), nil +} + +// yamlBody is the value written for a body: nil when empty, a binaryScalar +// when it is not valid UTF-8, else the string. +func yamlBody(body string) any { + switch { + case body == "": + return nil + case !utf8.ValidString(body): + return binaryScalar(body) + } + return body +} + // UpstreamCompareHeaders are the request headers the fake asserts. A header // is compared when either the recorded or the sent request carries it. var UpstreamCompareHeaders = []string{ @@ -305,9 +342,13 @@ func (f *UpstreamFake) response(r UpstreamResponse, req *http.Request) (*http.Re } h.Set(k, ev) } - body, err := f.store.Expand(r.Body) - if err != nil { - return nil, fmt.Errorf("response body: %w", err) + body := r.Body + if utf8.ValidString(body) { + // A binary body is replayed byte for byte, never expanded. + var err error + if body, err = f.store.Expand(body); err != nil { + return nil, fmt.Errorf("response body: %w", err) + } } return &http.Response{ Status: fmt.Sprintf("%d %s", r.Status, http.StatusText(r.Status)), @@ -598,7 +639,11 @@ func maskUpstream(s UpstreamSidecar, store *Store) (UpstreamSidecar, error) { } resp := ex.Response resp.Headers = scrubMap(resp.Headers, pairs, true) - resp.Body = replaceAll(resp.Body, pairs, false) + if utf8.ValidString(resp.Body) { + // A binary body (an image) is kept byte for byte: masking a + // short variable value inside it would corrupt the file. + resp.Body = replaceAll(resp.Body, pairs, false) + } out.Exchanges[i] = UpstreamExchange{Request: req, Response: resp} } return out, nil diff --git a/modules/tide/upstream_test.go b/modules/tide/upstream_test.go index 9a19cb2..ab88255 100644 --- a/modules/tide/upstream_test.go +++ b/modules/tide/upstream_test.go @@ -286,3 +286,51 @@ func TestUpstreamFakeHashesBase64Bodies(t *testing.T) { t.Fatalf("mismatch message not clipped: %d bytes", len(err.Error())) } } + +// TestUpstreamBinaryBodyRoundTrip pins that a response body that is not +// valid UTF-8 (an image) is written as !!binary, kept unmasked, read back +// byte for byte and replayed by the fake unchanged. +func TestUpstreamBinaryBodyRoundTrip(t *testing.T) { + store := upstreamTestStore(t) + store.Set("id:short", "7") + img := "\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01 7 {{ example-token-value \xff\xd9" + s := UpstreamSidecar{Version: 1, Exchanges: []UpstreamExchange{{ + Request: UpstreamRequest{Method: "GET", URL: "https://img.example.test/cover.jpg"}, + Response: UpstreamResponse{Status: 200, Headers: map[string]string{"Content-Type": "image/jpeg"}, Body: img}, + }, { + Request: UpstreamRequest{Method: "GET", URL: "https://api.example.test/v1/me"}, + Response: UpstreamResponse{Status: 200, Body: `{"token":"example-token-value"}`}, + }}} + path := filepath.Join(t.TempDir(), "bin.upstream.yaml") + if err := WriteUpstream(path, s, store); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(raw, []byte("body: !!binary "+base64.StdEncoding.EncodeToString([]byte(img)))) { + t.Fatalf("binary body not written as !!binary:\n%s", raw) + } + if !bytes.Contains(raw, []byte(`{{secret:example-token}}`)) { + t.Fatalf("text body not masked:\n%s", raw) + } + got, err := LoadUpstream(path) + if err != nil { + t.Fatal(err) + } + if got.Exchanges[0].Response.Body != img { + t.Fatalf("binary body changed: %q", got.Exchanges[0].Response.Body) + } + fake := NewUpstreamFake(got, store) + req, _ := http.NewRequest(http.MethodGet, "https://img.example.test/cover.jpg", nil) + res, err := fake.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + _, _ = buf.ReadFrom(res.Body) + if buf.String() != img { + t.Fatalf("replayed body %q", buf.String()) + } +}