From e06e0cc8bfe883be409f832281e1d341bd37d478 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Fri, 2 Oct 2026 11:33:42 +0200 Subject: [PATCH] feat(12-01): record multipart uploads and match Winter upload URLs - attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the image size from the header before decoding - tide requests carry multipart parts (files beside the fixture pinned by sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive byte-identical bodies - tide masks the random partition, disk name and file id of url/thumb_url upload URLs while still diffing prefix, size, mode and extension, and NormalizePublications masks Carbon dates in the published album --- docs/database/attachments.md | 4 +- docs/services/parity-testing.md | 23 ++- docs/services/storage.md | 14 +- go.mod | 2 +- go.sum | 3 +- modules/lagoon/README.md | 7 +- modules/lagoon/attach/thumb.go | 42 ++++- modules/lagoon/attach/url_test.go | 107 +++++++++++ modules/tide/README.md | 13 +- modules/tide/centrifugo_golden.go | 6 + modules/tide/diff.go | 8 +- modules/tide/fixture.go | 36 +++- modules/tide/flow.go | 23 ++- modules/tide/multipart.go | 201 ++++++++++++++++++++ modules/tide/multipart_test.go | 296 ++++++++++++++++++++++++++++++ modules/tide/normalize.go | 77 +++++++- modules/tide/record.go | 4 + modules/tide/replay.go | 12 +- modules/tide/variables.go | 26 +++ 19 files changed, 870 insertions(+), 34 deletions(-) create mode 100644 modules/lagoon/attach/url_test.go create mode 100644 modules/tide/multipart.go create mode 100644 modules/tide/multipart_test.go diff --git a/docs/database/attachments.md b/docs/database/attachments.md index d286b7b..d31306b 100644 --- a/docs/database/attachments.md +++ b/docs/database/attachments.md @@ -82,7 +82,9 @@ fmt.Println(url) // /storage/uploads/5f1/d0c/2e9/thumb_7_200_200_0_0_crop.jpg ``` -URLs start with `storage.uploads.public_path_prefix` (`/storage/uploads` by default). `attach.StaticHandler` serves originals and thumbnails under that prefix; `attach.StaticHandlerPublic` does the same and answers 404 for a row whose `is_public` flag is false. Mount the gated handler when a bucket holds any private file. +URLs start with `storage.uploads.public_path_prefix` (`/storage/uploads` by default). `attach.File.URL` returns the URL of the original, the path WinterCMS's `File::getPath()` returns, and `attach.PublicURL` the URL of any blob key; [Storage](../services/storage.md#the-wintercms-layout) shows the configuration that reproduces WinterCMS's URLs exactly. + +Originals in JPEG, PNG, GIF and WebP can be thumbnailed. The thumbnailer cannot write WebP, so the thumbnail of a `.webp` original holds JPEG bytes under the original's `.webp` name, and it is stored with the `image/jpeg` content type. Before decoding, the thumbnailer reads the image size from the file header and refuses an image larger than 4096 by 4096 pixels. `attach.StaticHandler` serves originals and thumbnails under that prefix; `attach.StaticHandlerPublic` does the same and answers 404 for a row whose `is_public` flag is false. Mount the gated handler when a bucket holds any private file. > [!WARNING] > Serve uploads from a separate origin, or at least never mount the ungated handler on the application's own origin. An uploaded file served with its own content type from the API's origin can run script in that origin. diff --git a/docs/services/parity-testing.md b/docs/services/parity-testing.md index be0ccdb..bdfe26a 100644 --- a/docs/services/parity-testing.md +++ b/docs/services/parity-testing.md @@ -77,6 +77,27 @@ for _, target := range []string{port.URL, broken.URL} { The first port returns other IDs and timestamps and passes; the second changed a title and fails with the JSON path of the difference. A difference makes `tide.ReplayFlow` return a `tide.MismatchError` that carries the full `tide.Result`. +## Uploads + +A step that uploads a file describes its multipart body as `parts` instead of a `body`. Each `tide.Part` is a text field with a `value`, or a file field whose bytes live in a file beside the fixture: + +```yaml + request: + method: POST + path: /api/blog/posts/{{id:post}}/photos + parts: + - name: caption + value: Cover + - name: file + file: files/cover.png + content_type: image/png + sha256: +``` + +Recording and replaying encode the parts in order with one fixed boundary, `tide.MultipartBoundary`, so the reference backend and the port receive the same bytes. Set `BaseDir` on `tide.RecordConfig` and `tide.ReplayConfig` to the fixture directory the files are read from. `tide.LoadFlow` refuses a fixture whose part file is missing or no longer matches its `sha256`, and the file bytes are never copied into the YAML. + +Upload responses carry URLs with random parts: the partition and disk name of the original, and the file id in a thumbnail name. Under `url` and `thumb_url` keys, the normalizer checks the WinterCMS shape below the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, `tide.DefaultUploadPrefix` by default) and masks only the random parts. The prefix, the thumbnail size and mode and the extension are still compared, so a port that serves `/storage/uploads/...` or makes 100 by 100 thumbnails instead of 200 by 200 fails the diff. + ## The parity commands The `summer` CLI wraps tide. Run the reference backend and the port on loopback addresses: @@ -100,6 +121,6 @@ Realtime side effects are part of the contract too. `summer parity:broadcasts` r summer parity:broadcasts --flow testdata/broadcasts/flows/post-lifecycle.yaml --step delete --name deleted --target http://127.0.0.1:8000 --vars /tmp/parity/vars.yaml --out testdata/broadcasts/deleted.yaml ``` -Point the reference backend's Centrifugo API URL at the recorder (`127.0.0.1:8424` by default). `--step` keeps only the publications of one step, running the earlier steps as setup. Timestamps, the actor and captured IDs are masked (`tide.NormalizePublications`), so the Go port's publications, recorded the same way, compare with `tide.DiffPublications`. A golden with `--pending` set is recorded but not yet asserted. +Point the reference backend's Centrifugo API URL at the recorder (`127.0.0.1:8424` by default). `--step` keeps only the publications of one step, running the earlier steps as setup. Timestamps, the actor, the `*_at` dates inside a published album and captured IDs are masked (`tide.NormalizePublications`), so the Go port's publications, recorded the same way, compare with `tide.DiffPublications`. A golden with `--pending` set is recorded but not yet asserted. On the Go side, the memory realtime driver records publications the same way in tests; see [Realtime](realtime.md). diff --git a/docs/services/storage.md b/docs/services/storage.md index 5f1c365..44951a9 100644 --- a/docs/services/storage.md +++ b/docs/services/storage.md @@ -23,7 +23,19 @@ uploads: public_path_prefix: /storage/uploads ``` -Files are laid out as WinterCMS lays out its uploads disk, so a copy of a WinterCMS `storage/app/uploads/public` directory can serve as the bucket after a port. `public_path_prefix` is the URL prefix that file and thumbnail URLs start with. +Files are laid out as WinterCMS lays out its uploads disk, so a copy of a WinterCMS `storage/app/uploads/public` directory can serve as the bucket after a port. `public_path_prefix` is the URL prefix that file and thumbnail URLs start with. `attach.PublicURL` builds the URL of any blob key from it, and `attach.File.URL` the URL of an original. + +### The WinterCMS layout + +A port whose clients already store or compare upload URLs keeps WinterCMS's URLs too. WinterCMS serves public uploads from `storage/app/uploads/public` under the URL path `/storage/app/uploads/public` (`cms.storage.uploads.path` plus `/public`). Root the bucket at that directory and use the same prefix: + +```yaml +uploads: + bucket_url: file://./storage/app/uploads/public + public_path_prefix: /storage/app/uploads/public +``` + +An original then has the URL `/storage/app/uploads/public//`, as WinterCMS's `File::getPath()` returns, and a 200 by 200 cropped thumbnail `/storage/app/uploads/public//thumb__200_200_0_0_crop.`, as `getThumb()` returns. The framework default stays `/storage/uploads`, so only an application that sets these keys changes its URLs. Application code gets the bucket with `app.Lookup[*blob.Bucket]()` and reads and writes it through the `gocloud.dev/blob` API. Model attachments, thumbnails and deleting files after commit are covered in [Attachments](../database/attachments.md). diff --git a/go.mod b/go.mod index ef6d3ad..6773cd0 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/yuin/goldmark v1.8.6 gocloud.dev v0.46.0 golang.org/x/crypto v0.57.0 + golang.org/x/image v0.46.0 golang.org/x/net v0.58.0 golang.org/x/term v0.46.0 golang.org/x/text v0.42.0 @@ -111,7 +112,6 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 21d6bf4..c1e4aee 100644 --- a/go.sum +++ b/go.sum @@ -320,8 +320,9 @@ gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= +golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= diff --git a/modules/lagoon/README.md b/modules/lagoon/README.md index 03f0f54..de60d50 100644 --- a/modules/lagoon/README.md +++ b/modules/lagoon/README.md @@ -24,7 +24,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he - Column types: `lagoon.Encrypted` stores AES-256-GCM ciphertext under a key derived from `app.key`, decrypts with previous keys during rotation, and always redacts itself in JSON and string output; `lagoon.Jsonable` stores JSON as TEXT and keeps SQL NULL distinct from an empty value. - Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns. - Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports. -- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), on-demand thumbnails through `attach.File.Thumb`, static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`). +- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`). ## Usage @@ -151,7 +151,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration { | `lagoon.WithSoftDeleteCascade` | Runs a cascade inside the parent delete transaction. | | `lagoon.RegisterJoinTable` | Registers a custom pivot model for a many-to-many field. | | `lagoon.DecryptLaravelPayload` | Decrypts a Laravel AES-256-CBC payload for data imports. | -| `attach.File` | The `system_files` row model. | +| `attach.File` | The `system_files` row model; `attach.File.URL` is the public URL of the original. | +| `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. | | `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. | | `attach.OpenBucket` | Opens the uploads bucket from config. | | `attach.Publish` | Stores the bucket on the `backpack.App`. | @@ -198,7 +199,7 @@ storage: - SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages). - Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`). -- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails). +- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding). - Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others. ## Testing diff --git a/modules/lagoon/attach/thumb.go b/modules/lagoon/attach/thumb.go index 93fac1c..5c9aa5d 100644 --- a/modules/lagoon/attach/thumb.go +++ b/modules/lagoon/attach/thumb.go @@ -1,6 +1,7 @@ package attach import ( + "bytes" "context" "fmt" "image" @@ -14,6 +15,10 @@ import ( "github.com/disintegration/imaging" "gocloud.dev/blob" + // The webp decoder lets image.DecodeConfig and File.Thumb read .webp + // originals. imaging cannot encode webp, so a webp thumbnail holds JPEG + // bytes under the original's .webp name (see defaultEncodeImage). + _ "golang.org/x/image/webp" ) const ( @@ -67,7 +72,11 @@ func fileExt(diskName string) string { return strings.ToLower(ext) } -func publicURL(key string) string { +// PublicURL returns the public URL of a blob key: storage.uploads. +// public_path_prefix and the key joined by exactly one slash. With the +// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix +// /storage/app/uploads/public) it is Winter's File::getPath() path. +func PublicURL(key string) string { prefix := strings.TrimRight(PublicPathPrefix(), "/") key = strings.TrimLeft(key, "/") if prefix == "" { @@ -76,6 +85,15 @@ func publicURL(key string) string { return prefix + "/" + key } +// URL returns the public URL of the original file, Winter's File::getPath(): +// PublicURL of BlobKey(DiskName). +func (f *File) URL() string { + if f == nil { + return "" + } + return PublicURL(BlobKey(f.DiskName)) +} + func defaultResizeImage(src image.Image, w, h int, mode string) image.Image { switch strings.ToLower(mode) { case "crop": @@ -133,21 +151,35 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st return "", fmt.Errorf("attach: thumb exists: %w", err) } if exists { - return publicURL(thumbKey), nil + return PublicURL(thumbKey), nil } origKey := part + f.DiskName r, err := bucket.NewReader(ctx, origKey, nil) if err != nil { return "", fmt.Errorf("attach: read original: %w", err) } - src, _, err := image.Decode(io.LimitReader(r, maxThumbSourceBytes)) + raw, err := io.ReadAll(io.LimitReader(r, maxThumbSourceBytes)) closeErr := r.Close() if err != nil { - return "", fmt.Errorf("attach: decode original: %w", err) + return "", fmt.Errorf("attach: read original: %w", err) } if closeErr != nil { return "", closeErr } + // Check the dimensions from the header before decoding the pixels, so + // a small file that declares a huge image is refused without + // allocating it. + cfg, _, err := image.DecodeConfig(bytes.NewReader(raw)) + if err != nil { + return "", fmt.Errorf("attach: decode original: %w", err) + } + if int64(cfg.Width)*int64(cfg.Height) > maxThumbSourcePixels { + return "", fmt.Errorf("attach: original image is too large") + } + src, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return "", fmt.Errorf("attach: decode original: %w", err) + } bounds := src.Bounds() if int64(bounds.Dx())*int64(bounds.Dy()) > maxThumbSourcePixels { return "", fmt.Errorf("attach: original image is too large") @@ -173,5 +205,5 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st } return "", closeErr } - return publicURL(thumbKey), nil + return PublicURL(thumbKey), nil } diff --git a/modules/lagoon/attach/url_test.go b/modules/lagoon/attach/url_test.go new file mode 100644 index 0000000..7e77d1e --- /dev/null +++ b/modules/lagoon/attach/url_test.go @@ -0,0 +1,107 @@ +package attach + +import ( + "bytes" + "image" + "image/jpeg" + "os" + "path/filepath" + "testing" + + "git.golem15.com/golem15/summercms/modules/compass" + "gocloud.dev/blob" +) + +// winterLayoutBucket opens a mem:// bucket with the WinterCMS public prefix +// and restores the framework default afterwards. +func winterLayoutBucket(t *testing.T) *blob.Bucket { + t.Helper() + dir := t.TempDir() + body := "uploads:\n bucket_url: \"mem://\"\n public_path_prefix: \"/storage/app/uploads/public\"\n" + if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Env: "development", Environ: []string{"SUMMER_ENV=development"}}) + if err != nil { + t.Fatal(err) + } + bucket, err := OpenBucket(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = bucket.Close() + setPublicPathPrefix(defaultPublicPathPrefix) + }) + return bucket +} + +func TestFileURLWinterLayout(t *testing.T) { + bucket := winterLayoutBucket(t) + // A WinterCMS disk name: uniqid('', true) without the dot, plus the + // extension. + f := &File{ID: 12, DiskName: "651a2b3c4d5e61234567.png"} + if got, want := f.URL(), "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want { + t.Fatalf("URL = %q, want %q", got, want) + } + if got, want := PublicURL("/651/a2b/3c4/x.png"), "/storage/app/uploads/public/651/a2b/3c4/x.png"; got != want { + t.Fatalf("PublicURL = %q, want %q", got, want) + } + if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), &blob.WriterOptions{ContentType: "image/png"}); err != nil { + t.Fatal(err) + } + thumb, err := f.Thumb(t.Context(), bucket, 200, 200, "crop") + if err != nil { + t.Fatal(err) + } + // Winter getThumbFilename: implode('_', [thumb, id, w, h, ox, oy, mode.ext]). + if want := "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png"; thumb != want { + t.Fatalf("thumb = %q, want %q", thumb, want) + } + var nilFile *File + if nilFile.URL() != "" { + t.Fatal("nil file URL must be empty") + } +} + +// webpFixture is a 16x12 lossless WebP: blue left half, red right half. +var webpFixture = []byte{ + 0x52, 0x49, 0x46, 0x46, 0x2a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, 0x50, 0x38, 0x4c, + 0x1d, 0x00, 0x00, 0x00, 0x2f, 0x0f, 0xc0, 0x02, 0x00, 0x0f, 0x70, 0x14, 0xfb, 0x53, 0xd0, 0x5e, + 0x88, 0x7b, 0xfe, 0x83, 0x07, 0x62, 0xc1, 0x64, 0xfe, 0xd2, 0xbd, 0x21, 0x44, 0xf4, 0x3f, 0x74, + 0x01, 0x00, +} + +func TestThumbWebP(t *testing.T) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(webpFixture)) + if err != nil { + t.Fatalf("DecodeConfig: %v", err) + } + if format != "webp" || cfg.Width != 16 || cfg.Height != 12 { + t.Fatalf("config = %s %dx%d", format, cfg.Width, cfg.Height) + } + bucket := winterLayoutBucket(t) + f := &File{ID: 5, DiskName: "abcdef123456789012345.webp"} + if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), webpFixture, &blob.WriterOptions{ContentType: "image/webp"}); err != nil { + t.Fatal(err) + } + url, err := f.Thumb(t.Context(), bucket, 200, 200, "crop") + if err != nil { + t.Fatalf("Thumb: %v", err) + } + if want := "/storage/app/uploads/public/abc/def/123/thumb_5_200_200_0_0_crop.webp"; url != want { + t.Fatalf("thumb url = %q, want %q", url, want) + } + raw, err := bucket.ReadAll(t.Context(), "abc/def/123/thumb_5_200_200_0_0_crop.webp") + if err != nil { + t.Fatal(err) + } + // imaging cannot encode webp: the thumb is JPEG bytes under the .webp name. + img, err := jpeg.Decode(bytes.NewReader(raw)) + if err != nil { + t.Fatalf("thumb is not JPEG: %v", err) + } + if b := img.Bounds(); b.Dx() != 200 || b.Dy() != 200 { + t.Fatalf("thumb size = %v", b) + } +} diff --git a/modules/tide/README.md b/modules/tide/README.md index 796ef06..de4164f 100644 --- a/modules/tide/README.md +++ b/modules/tide/README.md @@ -12,13 +12,14 @@ HTTP parity toolkit that records request and response fixtures from a reference - Flow fixtures: `tide.Flow` is a versioned, ordered list of `tide.Step` values, loaded strictly (unknown fields rejected) with `tide.LoadFlow` and `tide.ParseFlow`, and written atomically with `tide.SaveFlow` or `tide.SaveFlowExclusive`. Response bodies can live in sidecar files, confined to the fixture directory and checked against an optional SHA-256 digest. - Recording: `tide.RecordFlow` executes a spec flow against a target URL and fills in the responses, with a bounded body size (`tide.DefaultMaxBody`, 8 MiB). +- Multipart uploads: a request can carry `parts` (`tide.Part`: a text field with `value`, or a file field with `file`, `filename`, `content_type` and `sha256`) instead of a `body`. File bytes stay in files beside the fixture (for example `files/cover.png`, read from `tide.RecordConfig` or `tide.ReplayConfig` `BaseDir`) and are never inlined into the YAML. Parts are encoded in declaration order with one fixed boundary, `tide.MultipartBoundary`, so the reference backend and the port receive byte-identical bodies; the encoded Content-Type replaces a recorded multipart Content-Type. Variables expand in part values, never in file bytes. `tide.LoadFlow` fails when a part file is missing or its SHA-256 differs, naming the part, and a request with both `body` and `parts` is invalid. - Recording proxy: `tide.NewProxy` builds a reverse proxy that only binds to and forwards to loopback addresses, groups traffic into named sessions (from the `tide.SessionHeader` request header or a default session) and writes one fixture per complete session on `tide.Proxy.Flush`. - Capture rules: `tide.Rules` (loaded with `tide.LoadRules`) decide which request and response headers are kept per route and which values are captured into variables, from response JSON paths, headers, redirect query strings or form fields. - Variables: `tide.Store` holds captured values such as tokens and IDs in a mode-0600 file, `tide.Store.Expand` substitutes `{{name}}` placeholders before a request is sent, and `tide.ScrubStep` puts placeholders back into fixtures. Scrubbing fails when a step still holds an unclassified token- or password-shaped value, so credentials do not leak into committed fixtures. -- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. +- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. Uploaded-file URLs under `url` and `thumb_url` keys are compared by shape: under the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, default `tide.DefaultUploadPrefix`, `/storage/app/uploads/public`) an original must be `/xxx/yyy/zzz/` with the partition taken from the disk name, and a thumbnail `/xxx/yyy/zzz/thumb______.`. The partition, disk name and file id are masked; the prefix, size, offsets, mode and extension stay, so a thumbnail of another size or an upload URL under another prefix is still a difference. - Fake Centrifugo: `tide.NewCentrifugoRecorder` returns an `http.Handler` that records every POST to a path ending in `/publish` or `/broadcast` as a `tide.Publication` (method, path, whether `Authorization: apikey ` carried the configured key, JSON body) and answers `{"result":{}}`. Paths ending in `/presence` answer `{"result":{"presence":{}}}`, `/unsubscribe` and `/info` answer `{"result":{}}`, anything else is 404. Bodies are capped at `tide.MaxPublicationBody` (1 MiB). The API key is only compared, never stored. `tide.CentrifugoRecorder.ListenAndServe` binds loopback addresses only, like the recording proxy. - Broadcast goldens: `tide.RecordBroadcasts` runs a flow against a loopback reference backend whose Centrifugo API URL points at a recorder on `tide.DefaultCentrifugoListen` (`127.0.0.1:8424`). With `tide.BroadcastConfig` `Step` set, earlier steps run as setup and only that step's publications are kept. The result is a `tide.BroadcastGolden`, written with `tide.WriteBroadcastGolden` (which refuses token-shaped bodies) and read strictly with `tide.LoadBroadcastGolden`. A golden with `pending` set is recorded but not yet asserted. -- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`. +- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. Carbon `+00:00` values of `*_at` keys anywhere under `$.data.payload.album` become `"{{datetime}}"`; a date of another shape is left as it is, so a format change shows as a difference. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`. - Manifests: `tide.Manifest` lists routes with auth groups, a pending or ported status, cases and fixture paths; `tide.RecordManifest` records missing cases in batches of at most `tide.MaxBatch`, and `tide.ReplayManifest` replays every recorded case into a `tide.Coverage` table. ## Usage @@ -88,6 +89,12 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) { | `tide.LoadFlow` | Reads and validates a flow file. | | `tide.SaveFlow` | Writes a validated flow atomically. | | `tide.RecordFlow` | Executes a spec flow against a target and returns the recorded flow. | +| `tide.Request` | The outbound call of a step: method, path, query, headers and a `Body` or multipart `Parts`. | +| `tide.Part` | One multipart field: a text value, or a file beside the fixture pinned by its SHA-256. | +| `tide.MultipartBoundary` | The fixed boundary multipart request bodies are encoded with. | +| `tide.RecordConfig` | Target, client, body cap, vars store, capture rules and part-file directory for `tide.RecordFlow`. | +| `tide.ReplayConfig` | Target, client, body cap, vars store, fixture directory and upload URL prefix for `tide.ReplayFlow`. | +| `tide.DefaultUploadPrefix` | The default uploads URL prefix, `/storage/app/uploads/public`. | | `tide.ReplayFlow` | Replays a recorded flow against a target and diffs every step. | | `tide.Result` | Replay outcome: overall status and per-step `tide.StepResult` values. | | `tide.Diff` | One structural JSON or byte-level mismatch. | @@ -109,7 +116,7 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) { | `tide.BroadcastGolden` | Versioned broadcast golden: name, flow, optional pending reason, publications. | | `tide.LoadBroadcastGolden` | Reads a golden strictly and checks every body parses. | | `tide.WriteBroadcastGolden` | Writes a golden atomically, refusing token-shaped bodies. | -| `tide.NormalizePublications` | Masks timestamps, the actor and captured ids in publication bodies. | +| `tide.NormalizePublications` | Masks timestamps, the actor, album dates and captured ids in publication bodies. | | `tide.DiffPublications` | Structural diff of two publication lists. | | `tide.DefaultCentrifugoListen` | Default recorder address, `127.0.0.1:8424`. | | `tide.Coverage` | Recorded, passing, failing and unrecorded counts, with table rows and a summary line. | diff --git a/modules/tide/centrifugo_golden.go b/modules/tide/centrifugo_golden.go index 333aa18..6153c4e 100644 --- a/modules/tide/centrifugo_golden.go +++ b/modules/tide/centrifugo_golden.go @@ -164,6 +164,9 @@ var channelIDRe = regexp.MustCompile(`^((?:[a-z][a-z0-9_.-]*:)+)([0-9]+)$`) // when they have the ISO 8601 offset shape; // - $.data.payload.actor becomes "{{actor}}" when it is an object with // exactly user_id and name; +// - a *_at value anywhere under $.data.payload.album becomes +// "{{datetime}}" when it has Carbon's +00:00 shape; any other shape is +// left as it is, so a format change shows up as a diff; // - a number or string under an id, *_id or *_ids key that equals an id:* // variable of store becomes that {{id:name}} placeholder (bare for a // number, quoted for a string), and so does the numeric last segment of @@ -304,6 +307,9 @@ func (n *normalizer) walk(path, key string, v *onode) *onode { } return v case kindString: + if strings.HasPrefix(path, "$.data.payload.album.") && strings.HasSuffix(key, "_at") && carbonOffsetRe.MatchString(v.str) { + return &onode{kind: kindString, str: "{{datetime}}"} + } if key != "" && isIDKey(key) { if name, ok := n.lookup(v.str); ok { return &onode{kind: kindString, str: "{{" + name + "}}"} diff --git a/modules/tide/diff.go b/modules/tide/diff.go index 83716d0..4baee7f 100644 --- a/modules/tide/diff.go +++ b/modules/tide/diff.go @@ -13,11 +13,15 @@ import ( ) func compareBodies(want, got Response, step Step) []Diff { + return compareBodiesWith(want, got, step, maskOptions{}) +} + +func compareBodiesWith(want, got Response, step Step, opts maskOptions) []Diff { wantJSON := isJSONContentType(want.Headers) gotJSON := isJSONContentType(got.Headers) if wantJSON && gotJSON { - wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step) - gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step) + wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step, opts) + gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step, opts) diffs := append([]Diff{}, wantDiffs...) diffs = append(diffs, gotDiffs...) diffs = append(diffs, diffJSON(wantRaw, gotRaw)...) diff --git a/modules/tide/fixture.go b/modules/tide/fixture.go index 9f1a4f7..4be6bdb 100644 --- a/modules/tide/fixture.go +++ b/modules/tide/fixture.go @@ -25,13 +25,21 @@ func (b *Body) UnmarshalYAML(data []byte) error { return nil } -// LoadFlow reads and validates a version-1 YAML flow from path. +// LoadFlow reads and validates a version-1 YAML flow from path. Request +// part files are checked against their sha256, relative to path's directory. func LoadFlow(path string) (Flow, error) { raw, err := os.ReadFile(path) if err != nil { return Flow{}, fmt.Errorf("tide: read %s: %w", path, err) } - return ParseFlow(raw) + flow, err := ParseFlow(raw) + if err != nil { + return Flow{}, err + } + if err := verifyPartFiles(filepath.Dir(path), flow); err != nil { + return Flow{}, err + } + return flow, nil } // ParseFlow decodes a version-1 YAML flow, rejecting unknown fields. @@ -163,6 +171,30 @@ func writeRequest(b *strings.Builder, indent int, req Request) { } writeHeaders(b, indent, req.Headers) writeBody(b, indent, string(req.Body)) + writeParts(b, indent, req.Parts) +} + +func writeParts(b *strings.Builder, indent int, parts []Part) { + if len(parts) == 0 { + return + } + pad := strings.Repeat(" ", indent) + fmt.Fprintf(b, "%sparts:\n", pad) + inner := strings.Repeat(" ", indent+2) + for _, p := range parts { + fmt.Fprintf(b, "%s- name: %s\n", inner, encodeScalar(p.Name)) + for _, kv := range [][2]string{ + {"value", p.Value}, + {"file", p.File}, + {"filename", p.Filename}, + {"content_type", p.ContentType}, + {"sha256", p.SHA256}, + } { + if kv[1] != "" { + fmt.Fprintf(b, "%s %s: %s\n", inner, kv[0], encodeScalar(kv[1])) + } + } + } } func writeResponse(b *strings.Builder, indent int, resp Response) { diff --git a/modules/tide/flow.go b/modules/tide/flow.go index 997537d..f608181 100644 --- a/modules/tide/flow.go +++ b/modules/tide/flow.go @@ -42,6 +42,9 @@ type Request struct { Query string `yaml:"query,omitempty"` Headers map[string]string `yaml:"headers,omitempty"` Body Body `yaml:"body,omitempty"` + // Parts is a multipart/form-data body, encoded with MultipartBoundary + // when the step runs. A request has Body or Parts, never both. + Parts []Part `yaml:"parts,omitempty"` } // Response is the recorded or expected HTTP reply. @@ -73,21 +76,28 @@ type NormalizeRule struct { type Body string // RecordConfig injects the HTTP target, client and body bound for recording. +// BaseDir is the fixture directory that request part files are read from. type RecordConfig struct { Target string Client *http.Client MaxBody int64 Store *Store Rules Rules + BaseDir string } // ReplayConfig injects the HTTP target, client and body bound for replay. +// BaseDir is the fixture directory that body_file sidecars and request part +// files are read from. +// UploadPrefix is the public URL prefix of uploaded files whose url and +// thumb_url values are compared by shape; empty means DefaultUploadPrefix. type ReplayConfig struct { - Target string - Client *http.Client - MaxBody int64 - Store *Store - BaseDir string + Target string + Client *http.Client + MaxBody int64 + Store *Store + BaseDir string + UploadPrefix string } // Result is the outcome of replaying a flow. @@ -169,6 +179,9 @@ func validateFlow(flow Flow) error { if err := validateSidecar(step.Response.BodyFile); err != nil { return fmt.Errorf("tide: step %s: %w", step.ID, err) } + if err := validateParts(step.Request); err != nil { + return fmt.Errorf("tide: step %s: %w", step.ID, err) + } } return nil } diff --git a/modules/tide/multipart.go b/modules/tide/multipart.go new file mode 100644 index 0000000..7695bfc --- /dev/null +++ b/modules/tide/multipart.go @@ -0,0 +1,201 @@ +package tide + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "mime" + "mime/multipart" + "net/textproto" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +// MultipartBoundary is the fixed boundary every multipart request body is +// encoded with, so a recording against one backend and a replay against +// another send byte-identical bodies. +const MultipartBoundary = "SummerTideMultipartBoundary7MA4YWxkTrZu0gW" + +// Part is one field of a multipart/form-data request. A text field sets +// Value; a file field sets File, a path relative to the fixture directory +// (for example files/cover.png), whose bytes are never inlined into the +// YAML, with SHA256 pinning its content. Filename defaults to the base name +// of File and ContentType to application/octet-stream. +type Part struct { + Name string `yaml:"name"` + Value string `yaml:"value,omitempty"` + File string `yaml:"file,omitempty"` + Filename string `yaml:"filename,omitempty"` + ContentType string `yaml:"content_type,omitempty"` + SHA256 string `yaml:"sha256,omitempty"` +} + +var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`) + +// validateParts checks the shape of a request's parts without reading files. +func validateParts(req Request) error { + if len(req.Parts) == 0 { + return nil + } + if req.Body != "" { + return fmt.Errorf("request has both body and parts") + } + for i, p := range req.Parts { + if strings.TrimSpace(p.Name) == "" { + return fmt.Errorf("parts[%d] is missing name", i) + } + if p.File == "" { + if p.Filename != "" || p.ContentType != "" || p.SHA256 != "" { + return fmt.Errorf("part %q has file attributes but no file", p.Name) + } + continue + } + if p.Value != "" { + return fmt.Errorf("part %q has both value and file", p.Name) + } + if err := validateSidecar(p.File); err != nil { + return fmt.Errorf("part %q: %w", p.Name, err) + } + if !sha256Hex.MatchString(p.SHA256) { + return fmt.Errorf("part %q needs the sha256 of its file", p.Name) + } + } + return nil +} + +// verifyPartFiles checks that every part file of flow exists under base and +// matches its sha256. +func verifyPartFiles(base string, flow Flow) error { + for _, step := range flow.Steps { + for _, p := range step.Request.Parts { + if p.File == "" { + continue + } + if _, err := readPartFile(base, p); err != nil { + return fmt.Errorf("tide: step %s: %w", step.ID, err) + } + } + } + return nil +} + +func readPartFile(base string, p Part) ([]byte, error) { + if err := validateSidecar(p.File); err != nil { + return nil, fmt.Errorf("part %q: %w", p.Name, err) + } + full := p.File + if base != "" { + full = filepath.Join(base, p.File) + } + resolved, err := resolvePath(full) + if err != nil { + return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err) + } + if base != "" { + root, err := resolvePath(base) + if err != nil { + return nil, fmt.Errorf("fixture dir: %w", err) + } + if resolved != root && !strings.HasPrefix(resolved, root+string(os.PathSeparator)) { + return nil, fmt.Errorf("part %q file %q escapes the fixture directory", p.Name, p.File) + } + } + raw, err := os.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err) + } + sum := sha256.Sum256(raw) + if !strings.EqualFold(hex.EncodeToString(sum[:]), p.SHA256) { + return nil, fmt.Errorf("part %q file %s sha256 mismatch", p.Name, p.File) + } + return raw, nil +} + +var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) + +// encodeParts writes parts in declaration order as a multipart/form-data +// body with MultipartBoundary and returns the body and its Content-Type. +func encodeParts(base string, parts []Part) ([]byte, string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + if err := w.SetBoundary(MultipartBoundary); err != nil { + return nil, "", err + } + marker := []byte(MultipartBoundary) + for _, p := range parts { + h := make(textproto.MIMEHeader) + var content []byte + if p.File == "" { + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, quoteEscaper.Replace(p.Name))) + content = []byte(p.Value) + } else { + raw, err := readPartFile(base, p) + if err != nil { + return nil, "", err + } + filename := p.Filename + if filename == "" { + filename = path.Base(filepath.ToSlash(p.File)) + } + ct := p.ContentType + if ct == "" { + ct = "application/octet-stream" + } + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, quoteEscaper.Replace(p.Name), quoteEscaper.Replace(filename))) + h.Set("Content-Type", ct) + content = raw + } + if bytes.Contains(content, marker) { + return nil, "", fmt.Errorf("part %q contains the multipart boundary", p.Name) + } + pw, err := w.CreatePart(h) + if err != nil { + return nil, "", err + } + if _, err := pw.Write(content); err != nil { + return nil, "", err + } + } + if err := w.Close(); err != nil { + return nil, "", err + } + return buf.Bytes(), w.FormDataContentType(), nil +} + +// prepareRequest encodes a request's parts into its body. The multipart +// Content-Type replaces a recorded multipart Content-Type (whose boundary +// would be stale) and is added when none is recorded; a recorded non- +// multipart Content-Type is kept as it is. +func prepareRequest(req Request, base string) (Request, error) { + if len(req.Parts) == 0 { + return req, nil + } + if err := validateParts(req); err != nil { + return Request{}, err + } + body, ct, err := encodeParts(base, req.Parts) + if err != nil { + return Request{}, err + } + out := req + out.Body = Body(body) + out.Headers = make(map[string]string, len(req.Headers)+1) + set := false + for k, v := range req.Headers { + if strings.EqualFold(k, "Content-Type") { + set = true + if media, _, err := mime.ParseMediaType(v); err != nil || strings.HasPrefix(media, "multipart/") { + v = ct + } + } + out.Headers[k] = v + } + if !set { + out.Headers["Content-Type"] = ct + } + return out, nil +} diff --git a/modules/tide/multipart_test.go b/modules/tide/multipart_test.go new file mode 100644 index 0000000..8845d99 --- /dev/null +++ b/modules/tide/multipart_test.go @@ -0,0 +1,296 @@ +package tide + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// multipartFixture writes files/cover.png under a temp fixture directory and +// returns the directory and a one-step upload flow that references it. +func multipartFixture(t *testing.T) (string, Flow) { + t.Helper() + dir := t.TempDir() + content := []byte("\x89PNG\r\n\x1a\nfake image bytes") + if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), content, 0o644); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(content) + flow := Flow{ + Version: CurrentVersion, + Name: "upload", + Steps: []Step{{ + ID: "upload", + Request: Request{ + Method: "POST", + Path: "/posts/1/photos", + Headers: map[string]string{"Content-Type": "multipart/form-data; boundary=recorded-elsewhere"}, + Parts: []Part{ + {Name: "title", Value: "Cover {{id:post}}"}, + {Name: "file", File: "files/cover.png", ContentType: "image/png", SHA256: hex.EncodeToString(sum[:])}, + }, + }, + }}, + } + return dir, flow +} + +type rawCapture struct { + mu sync.Mutex + bodies [][]byte + types []string +} + +func (c *rawCapture) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + c.mu.Lock() + c.bodies = append(c.bodies, raw) + c.types = append(c.types, r.Header.Get("Content-Type")) + c.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + } +} + +func TestMultipartRecordReplaySendsIdenticalBytes(t *testing.T) { + dir, flow := multipartFixture(t) + capture := &rawCapture{} + srv := httptest.NewServer(capture.handler(t)) + t.Cleanup(srv.Close) + store := mustMemoryStore() + store.Set("id:post", "7") + + recorded, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}) + if err != nil { + t.Fatalf("record: %v", err) + } + if _, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil { + t.Fatalf("second record: %v", err) + } + if _, err := ReplayFlow(t.Context(), recorded, ReplayConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil { + t.Fatalf("replay: %v", err) + } + if len(capture.bodies) != 3 { + t.Fatalf("requests = %d", len(capture.bodies)) + } + for i := 1; i < 3; i++ { + if !bytes.Equal(capture.bodies[0], capture.bodies[i]) { + t.Fatalf("body %d differs:\n%q\n%q", i, capture.bodies[0], capture.bodies[i]) + } + if capture.types[i] != capture.types[0] { + t.Fatalf("content type %d = %q", i, capture.types[i]) + } + } + media, params, err := mime.ParseMediaType(capture.types[0]) + if err != nil || media != "multipart/form-data" || params["boundary"] != MultipartBoundary { + t.Fatalf("content type = %q", capture.types[0]) + } + form, err := multipart.NewReader(bytes.NewReader(capture.bodies[0]), MultipartBoundary).ReadForm(1 << 20) + if err != nil { + t.Fatal(err) + } + if got := form.Value["title"]; len(got) != 1 || got[0] != "Cover 7" { + t.Fatalf("title = %q (variables expand in values)", got) + } + fh := form.File["file"] + if len(fh) != 1 || fh[0].Filename != "cover.png" || fh[0].Header.Get("Content-Type") != "image/png" { + t.Fatalf("file part = %+v", fh) + } + + // The recording keeps the parts, never the file bytes. + raw, err := marshalFlow(recorded) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "fake image bytes") || !strings.Contains(string(raw), "file: files/cover.png") { + t.Fatalf("recorded flow:\n%s", raw) + } + parsed, err := ParseFlow(raw) + if err != nil { + t.Fatalf("parse recorded flow: %v", err) + } + if len(parsed.Steps[0].Request.Parts) != 2 || parsed.Steps[0].Request.Parts[1].SHA256 != flow.Steps[0].Request.Parts[1].SHA256 { + t.Fatalf("parts after round trip = %+v", parsed.Steps[0].Request.Parts) + } +} + +func TestMultipartTamperedPartFileFailsLoad(t *testing.T) { + dir, flow := multipartFixture(t) + raw, err := marshalFlow(flow) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "upload.yaml") + if err := os.WriteFile(path, raw, 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadFlow(path); err != nil { + t.Fatalf("intact fixture: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), []byte("tampered"), 0o644); err != nil { + t.Fatal(err) + } + _, err = LoadFlow(path) + if err == nil || !strings.Contains(err.Error(), `part "file"`) || !strings.Contains(err.Error(), "sha256") { + t.Fatalf("tampered file: %v", err) + } + if err := os.Remove(filepath.Join(dir, "files", "cover.png")); err != nil { + t.Fatal(err) + } + if _, err := LoadFlow(path); err == nil { + t.Fatal("missing part file must fail the load") + } +} + +func TestMultipartRejectsInvalidParts(t *testing.T) { + _, flow := multipartFixture(t) + cases := map[string]func(*Request){ + "body and parts": func(r *Request) { r.Body = "x=1" }, + "escaping file": func(r *Request) { r.Parts[1].File = "../secret.png" }, + "missing sha256": func(r *Request) { r.Parts[1].SHA256 = "" }, + "value and file": func(r *Request) { r.Parts[1].Value = "x" }, + "missing name": func(r *Request) { r.Parts[0].Name = "" }, + } + for name, mutate := range cases { + f := flow + f.Steps = []Step{flow.Steps[0]} + f.Steps[0].Request.Parts = append([]Part(nil), flow.Steps[0].Request.Parts...) + mutate(&f.Steps[0].Request) + if err := validateFlow(f); err == nil { + t.Fatalf("%s: validateFlow accepted the flow", name) + } + } +} + +func TestMultipartKeepsNonMultipartContentType(t *testing.T) { + dir, flow := multipartFixture(t) + req := flow.Steps[0].Request + req.Headers = map[string]string{"Content-Type": "text/plain"} + out, err := prepareRequest(req, dir) + if err != nil { + t.Fatal(err) + } + if out.Headers["Content-Type"] != "text/plain" { + t.Fatalf("content type = %q", out.Headers["Content-Type"]) + } + req.Headers = nil + out, err = prepareRequest(req, dir) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out.Headers["Content-Type"], MultipartBoundary) { + t.Fatalf("content type = %q", out.Headers["Content-Type"]) + } +} + +func uploadBody(url, thumb string) []byte { + b, _ := json.Marshal(map[string]any{"data": map[string]any{"photos": []any{map[string]any{"id": 1, "url": url, "thumb_url": thumb}}}}) + return b +} + +func TestNormalizeUploadURLMasksRandomParts(t *testing.T) { + step := Step{ID: "u"} + php := uploadBody( + "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png", + "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png") + goBody := uploadBody( + "/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png") + if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(goBody)}, step); len(diffs) != 0 { + t.Fatalf("diffs = %+v", diffs) + } + // An external URL under url is not an upload and is compared as is. + ext := uploadBody("https://img.example.com/a.jpg", "https://img.example.com/b.jpg") + if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(ext)}, Response{Headers: jsonCT(), Body: Body(ext)}, step); len(diffs) != 0 { + t.Fatalf("external diffs = %+v", diffs) + } +} + +func TestNormalizeUploadURLReportsWrongShape(t *testing.T) { + step := Step{ID: "u"} + php := uploadBody( + "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png", + "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png") + cases := map[string][]byte{ + "thumb size": uploadBody( + "/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_100_100_0_0_crop.png"), + "prefix": uploadBody( + "/storage/uploads/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"), + "partition": uploadBody( + "/storage/app/uploads/public/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"), + "partition not from disk name": uploadBody( + "/storage/app/uploads/public/aaa/bbb/ccc/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"), + "extension": uploadBody( + "/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.jpg", + "/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"), + } + for name, got := range cases { + diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(got)}, step) + if len(diffs) == 0 { + t.Fatalf("%s: masking hid the mismatch", name) + } + } + // A configured prefix replaces the WinterCMS default. + alt := uploadBody("/files/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", "/files/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png") + alt2 := uploadBody("/files/123/456/789/1234567890abcdef.png", "/files/123/456/789/thumb_9_200_200_0_0_crop.png") + if diffs := compareBodiesWith(Response{Headers: jsonCT(), Body: Body(alt)}, Response{Headers: jsonCT(), Body: Body(alt2)}, step, maskOptions{uploadPrefix: "/files"}); len(diffs) != 0 { + t.Fatalf("custom prefix diffs = %+v", diffs) + } +} + +func TestNormalizePublicationAlbumDates(t *testing.T) { + store := mustMemoryStore() + pub := func(created, updated string) []Publication { + return []Publication{{Method: "POST", Path: "/api/publish", Body: json.RawMessage( + `{"channel":"c","data":{"payload":{"album":{"name":"x","created_at":"` + created + `","updated_at":"` + updated + + `","market_price_checked_at":null,"photos":[{"created_at":"` + created + `"}]},"created_at":"2026-01-01T00:00:00+00:00"}}}`)}} + } + a, err := NormalizePublications(pub("2026-09-30T11:21:55+00:00", "2026-09-30T11:21:56+00:00"), store) + if err != nil { + t.Fatal(err) + } + b, err := NormalizePublications(pub("2026-10-02T08:00:00+00:00", "2026-10-02T08:00:01+00:00"), store) + if err != nil { + t.Fatal(err) + } + if diffs := DiffPublications(a, b); len(diffs) != 0 { + t.Fatalf("diffs = %+v", diffs) + } + if !strings.Contains(string(a[0].Body), `"updated_at":"{{datetime}}"`) || !strings.Contains(string(a[0].Body), `"market_price_checked_at":null`) { + t.Fatalf("album dates not masked: %s", a[0].Body) + } + // Only the album subtree is masked; a payload date outside it stays. + if !strings.Contains(string(a[0].Body), `},"created_at":"2026-01-01T00:00:00+00:00"`) { + t.Fatalf("over-normalised: %s", a[0].Body) + } + // A Z-suffixed album date keeps its value, so a format change is a diff. + z, err := NormalizePublications(pub("2026-10-02T08:00:00Z", "2026-10-02T08:00:01+00:00"), store) + if err != nil { + t.Fatal(err) + } + if diffs := DiffPublications(a, z); len(diffs) == 0 { + t.Fatal("a Z album date must show as a diff") + } +} diff --git a/modules/tide/normalize.go b/modules/tide/normalize.go index c4e8edb..7691086 100644 --- a/modules/tide/normalize.go +++ b/modules/tide/normalize.go @@ -14,7 +14,25 @@ const ( maskID = "" ) -func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) { +// DefaultUploadPrefix is the WinterCMS public uploads URL prefix +// (cms.storage.uploads.path plus /public) that url and thumb_url values are +// checked against when ReplayConfig.UploadPrefix is empty. +const DefaultUploadPrefix = "/storage/app/uploads/public" + +// maskOptions configures the response-body normalizer. +type maskOptions struct { + uploadPrefix string +} + +func (o maskOptions) prefix() string { + p := strings.TrimRight(o.uploadPrefix, "/") + if p == "" { + return DefaultUploadPrefix + } + return p +} + +func normalizeJSON(raw []byte, step Step, opts maskOptions) ([]byte, []Diff) { if len(strings.TrimSpace(string(raw))) == 0 { return raw, nil } @@ -23,7 +41,7 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) { return raw, nil } var diffs []Diff - masked := maskValue("$", val, step, &diffs) + masked := maskValue("$", val, step, opts, &diffs) out, err := json.Marshal(masked) if err != nil { return raw, diffs @@ -31,30 +49,36 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) { return out, diffs } -func maskValue(path string, val any, step Step, diffs *[]Diff) any { +func maskValue(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any { switch v := val.(type) { case map[string]any: out := make(map[string]any, len(v)) for k, child := range v { - out[k] = maskValue(pathJoin(path, k), child, step, diffs) + out[k] = maskValue(pathJoin(path, k), child, step, opts, diffs) } return out case []any: out := make([]any, len(v)) for i, child := range v { - out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, diffs) + out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, opts, diffs) } return out default: - return maskLeaf(path, val, step, diffs) + return maskLeaf(path, val, step, opts, diffs) } } -func maskLeaf(path string, val any, step Step, diffs *[]Diff) any { +func maskLeaf(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any { key := lastPathKey(path) if key == "slug" || disabledPath(step, path, key) { return val } + if key == "url" || key == "thumb_url" { + if s, ok := val.(string); ok { + return maskUploadURL(path, s, opts.prefix(), diffs) + } + return val + } if key == "collection_key" || key == "client_id" { if val == nil { return nil @@ -154,3 +178,42 @@ func disabledPath(step Step, jsonPath, key string) bool { func strconvQuote(s string) string { return `"` + s + `"` } + +var ( + // uploadOriginalRe is /: Winter's partition + // directory (the first nine characters of the disk name in three groups) + // and a hex disk name with its extension. + uploadOriginalRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{9,})(\.[a-z0-9]+)?$`) + // uploadThumbRe is /thumb______. + // (Winter getThumbFilename). + uploadThumbRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/thumb_([0-9]+)_([0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+)$`) + // uploadShapeRe recognises an upload URL under any prefix. + uploadShapeRe = regexp.MustCompile(`/[0-9a-f]{3}/[0-9a-f]{3}/[0-9a-f]{3}/(?:thumb_[0-9]+_[0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+|[0-9a-f]{9,}(?:\.[a-z0-9]+)?)$`) +) + +// maskUploadURL masks the random parts of an uploaded file's URL, the +// partition and disk name of an original and the partition and file id of a +// thumbnail, after checking the shape. The prefix, thumbnail size, offsets, +// mode and extension stay visible, so a different size or extension still +// shows as a mismatch. A URL under another prefix that looks like an upload +// is a Diff; any other value is left as it is. +func maskUploadURL(path, s, prefix string, diffs *[]Diff) any { + if rest, ok := strings.CutPrefix(s, prefix); ok && strings.HasPrefix(rest, "/") { + if m := uploadOriginalRe.FindStringSubmatch(rest); m != nil { + if m[1]+m[2]+m[3] != m[4][:9] { + *diffs = append(*diffs, Diff{Path: path, Expected: "partition from the disk name", Actual: strconvQuote(s)}) + return s + } + return prefix + "//" + m[5] + } + if m := uploadThumbRe.FindStringSubmatch(rest); m != nil { + return prefix + "//thumb__" + m[5] + } + *diffs = append(*diffs, Diff{Path: path, Expected: "upload URL " + prefix + "/xxx/yyy/zzz/", Actual: strconvQuote(s)}) + return s + } + if uploadShapeRe.MatchString(s) && !strings.Contains(s, "://") { + *diffs = append(*diffs, Diff{Path: path, Expected: "upload URL under " + prefix, Actual: strconvQuote(s)}) + } + return s +} diff --git a/modules/tide/record.go b/modules/tide/record.go index 22387c7..c1ab48b 100644 --- a/modules/tide/record.go +++ b/modules/tide/record.go @@ -37,6 +37,10 @@ func RecordFlow(ctx context.Context, spec Flow, cfg RecordConfig) (Flow, error) if err != nil { return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) } + req, err = prepareRequest(req, cfg.BaseDir) + if err != nil { + return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) + } resp, err := doStep(ctx, client, cfg.Target, req, limit) if err != nil { return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) diff --git a/modules/tide/replay.go b/modules/tide/replay.go index 65238ec..25bcf83 100644 --- a/modules/tide/replay.go +++ b/modules/tide/replay.go @@ -44,6 +44,10 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error skipRest = true continue } + req, err = prepareRequest(req, cfg.BaseDir) + if err != nil { + return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err) + } got, err := doStep(ctx, client, cfg.Target, req, limit) if err != nil { return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err) @@ -71,7 +75,7 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error skipRest = true continue } - sr.Diffs = append(sr.Diffs, compareStep(want, live.Response)...) + sr.Diffs = append(sr.Diffs, compareStepWith(want, live.Response, maskOptions{uploadPrefix: cfg.UploadPrefix})...) if len(sr.Diffs) > 0 { sr.OK = false result.OK = false @@ -90,6 +94,10 @@ func mustMemoryStore() *Store { } func compareStep(want Step, got Response) []Diff { + return compareStepWith(want, got, maskOptions{}) +} + +func compareStepWith(want Step, got Response, opts maskOptions) []Diff { var diffs []Diff if want.Response.Status != 0 && got.Status != want.Response.Status { diffs = append(diffs, Diff{ @@ -99,6 +107,6 @@ func compareStep(want Step, got Response) []Diff { }) } diffs = append(diffs, compareHeaders(want.Response.Headers, got.Headers, want.Headers)...) - diffs = append(diffs, compareBodies(want.Response, got, want)...) + diffs = append(diffs, compareBodiesWith(want.Response, got, want, opts)...) return diffs } diff --git a/modules/tide/variables.go b/modules/tide/variables.go index 948456a..ffe2302 100644 --- a/modules/tide/variables.go +++ b/modules/tide/variables.go @@ -207,6 +207,19 @@ func expandRequest(req Request, store *Store) (Request, error) { return Request{}, err } out.Body = Body(body) + if len(req.Parts) > 0 { + // Variables expand in part values only, never in file bytes. + out.Parts = make([]Part, len(req.Parts)) + for i, p := range req.Parts { + out.Parts[i] = p + if p.File == "" { + out.Parts[i].Value, err = store.Expand(p.Value) + if err != nil { + return Request{}, err + } + } + } + } return out, nil } @@ -461,6 +474,14 @@ func ScrubStep(store *Store, step *Step) error { step.Request.Query = replaceAll(step.Request.Query, pairs, true) step.Request.Headers = scrubMap(step.Request.Headers, pairs, true) step.Request.Body = Body(replaceAll(string(step.Request.Body), pairs, false)) + if len(step.Request.Parts) > 0 { + parts := make([]Part, len(step.Request.Parts)) + for i, p := range step.Request.Parts { + parts[i] = p + parts[i].Value = replaceAll(p.Value, pairs, false) + } + step.Request.Parts = parts + } for _, rule := range step.Capture { if strings.TrimSpace(rule.From) != "request.form" { continue @@ -606,6 +627,11 @@ func rejectUnclassifiedCredentials(step Step) error { if err := check("request body", string(step.Request.Body)); err != nil { return err } + for _, p := range step.Request.Parts { + if err := check("request part "+p.Name, p.Value); err != nil { + return err + } + } for k, v := range step.Response.Headers { if err := check("response header "+k, v); err != nil { return err