feat(12-01): record multipart uploads and match Winter upload URLs
- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the image size from the header before decoding - tide requests carry multipart parts (files beside the fixture pinned by sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive byte-identical bodies - tide masks the random partition, disk name and file id of url/thumb_url upload URLs while still diffing prefix, size, mode and extension, and NormalizePublications masks Carbon dates in the published album
This commit is contained in:
@@ -24,7 +24,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he
|
||||
- Column types: `lagoon.Encrypted` stores AES-256-GCM ciphertext under a key derived from `app.key`, decrypts with previous keys during rotation, and always redacts itself in JSON and string output; `lagoon.Jsonable` stores JSON as TEXT and keeps SQL NULL distinct from an empty value.
|
||||
- Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns.
|
||||
- Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports.
|
||||
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), on-demand thumbnails through `attach.File.Thumb`, static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
|
||||
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -151,7 +151,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration {
|
||||
| `lagoon.WithSoftDeleteCascade` | Runs a cascade inside the parent delete transaction. |
|
||||
| `lagoon.RegisterJoinTable` | Registers a custom pivot model for a many-to-many field. |
|
||||
| `lagoon.DecryptLaravelPayload` | Decrypts a Laravel AES-256-CBC payload for data imports. |
|
||||
| `attach.File` | The `system_files` row model. |
|
||||
| `attach.File` | The `system_files` row model; `attach.File.URL` is the public URL of the original. |
|
||||
| `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. |
|
||||
| `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. |
|
||||
| `attach.OpenBucket` | Opens the uploads bucket from config. |
|
||||
| `attach.Publish` | Stores the bucket on the `backpack.App`. |
|
||||
@@ -198,7 +199,7 @@ storage:
|
||||
|
||||
- SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages).
|
||||
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`).
|
||||
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails).
|
||||
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding).
|
||||
- Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others.
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
@@ -14,6 +15,10 @@ import (
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"gocloud.dev/blob"
|
||||
// The webp decoder lets image.DecodeConfig and File.Thumb read .webp
|
||||
// originals. imaging cannot encode webp, so a webp thumbnail holds JPEG
|
||||
// bytes under the original's .webp name (see defaultEncodeImage).
|
||||
_ "golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -67,7 +72,11 @@ func fileExt(diskName string) string {
|
||||
return strings.ToLower(ext)
|
||||
}
|
||||
|
||||
func publicURL(key string) string {
|
||||
// PublicURL returns the public URL of a blob key: storage.uploads.
|
||||
// public_path_prefix and the key joined by exactly one slash. With the
|
||||
// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
|
||||
// /storage/app/uploads/public) it is Winter's File::getPath() path.
|
||||
func PublicURL(key string) string {
|
||||
prefix := strings.TrimRight(PublicPathPrefix(), "/")
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if prefix == "" {
|
||||
@@ -76,6 +85,15 @@ func publicURL(key string) string {
|
||||
return prefix + "/" + key
|
||||
}
|
||||
|
||||
// URL returns the public URL of the original file, Winter's File::getPath():
|
||||
// PublicURL of BlobKey(DiskName).
|
||||
func (f *File) URL() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
return PublicURL(BlobKey(f.DiskName))
|
||||
}
|
||||
|
||||
func defaultResizeImage(src image.Image, w, h int, mode string) image.Image {
|
||||
switch strings.ToLower(mode) {
|
||||
case "crop":
|
||||
@@ -133,21 +151,35 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
|
||||
return "", fmt.Errorf("attach: thumb exists: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return publicURL(thumbKey), nil
|
||||
return PublicURL(thumbKey), nil
|
||||
}
|
||||
origKey := part + f.DiskName
|
||||
r, err := bucket.NewReader(ctx, origKey, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: read original: %w", err)
|
||||
}
|
||||
src, _, err := image.Decode(io.LimitReader(r, maxThumbSourceBytes))
|
||||
raw, err := io.ReadAll(io.LimitReader(r, maxThumbSourceBytes))
|
||||
closeErr := r.Close()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
return "", fmt.Errorf("attach: read original: %w", err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", closeErr
|
||||
}
|
||||
// Check the dimensions from the header before decoding the pixels, so
|
||||
// a small file that declares a huge image is refused without
|
||||
// allocating it.
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > maxThumbSourcePixels {
|
||||
return "", fmt.Errorf("attach: original image is too large")
|
||||
}
|
||||
src, _, err := image.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
}
|
||||
bounds := src.Bounds()
|
||||
if int64(bounds.Dx())*int64(bounds.Dy()) > maxThumbSourcePixels {
|
||||
return "", fmt.Errorf("attach: original image is too large")
|
||||
@@ -173,5 +205,5 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
|
||||
}
|
||||
return "", closeErr
|
||||
}
|
||||
return publicURL(thumbKey), nil
|
||||
return PublicURL(thumbKey), nil
|
||||
}
|
||||
|
||||
107
modules/lagoon/attach/url_test.go
Normal file
107
modules/lagoon/attach/url_test.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
"gocloud.dev/blob"
|
||||
)
|
||||
|
||||
// winterLayoutBucket opens a mem:// bucket with the WinterCMS public prefix
|
||||
// and restores the framework default afterwards.
|
||||
func winterLayoutBucket(t *testing.T) *blob.Bucket {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
body := "uploads:\n bucket_url: \"mem://\"\n public_path_prefix: \"/storage/app/uploads/public\"\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := compass.Open(compass.Options{Dir: dir, Env: "development", Environ: []string{"SUMMER_ENV=development"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = bucket.Close()
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
})
|
||||
return bucket
|
||||
}
|
||||
|
||||
func TestFileURLWinterLayout(t *testing.T) {
|
||||
bucket := winterLayoutBucket(t)
|
||||
// A WinterCMS disk name: uniqid('', true) without the dot, plus the
|
||||
// extension.
|
||||
f := &File{ID: 12, DiskName: "651a2b3c4d5e61234567.png"}
|
||||
if got, want := f.URL(), "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
|
||||
t.Fatalf("URL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := PublicURL("/651/a2b/3c4/x.png"), "/storage/app/uploads/public/651/a2b/3c4/x.png"; got != want {
|
||||
t.Fatalf("PublicURL = %q, want %q", got, want)
|
||||
}
|
||||
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), &blob.WriterOptions{ContentType: "image/png"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
thumb, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Winter getThumbFilename: implode('_', [thumb, id, w, h, ox, oy, mode.ext]).
|
||||
if want := "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png"; thumb != want {
|
||||
t.Fatalf("thumb = %q, want %q", thumb, want)
|
||||
}
|
||||
var nilFile *File
|
||||
if nilFile.URL() != "" {
|
||||
t.Fatal("nil file URL must be empty")
|
||||
}
|
||||
}
|
||||
|
||||
// webpFixture is a 16x12 lossless WebP: blue left half, red right half.
|
||||
var webpFixture = []byte{
|
||||
0x52, 0x49, 0x46, 0x46, 0x2a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, 0x50, 0x38, 0x4c,
|
||||
0x1d, 0x00, 0x00, 0x00, 0x2f, 0x0f, 0xc0, 0x02, 0x00, 0x0f, 0x70, 0x14, 0xfb, 0x53, 0xd0, 0x5e,
|
||||
0x88, 0x7b, 0xfe, 0x83, 0x07, 0x62, 0xc1, 0x64, 0xfe, 0xd2, 0xbd, 0x21, 0x44, 0xf4, 0x3f, 0x74,
|
||||
0x01, 0x00,
|
||||
}
|
||||
|
||||
func TestThumbWebP(t *testing.T) {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(webpFixture))
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeConfig: %v", err)
|
||||
}
|
||||
if format != "webp" || cfg.Width != 16 || cfg.Height != 12 {
|
||||
t.Fatalf("config = %s %dx%d", format, cfg.Width, cfg.Height)
|
||||
}
|
||||
bucket := winterLayoutBucket(t)
|
||||
f := &File{ID: 5, DiskName: "abcdef123456789012345.webp"}
|
||||
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), webpFixture, &blob.WriterOptions{ContentType: "image/webp"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
|
||||
if err != nil {
|
||||
t.Fatalf("Thumb: %v", err)
|
||||
}
|
||||
if want := "/storage/app/uploads/public/abc/def/123/thumb_5_200_200_0_0_crop.webp"; url != want {
|
||||
t.Fatalf("thumb url = %q, want %q", url, want)
|
||||
}
|
||||
raw, err := bucket.ReadAll(t.Context(), "abc/def/123/thumb_5_200_200_0_0_crop.webp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// imaging cannot encode webp: the thumb is JPEG bytes under the .webp name.
|
||||
img, err := jpeg.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("thumb is not JPEG: %v", err)
|
||||
}
|
||||
if b := img.Bounds(); b.Dx() != 200 || b.Dy() != 200 {
|
||||
t.Fatalf("thumb size = %v", b)
|
||||
}
|
||||
}
|
||||
@@ -12,13 +12,14 @@ HTTP parity toolkit that records request and response fixtures from a reference
|
||||
|
||||
- Flow fixtures: `tide.Flow` is a versioned, ordered list of `tide.Step` values, loaded strictly (unknown fields rejected) with `tide.LoadFlow` and `tide.ParseFlow`, and written atomically with `tide.SaveFlow` or `tide.SaveFlowExclusive`. Response bodies can live in sidecar files, confined to the fixture directory and checked against an optional SHA-256 digest.
|
||||
- Recording: `tide.RecordFlow` executes a spec flow against a target URL and fills in the responses, with a bounded body size (`tide.DefaultMaxBody`, 8 MiB).
|
||||
- Multipart uploads: a request can carry `parts` (`tide.Part`: a text field with `value`, or a file field with `file`, `filename`, `content_type` and `sha256`) instead of a `body`. File bytes stay in files beside the fixture (for example `files/cover.png`, read from `tide.RecordConfig` or `tide.ReplayConfig` `BaseDir`) and are never inlined into the YAML. Parts are encoded in declaration order with one fixed boundary, `tide.MultipartBoundary`, so the reference backend and the port receive byte-identical bodies; the encoded Content-Type replaces a recorded multipart Content-Type. Variables expand in part values, never in file bytes. `tide.LoadFlow` fails when a part file is missing or its SHA-256 differs, naming the part, and a request with both `body` and `parts` is invalid.
|
||||
- Recording proxy: `tide.NewProxy` builds a reverse proxy that only binds to and forwards to loopback addresses, groups traffic into named sessions (from the `tide.SessionHeader` request header or a default session) and writes one fixture per complete session on `tide.Proxy.Flush`.
|
||||
- Capture rules: `tide.Rules` (loaded with `tide.LoadRules`) decide which request and response headers are kept per route and which values are captured into variables, from response JSON paths, headers, redirect query strings or form fields.
|
||||
- Variables: `tide.Store` holds captured values such as tokens and IDs in a mode-0600 file, `tide.Store.Expand` substitutes `{{name}}` placeholders before a request is sent, and `tide.ScrubStep` puts placeholders back into fixtures. Scrubbing fails when a step still holds an unclassified token- or password-shaped value, so credentials do not leak into committed fixtures.
|
||||
- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte.
|
||||
- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. Uploaded-file URLs under `url` and `thumb_url` keys are compared by shape: under the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, default `tide.DefaultUploadPrefix`, `/storage/app/uploads/public`) an original must be `<prefix>/xxx/yyy/zzz/<disk_name>` with the partition taken from the disk name, and a thumbnail `<prefix>/xxx/yyy/zzz/thumb_<id>_<w>_<h>_<ox>_<oy>_<mode>.<ext>`. The partition, disk name and file id are masked; the prefix, size, offsets, mode and extension stay, so a thumbnail of another size or an upload URL under another prefix is still a difference.
|
||||
- Fake Centrifugo: `tide.NewCentrifugoRecorder` returns an `http.Handler` that records every POST to a path ending in `/publish` or `/broadcast` as a `tide.Publication` (method, path, whether `Authorization: apikey <key>` carried the configured key, JSON body) and answers `{"result":{}}`. Paths ending in `/presence` answer `{"result":{"presence":{}}}`, `/unsubscribe` and `/info` answer `{"result":{}}`, anything else is 404. Bodies are capped at `tide.MaxPublicationBody` (1 MiB). The API key is only compared, never stored. `tide.CentrifugoRecorder.ListenAndServe` binds loopback addresses only, like the recording proxy.
|
||||
- Broadcast goldens: `tide.RecordBroadcasts` runs a flow against a loopback reference backend whose Centrifugo API URL points at a recorder on `tide.DefaultCentrifugoListen` (`127.0.0.1:8424`). With `tide.BroadcastConfig` `Step` set, earlier steps run as setup and only that step's publications are kept. The result is a `tide.BroadcastGolden`, written with `tide.WriteBroadcastGolden` (which refuses token-shaped bodies) and read strictly with `tide.LoadBroadcastGolden`. A golden with `pending` set is recorded but not yet asserted.
|
||||
- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`.
|
||||
- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. Carbon `+00:00` values of `*_at` keys anywhere under `$.data.payload.album` become `"{{datetime}}"`; a date of another shape is left as it is, so a format change shows as a difference. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`.
|
||||
- Manifests: `tide.Manifest` lists routes with auth groups, a pending or ported status, cases and fixture paths; `tide.RecordManifest` records missing cases in batches of at most `tide.MaxBatch`, and `tide.ReplayManifest` replays every recorded case into a `tide.Coverage` table.
|
||||
|
||||
## Usage
|
||||
@@ -88,6 +89,12 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) {
|
||||
| `tide.LoadFlow` | Reads and validates a flow file. |
|
||||
| `tide.SaveFlow` | Writes a validated flow atomically. |
|
||||
| `tide.RecordFlow` | Executes a spec flow against a target and returns the recorded flow. |
|
||||
| `tide.Request` | The outbound call of a step: method, path, query, headers and a `Body` or multipart `Parts`. |
|
||||
| `tide.Part` | One multipart field: a text value, or a file beside the fixture pinned by its SHA-256. |
|
||||
| `tide.MultipartBoundary` | The fixed boundary multipart request bodies are encoded with. |
|
||||
| `tide.RecordConfig` | Target, client, body cap, vars store, capture rules and part-file directory for `tide.RecordFlow`. |
|
||||
| `tide.ReplayConfig` | Target, client, body cap, vars store, fixture directory and upload URL prefix for `tide.ReplayFlow`. |
|
||||
| `tide.DefaultUploadPrefix` | The default uploads URL prefix, `/storage/app/uploads/public`. |
|
||||
| `tide.ReplayFlow` | Replays a recorded flow against a target and diffs every step. |
|
||||
| `tide.Result` | Replay outcome: overall status and per-step `tide.StepResult` values. |
|
||||
| `tide.Diff` | One structural JSON or byte-level mismatch. |
|
||||
@@ -109,7 +116,7 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) {
|
||||
| `tide.BroadcastGolden` | Versioned broadcast golden: name, flow, optional pending reason, publications. |
|
||||
| `tide.LoadBroadcastGolden` | Reads a golden strictly and checks every body parses. |
|
||||
| `tide.WriteBroadcastGolden` | Writes a golden atomically, refusing token-shaped bodies. |
|
||||
| `tide.NormalizePublications` | Masks timestamps, the actor and captured ids in publication bodies. |
|
||||
| `tide.NormalizePublications` | Masks timestamps, the actor, album dates and captured ids in publication bodies. |
|
||||
| `tide.DiffPublications` | Structural diff of two publication lists. |
|
||||
| `tide.DefaultCentrifugoListen` | Default recorder address, `127.0.0.1:8424`. |
|
||||
| `tide.Coverage` | Recorded, passing, failing and unrecorded counts, with table rows and a summary line. |
|
||||
|
||||
@@ -164,6 +164,9 @@ var channelIDRe = regexp.MustCompile(`^((?:[a-z][a-z0-9_.-]*:)+)([0-9]+)$`)
|
||||
// when they have the ISO 8601 offset shape;
|
||||
// - $.data.payload.actor becomes "{{actor}}" when it is an object with
|
||||
// exactly user_id and name;
|
||||
// - a *_at value anywhere under $.data.payload.album becomes
|
||||
// "{{datetime}}" when it has Carbon's +00:00 shape; any other shape is
|
||||
// left as it is, so a format change shows up as a diff;
|
||||
// - a number or string under an id, *_id or *_ids key that equals an id:*
|
||||
// variable of store becomes that {{id:name}} placeholder (bare for a
|
||||
// number, quoted for a string), and so does the numeric last segment of
|
||||
@@ -304,6 +307,9 @@ func (n *normalizer) walk(path, key string, v *onode) *onode {
|
||||
}
|
||||
return v
|
||||
case kindString:
|
||||
if strings.HasPrefix(path, "$.data.payload.album.") && strings.HasSuffix(key, "_at") && carbonOffsetRe.MatchString(v.str) {
|
||||
return &onode{kind: kindString, str: "{{datetime}}"}
|
||||
}
|
||||
if key != "" && isIDKey(key) {
|
||||
if name, ok := n.lookup(v.str); ok {
|
||||
return &onode{kind: kindString, str: "{{" + name + "}}"}
|
||||
|
||||
@@ -13,11 +13,15 @@ import (
|
||||
)
|
||||
|
||||
func compareBodies(want, got Response, step Step) []Diff {
|
||||
return compareBodiesWith(want, got, step, maskOptions{})
|
||||
}
|
||||
|
||||
func compareBodiesWith(want, got Response, step Step, opts maskOptions) []Diff {
|
||||
wantJSON := isJSONContentType(want.Headers)
|
||||
gotJSON := isJSONContentType(got.Headers)
|
||||
if wantJSON && gotJSON {
|
||||
wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step)
|
||||
gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step)
|
||||
wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step, opts)
|
||||
gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step, opts)
|
||||
diffs := append([]Diff{}, wantDiffs...)
|
||||
diffs = append(diffs, gotDiffs...)
|
||||
diffs = append(diffs, diffJSON(wantRaw, gotRaw)...)
|
||||
|
||||
@@ -25,13 +25,21 @@ func (b *Body) UnmarshalYAML(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadFlow reads and validates a version-1 YAML flow from path.
|
||||
// LoadFlow reads and validates a version-1 YAML flow from path. Request
|
||||
// part files are checked against their sha256, relative to path's directory.
|
||||
func LoadFlow(path string) (Flow, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Flow{}, fmt.Errorf("tide: read %s: %w", path, err)
|
||||
}
|
||||
return ParseFlow(raw)
|
||||
flow, err := ParseFlow(raw)
|
||||
if err != nil {
|
||||
return Flow{}, err
|
||||
}
|
||||
if err := verifyPartFiles(filepath.Dir(path), flow); err != nil {
|
||||
return Flow{}, err
|
||||
}
|
||||
return flow, nil
|
||||
}
|
||||
|
||||
// ParseFlow decodes a version-1 YAML flow, rejecting unknown fields.
|
||||
@@ -163,6 +171,30 @@ func writeRequest(b *strings.Builder, indent int, req Request) {
|
||||
}
|
||||
writeHeaders(b, indent, req.Headers)
|
||||
writeBody(b, indent, string(req.Body))
|
||||
writeParts(b, indent, req.Parts)
|
||||
}
|
||||
|
||||
func writeParts(b *strings.Builder, indent int, parts []Part) {
|
||||
if len(parts) == 0 {
|
||||
return
|
||||
}
|
||||
pad := strings.Repeat(" ", indent)
|
||||
fmt.Fprintf(b, "%sparts:\n", pad)
|
||||
inner := strings.Repeat(" ", indent+2)
|
||||
for _, p := range parts {
|
||||
fmt.Fprintf(b, "%s- name: %s\n", inner, encodeScalar(p.Name))
|
||||
for _, kv := range [][2]string{
|
||||
{"value", p.Value},
|
||||
{"file", p.File},
|
||||
{"filename", p.Filename},
|
||||
{"content_type", p.ContentType},
|
||||
{"sha256", p.SHA256},
|
||||
} {
|
||||
if kv[1] != "" {
|
||||
fmt.Fprintf(b, "%s %s: %s\n", inner, kv[0], encodeScalar(kv[1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeResponse(b *strings.Builder, indent int, resp Response) {
|
||||
|
||||
@@ -42,6 +42,9 @@ type Request struct {
|
||||
Query string `yaml:"query,omitempty"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Body Body `yaml:"body,omitempty"`
|
||||
// Parts is a multipart/form-data body, encoded with MultipartBoundary
|
||||
// when the step runs. A request has Body or Parts, never both.
|
||||
Parts []Part `yaml:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// Response is the recorded or expected HTTP reply.
|
||||
@@ -73,21 +76,28 @@ type NormalizeRule struct {
|
||||
type Body string
|
||||
|
||||
// RecordConfig injects the HTTP target, client and body bound for recording.
|
||||
// BaseDir is the fixture directory that request part files are read from.
|
||||
type RecordConfig struct {
|
||||
Target string
|
||||
Client *http.Client
|
||||
MaxBody int64
|
||||
Store *Store
|
||||
Rules Rules
|
||||
BaseDir string
|
||||
}
|
||||
|
||||
// ReplayConfig injects the HTTP target, client and body bound for replay.
|
||||
// BaseDir is the fixture directory that body_file sidecars and request part
|
||||
// files are read from.
|
||||
// UploadPrefix is the public URL prefix of uploaded files whose url and
|
||||
// thumb_url values are compared by shape; empty means DefaultUploadPrefix.
|
||||
type ReplayConfig struct {
|
||||
Target string
|
||||
Client *http.Client
|
||||
MaxBody int64
|
||||
Store *Store
|
||||
BaseDir string
|
||||
Target string
|
||||
Client *http.Client
|
||||
MaxBody int64
|
||||
Store *Store
|
||||
BaseDir string
|
||||
UploadPrefix string
|
||||
}
|
||||
|
||||
// Result is the outcome of replaying a flow.
|
||||
@@ -169,6 +179,9 @@ func validateFlow(flow Flow) error {
|
||||
if err := validateSidecar(step.Response.BodyFile); err != nil {
|
||||
return fmt.Errorf("tide: step %s: %w", step.ID, err)
|
||||
}
|
||||
if err := validateParts(step.Request); err != nil {
|
||||
return fmt.Errorf("tide: step %s: %w", step.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
201
modules/tide/multipart.go
Normal file
201
modules/tide/multipart.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MultipartBoundary is the fixed boundary every multipart request body is
|
||||
// encoded with, so a recording against one backend and a replay against
|
||||
// another send byte-identical bodies.
|
||||
const MultipartBoundary = "SummerTideMultipartBoundary7MA4YWxkTrZu0gW"
|
||||
|
||||
// Part is one field of a multipart/form-data request. A text field sets
|
||||
// Value; a file field sets File, a path relative to the fixture directory
|
||||
// (for example files/cover.png), whose bytes are never inlined into the
|
||||
// YAML, with SHA256 pinning its content. Filename defaults to the base name
|
||||
// of File and ContentType to application/octet-stream.
|
||||
type Part struct {
|
||||
Name string `yaml:"name"`
|
||||
Value string `yaml:"value,omitempty"`
|
||||
File string `yaml:"file,omitempty"`
|
||||
Filename string `yaml:"filename,omitempty"`
|
||||
ContentType string `yaml:"content_type,omitempty"`
|
||||
SHA256 string `yaml:"sha256,omitempty"`
|
||||
}
|
||||
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
|
||||
|
||||
// validateParts checks the shape of a request's parts without reading files.
|
||||
func validateParts(req Request) error {
|
||||
if len(req.Parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
if req.Body != "" {
|
||||
return fmt.Errorf("request has both body and parts")
|
||||
}
|
||||
for i, p := range req.Parts {
|
||||
if strings.TrimSpace(p.Name) == "" {
|
||||
return fmt.Errorf("parts[%d] is missing name", i)
|
||||
}
|
||||
if p.File == "" {
|
||||
if p.Filename != "" || p.ContentType != "" || p.SHA256 != "" {
|
||||
return fmt.Errorf("part %q has file attributes but no file", p.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if p.Value != "" {
|
||||
return fmt.Errorf("part %q has both value and file", p.Name)
|
||||
}
|
||||
if err := validateSidecar(p.File); err != nil {
|
||||
return fmt.Errorf("part %q: %w", p.Name, err)
|
||||
}
|
||||
if !sha256Hex.MatchString(p.SHA256) {
|
||||
return fmt.Errorf("part %q needs the sha256 of its file", p.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyPartFiles checks that every part file of flow exists under base and
|
||||
// matches its sha256.
|
||||
func verifyPartFiles(base string, flow Flow) error {
|
||||
for _, step := range flow.Steps {
|
||||
for _, p := range step.Request.Parts {
|
||||
if p.File == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := readPartFile(base, p); err != nil {
|
||||
return fmt.Errorf("tide: step %s: %w", step.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readPartFile(base string, p Part) ([]byte, error) {
|
||||
if err := validateSidecar(p.File); err != nil {
|
||||
return nil, fmt.Errorf("part %q: %w", p.Name, err)
|
||||
}
|
||||
full := p.File
|
||||
if base != "" {
|
||||
full = filepath.Join(base, p.File)
|
||||
}
|
||||
resolved, err := resolvePath(full)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
|
||||
}
|
||||
if base != "" {
|
||||
root, err := resolvePath(base)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fixture dir: %w", err)
|
||||
}
|
||||
if resolved != root && !strings.HasPrefix(resolved, root+string(os.PathSeparator)) {
|
||||
return nil, fmt.Errorf("part %q file %q escapes the fixture directory", p.Name, p.File)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if !strings.EqualFold(hex.EncodeToString(sum[:]), p.SHA256) {
|
||||
return nil, fmt.Errorf("part %q file %s sha256 mismatch", p.Name, p.File)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||
|
||||
// encodeParts writes parts in declaration order as a multipart/form-data
|
||||
// body with MultipartBoundary and returns the body and its Content-Type.
|
||||
func encodeParts(base string, parts []Part) ([]byte, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
if err := w.SetBoundary(MultipartBoundary); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
marker := []byte(MultipartBoundary)
|
||||
for _, p := range parts {
|
||||
h := make(textproto.MIMEHeader)
|
||||
var content []byte
|
||||
if p.File == "" {
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, quoteEscaper.Replace(p.Name)))
|
||||
content = []byte(p.Value)
|
||||
} else {
|
||||
raw, err := readPartFile(base, p)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
filename := p.Filename
|
||||
if filename == "" {
|
||||
filename = path.Base(filepath.ToSlash(p.File))
|
||||
}
|
||||
ct := p.ContentType
|
||||
if ct == "" {
|
||||
ct = "application/octet-stream"
|
||||
}
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, quoteEscaper.Replace(p.Name), quoteEscaper.Replace(filename)))
|
||||
h.Set("Content-Type", ct)
|
||||
content = raw
|
||||
}
|
||||
if bytes.Contains(content, marker) {
|
||||
return nil, "", fmt.Errorf("part %q contains the multipart boundary", p.Name)
|
||||
}
|
||||
pw, err := w.CreatePart(h)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if _, err := pw.Write(content); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf.Bytes(), w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// prepareRequest encodes a request's parts into its body. The multipart
|
||||
// Content-Type replaces a recorded multipart Content-Type (whose boundary
|
||||
// would be stale) and is added when none is recorded; a recorded non-
|
||||
// multipart Content-Type is kept as it is.
|
||||
func prepareRequest(req Request, base string) (Request, error) {
|
||||
if len(req.Parts) == 0 {
|
||||
return req, nil
|
||||
}
|
||||
if err := validateParts(req); err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
body, ct, err := encodeParts(base, req.Parts)
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
out := req
|
||||
out.Body = Body(body)
|
||||
out.Headers = make(map[string]string, len(req.Headers)+1)
|
||||
set := false
|
||||
for k, v := range req.Headers {
|
||||
if strings.EqualFold(k, "Content-Type") {
|
||||
set = true
|
||||
if media, _, err := mime.ParseMediaType(v); err != nil || strings.HasPrefix(media, "multipart/") {
|
||||
v = ct
|
||||
}
|
||||
}
|
||||
out.Headers[k] = v
|
||||
}
|
||||
if !set {
|
||||
out.Headers["Content-Type"] = ct
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
296
modules/tide/multipart_test.go
Normal file
296
modules/tide/multipart_test.go
Normal file
@@ -0,0 +1,296 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// multipartFixture writes files/cover.png under a temp fixture directory and
|
||||
// returns the directory and a one-step upload flow that references it.
|
||||
func multipartFixture(t *testing.T) (string, Flow) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
content := []byte("\x89PNG\r\n\x1a\nfake image bytes")
|
||||
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(content)
|
||||
flow := Flow{
|
||||
Version: CurrentVersion,
|
||||
Name: "upload",
|
||||
Steps: []Step{{
|
||||
ID: "upload",
|
||||
Request: Request{
|
||||
Method: "POST",
|
||||
Path: "/posts/1/photos",
|
||||
Headers: map[string]string{"Content-Type": "multipart/form-data; boundary=recorded-elsewhere"},
|
||||
Parts: []Part{
|
||||
{Name: "title", Value: "Cover {{id:post}}"},
|
||||
{Name: "file", File: "files/cover.png", ContentType: "image/png", SHA256: hex.EncodeToString(sum[:])},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
return dir, flow
|
||||
}
|
||||
|
||||
type rawCapture struct {
|
||||
mu sync.Mutex
|
||||
bodies [][]byte
|
||||
types []string
|
||||
}
|
||||
|
||||
func (c *rawCapture) handler(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.bodies = append(c.bodies, raw)
|
||||
c.types = append(c.types, r.Header.Get("Content-Type"))
|
||||
c.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartRecordReplaySendsIdenticalBytes(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
capture := &rawCapture{}
|
||||
srv := httptest.NewServer(capture.handler(t))
|
||||
t.Cleanup(srv.Close)
|
||||
store := mustMemoryStore()
|
||||
store.Set("id:post", "7")
|
||||
|
||||
recorded, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("record: %v", err)
|
||||
}
|
||||
if _, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
||||
t.Fatalf("second record: %v", err)
|
||||
}
|
||||
if _, err := ReplayFlow(t.Context(), recorded, ReplayConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
||||
t.Fatalf("replay: %v", err)
|
||||
}
|
||||
if len(capture.bodies) != 3 {
|
||||
t.Fatalf("requests = %d", len(capture.bodies))
|
||||
}
|
||||
for i := 1; i < 3; i++ {
|
||||
if !bytes.Equal(capture.bodies[0], capture.bodies[i]) {
|
||||
t.Fatalf("body %d differs:\n%q\n%q", i, capture.bodies[0], capture.bodies[i])
|
||||
}
|
||||
if capture.types[i] != capture.types[0] {
|
||||
t.Fatalf("content type %d = %q", i, capture.types[i])
|
||||
}
|
||||
}
|
||||
media, params, err := mime.ParseMediaType(capture.types[0])
|
||||
if err != nil || media != "multipart/form-data" || params["boundary"] != MultipartBoundary {
|
||||
t.Fatalf("content type = %q", capture.types[0])
|
||||
}
|
||||
form, err := multipart.NewReader(bytes.NewReader(capture.bodies[0]), MultipartBoundary).ReadForm(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := form.Value["title"]; len(got) != 1 || got[0] != "Cover 7" {
|
||||
t.Fatalf("title = %q (variables expand in values)", got)
|
||||
}
|
||||
fh := form.File["file"]
|
||||
if len(fh) != 1 || fh[0].Filename != "cover.png" || fh[0].Header.Get("Content-Type") != "image/png" {
|
||||
t.Fatalf("file part = %+v", fh)
|
||||
}
|
||||
|
||||
// The recording keeps the parts, never the file bytes.
|
||||
raw, err := marshalFlow(recorded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), "fake image bytes") || !strings.Contains(string(raw), "file: files/cover.png") {
|
||||
t.Fatalf("recorded flow:\n%s", raw)
|
||||
}
|
||||
parsed, err := ParseFlow(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse recorded flow: %v", err)
|
||||
}
|
||||
if len(parsed.Steps[0].Request.Parts) != 2 || parsed.Steps[0].Request.Parts[1].SHA256 != flow.Steps[0].Request.Parts[1].SHA256 {
|
||||
t.Fatalf("parts after round trip = %+v", parsed.Steps[0].Request.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartTamperedPartFileFailsLoad(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
raw, err := marshalFlow(flow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(dir, "upload.yaml")
|
||||
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(path); err != nil {
|
||||
t.Fatalf("intact fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), []byte("tampered"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = LoadFlow(path)
|
||||
if err == nil || !strings.Contains(err.Error(), `part "file"`) || !strings.Contains(err.Error(), "sha256") {
|
||||
t.Fatalf("tampered file: %v", err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(dir, "files", "cover.png")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(path); err == nil {
|
||||
t.Fatal("missing part file must fail the load")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartRejectsInvalidParts(t *testing.T) {
|
||||
_, flow := multipartFixture(t)
|
||||
cases := map[string]func(*Request){
|
||||
"body and parts": func(r *Request) { r.Body = "x=1" },
|
||||
"escaping file": func(r *Request) { r.Parts[1].File = "../secret.png" },
|
||||
"missing sha256": func(r *Request) { r.Parts[1].SHA256 = "" },
|
||||
"value and file": func(r *Request) { r.Parts[1].Value = "x" },
|
||||
"missing name": func(r *Request) { r.Parts[0].Name = "" },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
f := flow
|
||||
f.Steps = []Step{flow.Steps[0]}
|
||||
f.Steps[0].Request.Parts = append([]Part(nil), flow.Steps[0].Request.Parts...)
|
||||
mutate(&f.Steps[0].Request)
|
||||
if err := validateFlow(f); err == nil {
|
||||
t.Fatalf("%s: validateFlow accepted the flow", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartKeepsNonMultipartContentType(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
req := flow.Steps[0].Request
|
||||
req.Headers = map[string]string{"Content-Type": "text/plain"}
|
||||
out, err := prepareRequest(req, dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Headers["Content-Type"] != "text/plain" {
|
||||
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
||||
}
|
||||
req.Headers = nil
|
||||
out, err = prepareRequest(req, dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.Headers["Content-Type"], MultipartBoundary) {
|
||||
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
||||
}
|
||||
}
|
||||
|
||||
func uploadBody(url, thumb string) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"data": map[string]any{"photos": []any{map[string]any{"id": 1, "url": url, "thumb_url": thumb}}}})
|
||||
return b
|
||||
}
|
||||
|
||||
func TestNormalizeUploadURLMasksRandomParts(t *testing.T) {
|
||||
step := Step{ID: "u"}
|
||||
php := uploadBody(
|
||||
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
||||
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
||||
goBody := uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
||||
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(goBody)}, step); len(diffs) != 0 {
|
||||
t.Fatalf("diffs = %+v", diffs)
|
||||
}
|
||||
// An external URL under url is not an upload and is compared as is.
|
||||
ext := uploadBody("https://img.example.com/a.jpg", "https://img.example.com/b.jpg")
|
||||
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(ext)}, Response{Headers: jsonCT(), Body: Body(ext)}, step); len(diffs) != 0 {
|
||||
t.Fatalf("external diffs = %+v", diffs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeUploadURLReportsWrongShape(t *testing.T) {
|
||||
step := Step{ID: "u"}
|
||||
php := uploadBody(
|
||||
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
||||
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
||||
cases := map[string][]byte{
|
||||
"thumb size": uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_100_100_0_0_crop.png"),
|
||||
"prefix": uploadBody(
|
||||
"/storage/uploads/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"partition": uploadBody(
|
||||
"/storage/app/uploads/public/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"partition not from disk name": uploadBody(
|
||||
"/storage/app/uploads/public/aaa/bbb/ccc/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"extension": uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.jpg",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
}
|
||||
for name, got := range cases {
|
||||
diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(got)}, step)
|
||||
if len(diffs) == 0 {
|
||||
t.Fatalf("%s: masking hid the mismatch", name)
|
||||
}
|
||||
}
|
||||
// A configured prefix replaces the WinterCMS default.
|
||||
alt := uploadBody("/files/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", "/files/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
||||
alt2 := uploadBody("/files/123/456/789/1234567890abcdef.png", "/files/123/456/789/thumb_9_200_200_0_0_crop.png")
|
||||
if diffs := compareBodiesWith(Response{Headers: jsonCT(), Body: Body(alt)}, Response{Headers: jsonCT(), Body: Body(alt2)}, step, maskOptions{uploadPrefix: "/files"}); len(diffs) != 0 {
|
||||
t.Fatalf("custom prefix diffs = %+v", diffs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePublicationAlbumDates(t *testing.T) {
|
||||
store := mustMemoryStore()
|
||||
pub := func(created, updated string) []Publication {
|
||||
return []Publication{{Method: "POST", Path: "/api/publish", Body: json.RawMessage(
|
||||
`{"channel":"c","data":{"payload":{"album":{"name":"x","created_at":"` + created + `","updated_at":"` + updated +
|
||||
`","market_price_checked_at":null,"photos":[{"created_at":"` + created + `"}]},"created_at":"2026-01-01T00:00:00+00:00"}}}`)}}
|
||||
}
|
||||
a, err := NormalizePublications(pub("2026-09-30T11:21:55+00:00", "2026-09-30T11:21:56+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := NormalizePublications(pub("2026-10-02T08:00:00+00:00", "2026-10-02T08:00:01+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diffs := DiffPublications(a, b); len(diffs) != 0 {
|
||||
t.Fatalf("diffs = %+v", diffs)
|
||||
}
|
||||
if !strings.Contains(string(a[0].Body), `"updated_at":"{{datetime}}"`) || !strings.Contains(string(a[0].Body), `"market_price_checked_at":null`) {
|
||||
t.Fatalf("album dates not masked: %s", a[0].Body)
|
||||
}
|
||||
// Only the album subtree is masked; a payload date outside it stays.
|
||||
if !strings.Contains(string(a[0].Body), `},"created_at":"2026-01-01T00:00:00+00:00"`) {
|
||||
t.Fatalf("over-normalised: %s", a[0].Body)
|
||||
}
|
||||
// A Z-suffixed album date keeps its value, so a format change is a diff.
|
||||
z, err := NormalizePublications(pub("2026-10-02T08:00:00Z", "2026-10-02T08:00:01+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diffs := DiffPublications(a, z); len(diffs) == 0 {
|
||||
t.Fatal("a Z album date must show as a diff")
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,25 @@ const (
|
||||
maskID = "<id>"
|
||||
)
|
||||
|
||||
func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
|
||||
// DefaultUploadPrefix is the WinterCMS public uploads URL prefix
|
||||
// (cms.storage.uploads.path plus /public) that url and thumb_url values are
|
||||
// checked against when ReplayConfig.UploadPrefix is empty.
|
||||
const DefaultUploadPrefix = "/storage/app/uploads/public"
|
||||
|
||||
// maskOptions configures the response-body normalizer.
|
||||
type maskOptions struct {
|
||||
uploadPrefix string
|
||||
}
|
||||
|
||||
func (o maskOptions) prefix() string {
|
||||
p := strings.TrimRight(o.uploadPrefix, "/")
|
||||
if p == "" {
|
||||
return DefaultUploadPrefix
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func normalizeJSON(raw []byte, step Step, opts maskOptions) ([]byte, []Diff) {
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return raw, nil
|
||||
}
|
||||
@@ -23,7 +41,7 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
|
||||
return raw, nil
|
||||
}
|
||||
var diffs []Diff
|
||||
masked := maskValue("$", val, step, &diffs)
|
||||
masked := maskValue("$", val, step, opts, &diffs)
|
||||
out, err := json.Marshal(masked)
|
||||
if err != nil {
|
||||
return raw, diffs
|
||||
@@ -31,30 +49,36 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
|
||||
return out, diffs
|
||||
}
|
||||
|
||||
func maskValue(path string, val any, step Step, diffs *[]Diff) any {
|
||||
func maskValue(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any {
|
||||
switch v := val.(type) {
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for k, child := range v {
|
||||
out[k] = maskValue(pathJoin(path, k), child, step, diffs)
|
||||
out[k] = maskValue(pathJoin(path, k), child, step, opts, diffs)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
out := make([]any, len(v))
|
||||
for i, child := range v {
|
||||
out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, diffs)
|
||||
out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, opts, diffs)
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return maskLeaf(path, val, step, diffs)
|
||||
return maskLeaf(path, val, step, opts, diffs)
|
||||
}
|
||||
}
|
||||
|
||||
func maskLeaf(path string, val any, step Step, diffs *[]Diff) any {
|
||||
func maskLeaf(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any {
|
||||
key := lastPathKey(path)
|
||||
if key == "slug" || disabledPath(step, path, key) {
|
||||
return val
|
||||
}
|
||||
if key == "url" || key == "thumb_url" {
|
||||
if s, ok := val.(string); ok {
|
||||
return maskUploadURL(path, s, opts.prefix(), diffs)
|
||||
}
|
||||
return val
|
||||
}
|
||||
if key == "collection_key" || key == "client_id" {
|
||||
if val == nil {
|
||||
return nil
|
||||
@@ -154,3 +178,42 @@ func disabledPath(step Step, jsonPath, key string) bool {
|
||||
func strconvQuote(s string) string {
|
||||
return `"` + s + `"`
|
||||
}
|
||||
|
||||
var (
|
||||
// uploadOriginalRe is <partition>/<disk_name>: Winter's partition
|
||||
// directory (the first nine characters of the disk name in three groups)
|
||||
// and a hex disk name with its extension.
|
||||
uploadOriginalRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{9,})(\.[a-z0-9]+)?$`)
|
||||
// uploadThumbRe is <partition>/thumb_<id>_<w>_<h>_<ox>_<oy>_<mode>.<ext>
|
||||
// (Winter getThumbFilename).
|
||||
uploadThumbRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/thumb_([0-9]+)_([0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+)$`)
|
||||
// uploadShapeRe recognises an upload URL under any prefix.
|
||||
uploadShapeRe = regexp.MustCompile(`/[0-9a-f]{3}/[0-9a-f]{3}/[0-9a-f]{3}/(?:thumb_[0-9]+_[0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+|[0-9a-f]{9,}(?:\.[a-z0-9]+)?)$`)
|
||||
)
|
||||
|
||||
// maskUploadURL masks the random parts of an uploaded file's URL, the
|
||||
// partition and disk name of an original and the partition and file id of a
|
||||
// thumbnail, after checking the shape. The prefix, thumbnail size, offsets,
|
||||
// mode and extension stay visible, so a different size or extension still
|
||||
// shows as a mismatch. A URL under another prefix that looks like an upload
|
||||
// is a Diff; any other value is left as it is.
|
||||
func maskUploadURL(path, s, prefix string, diffs *[]Diff) any {
|
||||
if rest, ok := strings.CutPrefix(s, prefix); ok && strings.HasPrefix(rest, "/") {
|
||||
if m := uploadOriginalRe.FindStringSubmatch(rest); m != nil {
|
||||
if m[1]+m[2]+m[3] != m[4][:9] {
|
||||
*diffs = append(*diffs, Diff{Path: path, Expected: "partition from the disk name", Actual: strconvQuote(s)})
|
||||
return s
|
||||
}
|
||||
return prefix + "/<partition>/<disk_name>" + m[5]
|
||||
}
|
||||
if m := uploadThumbRe.FindStringSubmatch(rest); m != nil {
|
||||
return prefix + "/<partition>/thumb_<id>_" + m[5]
|
||||
}
|
||||
*diffs = append(*diffs, Diff{Path: path, Expected: "upload URL " + prefix + "/xxx/yyy/zzz/<disk_name>", Actual: strconvQuote(s)})
|
||||
return s
|
||||
}
|
||||
if uploadShapeRe.MatchString(s) && !strings.Contains(s, "://") {
|
||||
*diffs = append(*diffs, Diff{Path: path, Expected: "upload URL under " + prefix, Actual: strconvQuote(s)})
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -37,6 +37,10 @@ func RecordFlow(ctx context.Context, spec Flow, cfg RecordConfig) (Flow, error)
|
||||
if err != nil {
|
||||
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)
|
||||
}
|
||||
req, err = prepareRequest(req, cfg.BaseDir)
|
||||
if err != nil {
|
||||
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)
|
||||
}
|
||||
resp, err := doStep(ctx, client, cfg.Target, req, limit)
|
||||
if err != nil {
|
||||
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)
|
||||
|
||||
@@ -44,6 +44,10 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
|
||||
skipRest = true
|
||||
continue
|
||||
}
|
||||
req, err = prepareRequest(req, cfg.BaseDir)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err)
|
||||
}
|
||||
got, err := doStep(ctx, client, cfg.Target, req, limit)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err)
|
||||
@@ -71,7 +75,7 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
|
||||
skipRest = true
|
||||
continue
|
||||
}
|
||||
sr.Diffs = append(sr.Diffs, compareStep(want, live.Response)...)
|
||||
sr.Diffs = append(sr.Diffs, compareStepWith(want, live.Response, maskOptions{uploadPrefix: cfg.UploadPrefix})...)
|
||||
if len(sr.Diffs) > 0 {
|
||||
sr.OK = false
|
||||
result.OK = false
|
||||
@@ -90,6 +94,10 @@ func mustMemoryStore() *Store {
|
||||
}
|
||||
|
||||
func compareStep(want Step, got Response) []Diff {
|
||||
return compareStepWith(want, got, maskOptions{})
|
||||
}
|
||||
|
||||
func compareStepWith(want Step, got Response, opts maskOptions) []Diff {
|
||||
var diffs []Diff
|
||||
if want.Response.Status != 0 && got.Status != want.Response.Status {
|
||||
diffs = append(diffs, Diff{
|
||||
@@ -99,6 +107,6 @@ func compareStep(want Step, got Response) []Diff {
|
||||
})
|
||||
}
|
||||
diffs = append(diffs, compareHeaders(want.Response.Headers, got.Headers, want.Headers)...)
|
||||
diffs = append(diffs, compareBodies(want.Response, got, want)...)
|
||||
diffs = append(diffs, compareBodiesWith(want.Response, got, want, opts)...)
|
||||
return diffs
|
||||
}
|
||||
|
||||
@@ -207,6 +207,19 @@ func expandRequest(req Request, store *Store) (Request, error) {
|
||||
return Request{}, err
|
||||
}
|
||||
out.Body = Body(body)
|
||||
if len(req.Parts) > 0 {
|
||||
// Variables expand in part values only, never in file bytes.
|
||||
out.Parts = make([]Part, len(req.Parts))
|
||||
for i, p := range req.Parts {
|
||||
out.Parts[i] = p
|
||||
if p.File == "" {
|
||||
out.Parts[i].Value, err = store.Expand(p.Value)
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -461,6 +474,14 @@ func ScrubStep(store *Store, step *Step) error {
|
||||
step.Request.Query = replaceAll(step.Request.Query, pairs, true)
|
||||
step.Request.Headers = scrubMap(step.Request.Headers, pairs, true)
|
||||
step.Request.Body = Body(replaceAll(string(step.Request.Body), pairs, false))
|
||||
if len(step.Request.Parts) > 0 {
|
||||
parts := make([]Part, len(step.Request.Parts))
|
||||
for i, p := range step.Request.Parts {
|
||||
parts[i] = p
|
||||
parts[i].Value = replaceAll(p.Value, pairs, false)
|
||||
}
|
||||
step.Request.Parts = parts
|
||||
}
|
||||
for _, rule := range step.Capture {
|
||||
if strings.TrimSpace(rule.From) != "request.form" {
|
||||
continue
|
||||
@@ -606,6 +627,11 @@ func rejectUnclassifiedCredentials(step Step) error {
|
||||
if err := check("request body", string(step.Request.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range step.Request.Parts {
|
||||
if err := check("request part "+p.Name, p.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for k, v := range step.Response.Headers {
|
||||
if err := check("response header "+k, v); err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user