feat(12-01): record multipart uploads and match Winter upload URLs

- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the
  thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the
  image size from the header before decoding
- tide requests carry multipart parts (files beside the fixture pinned by
  sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive
  byte-identical bodies
- tide masks the random partition, disk name and file id of url/thumb_url
  upload URLs while still diffing prefix, size, mode and extension, and
  NormalizePublications masks Carbon dates in the published album
This commit is contained in:
Jakub Zych
2026-10-02 11:33:42 +02:00
parent f9b7f2ea33
commit e06e0cc8bf
19 changed files with 870 additions and 34 deletions

View File

@@ -24,7 +24,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he
- Column types: `lagoon.Encrypted` stores AES-256-GCM ciphertext under a key derived from `app.key`, decrypts with previous keys during rotation, and always redacts itself in JSON and string output; `lagoon.Jsonable` stores JSON as TEXT and keeps SQL NULL distinct from an empty value.
- Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns.
- Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports.
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), on-demand thumbnails through `attach.File.Thumb`, static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
## Usage
@@ -151,7 +151,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration {
| `lagoon.WithSoftDeleteCascade` | Runs a cascade inside the parent delete transaction. |
| `lagoon.RegisterJoinTable` | Registers a custom pivot model for a many-to-many field. |
| `lagoon.DecryptLaravelPayload` | Decrypts a Laravel AES-256-CBC payload for data imports. |
| `attach.File` | The `system_files` row model. |
| `attach.File` | The `system_files` row model; `attach.File.URL` is the public URL of the original. |
| `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. |
| `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. |
| `attach.OpenBucket` | Opens the uploads bucket from config. |
| `attach.Publish` | Stores the bucket on the `backpack.App`. |
@@ -198,7 +199,7 @@ storage:
- SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages).
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`).
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails).
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding).
- Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others.
## Testing

View File

@@ -1,6 +1,7 @@
package attach
import (
"bytes"
"context"
"fmt"
"image"
@@ -14,6 +15,10 @@ import (
"github.com/disintegration/imaging"
"gocloud.dev/blob"
// The webp decoder lets image.DecodeConfig and File.Thumb read .webp
// originals. imaging cannot encode webp, so a webp thumbnail holds JPEG
// bytes under the original's .webp name (see defaultEncodeImage).
_ "golang.org/x/image/webp"
)
const (
@@ -67,7 +72,11 @@ func fileExt(diskName string) string {
return strings.ToLower(ext)
}
func publicURL(key string) string {
// PublicURL returns the public URL of a blob key: storage.uploads.
// public_path_prefix and the key joined by exactly one slash. With the
// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
// /storage/app/uploads/public) it is Winter's File::getPath() path.
func PublicURL(key string) string {
prefix := strings.TrimRight(PublicPathPrefix(), "/")
key = strings.TrimLeft(key, "/")
if prefix == "" {
@@ -76,6 +85,15 @@ func publicURL(key string) string {
return prefix + "/" + key
}
// URL returns the public URL of the original file, Winter's File::getPath():
// PublicURL of BlobKey(DiskName).
func (f *File) URL() string {
if f == nil {
return ""
}
return PublicURL(BlobKey(f.DiskName))
}
func defaultResizeImage(src image.Image, w, h int, mode string) image.Image {
switch strings.ToLower(mode) {
case "crop":
@@ -133,21 +151,35 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
return "", fmt.Errorf("attach: thumb exists: %w", err)
}
if exists {
return publicURL(thumbKey), nil
return PublicURL(thumbKey), nil
}
origKey := part + f.DiskName
r, err := bucket.NewReader(ctx, origKey, nil)
if err != nil {
return "", fmt.Errorf("attach: read original: %w", err)
}
src, _, err := image.Decode(io.LimitReader(r, maxThumbSourceBytes))
raw, err := io.ReadAll(io.LimitReader(r, maxThumbSourceBytes))
closeErr := r.Close()
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
return "", fmt.Errorf("attach: read original: %w", err)
}
if closeErr != nil {
return "", closeErr
}
// Check the dimensions from the header before decoding the pixels, so
// a small file that declares a huge image is refused without
// allocating it.
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
}
if int64(cfg.Width)*int64(cfg.Height) > maxThumbSourcePixels {
return "", fmt.Errorf("attach: original image is too large")
}
src, _, err := image.Decode(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
}
bounds := src.Bounds()
if int64(bounds.Dx())*int64(bounds.Dy()) > maxThumbSourcePixels {
return "", fmt.Errorf("attach: original image is too large")
@@ -173,5 +205,5 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
}
return "", closeErr
}
return publicURL(thumbKey), nil
return PublicURL(thumbKey), nil
}

View File

@@ -0,0 +1,107 @@
package attach
import (
"bytes"
"image"
"image/jpeg"
"os"
"path/filepath"
"testing"
"git.golem15.com/golem15/summercms/modules/compass"
"gocloud.dev/blob"
)
// winterLayoutBucket opens a mem:// bucket with the WinterCMS public prefix
// and restores the framework default afterwards.
func winterLayoutBucket(t *testing.T) *blob.Bucket {
t.Helper()
dir := t.TempDir()
body := "uploads:\n bucket_url: \"mem://\"\n public_path_prefix: \"/storage/app/uploads/public\"\n"
if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
cfg, err := compass.Open(compass.Options{Dir: dir, Env: "development", Environ: []string{"SUMMER_ENV=development"}})
if err != nil {
t.Fatal(err)
}
bucket, err := OpenBucket(t.Context(), cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = bucket.Close()
setPublicPathPrefix(defaultPublicPathPrefix)
})
return bucket
}
func TestFileURLWinterLayout(t *testing.T) {
bucket := winterLayoutBucket(t)
// A WinterCMS disk name: uniqid('', true) without the dot, plus the
// extension.
f := &File{ID: 12, DiskName: "651a2b3c4d5e61234567.png"}
if got, want := f.URL(), "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
t.Fatalf("URL = %q, want %q", got, want)
}
if got, want := PublicURL("/651/a2b/3c4/x.png"), "/storage/app/uploads/public/651/a2b/3c4/x.png"; got != want {
t.Fatalf("PublicURL = %q, want %q", got, want)
}
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), &blob.WriterOptions{ContentType: "image/png"}); err != nil {
t.Fatal(err)
}
thumb, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
if err != nil {
t.Fatal(err)
}
// Winter getThumbFilename: implode('_', [thumb, id, w, h, ox, oy, mode.ext]).
if want := "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png"; thumb != want {
t.Fatalf("thumb = %q, want %q", thumb, want)
}
var nilFile *File
if nilFile.URL() != "" {
t.Fatal("nil file URL must be empty")
}
}
// webpFixture is a 16x12 lossless WebP: blue left half, red right half.
var webpFixture = []byte{
0x52, 0x49, 0x46, 0x46, 0x2a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, 0x50, 0x38, 0x4c,
0x1d, 0x00, 0x00, 0x00, 0x2f, 0x0f, 0xc0, 0x02, 0x00, 0x0f, 0x70, 0x14, 0xfb, 0x53, 0xd0, 0x5e,
0x88, 0x7b, 0xfe, 0x83, 0x07, 0x62, 0xc1, 0x64, 0xfe, 0xd2, 0xbd, 0x21, 0x44, 0xf4, 0x3f, 0x74,
0x01, 0x00,
}
func TestThumbWebP(t *testing.T) {
cfg, format, err := image.DecodeConfig(bytes.NewReader(webpFixture))
if err != nil {
t.Fatalf("DecodeConfig: %v", err)
}
if format != "webp" || cfg.Width != 16 || cfg.Height != 12 {
t.Fatalf("config = %s %dx%d", format, cfg.Width, cfg.Height)
}
bucket := winterLayoutBucket(t)
f := &File{ID: 5, DiskName: "abcdef123456789012345.webp"}
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), webpFixture, &blob.WriterOptions{ContentType: "image/webp"}); err != nil {
t.Fatal(err)
}
url, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
if err != nil {
t.Fatalf("Thumb: %v", err)
}
if want := "/storage/app/uploads/public/abc/def/123/thumb_5_200_200_0_0_crop.webp"; url != want {
t.Fatalf("thumb url = %q, want %q", url, want)
}
raw, err := bucket.ReadAll(t.Context(), "abc/def/123/thumb_5_200_200_0_0_crop.webp")
if err != nil {
t.Fatal(err)
}
// imaging cannot encode webp: the thumb is JPEG bytes under the .webp name.
img, err := jpeg.Decode(bytes.NewReader(raw))
if err != nil {
t.Fatalf("thumb is not JPEG: %v", err)
}
if b := img.Bounds(); b.Dx() != 200 || b.Dy() != 200 {
t.Fatalf("thumb size = %v", b)
}
}

View File

@@ -12,13 +12,14 @@ HTTP parity toolkit that records request and response fixtures from a reference
- Flow fixtures: `tide.Flow` is a versioned, ordered list of `tide.Step` values, loaded strictly (unknown fields rejected) with `tide.LoadFlow` and `tide.ParseFlow`, and written atomically with `tide.SaveFlow` or `tide.SaveFlowExclusive`. Response bodies can live in sidecar files, confined to the fixture directory and checked against an optional SHA-256 digest.
- Recording: `tide.RecordFlow` executes a spec flow against a target URL and fills in the responses, with a bounded body size (`tide.DefaultMaxBody`, 8 MiB).
- Multipart uploads: a request can carry `parts` (`tide.Part`: a text field with `value`, or a file field with `file`, `filename`, `content_type` and `sha256`) instead of a `body`. File bytes stay in files beside the fixture (for example `files/cover.png`, read from `tide.RecordConfig` or `tide.ReplayConfig` `BaseDir`) and are never inlined into the YAML. Parts are encoded in declaration order with one fixed boundary, `tide.MultipartBoundary`, so the reference backend and the port receive byte-identical bodies; the encoded Content-Type replaces a recorded multipart Content-Type. Variables expand in part values, never in file bytes. `tide.LoadFlow` fails when a part file is missing or its SHA-256 differs, naming the part, and a request with both `body` and `parts` is invalid.
- Recording proxy: `tide.NewProxy` builds a reverse proxy that only binds to and forwards to loopback addresses, groups traffic into named sessions (from the `tide.SessionHeader` request header or a default session) and writes one fixture per complete session on `tide.Proxy.Flush`.
- Capture rules: `tide.Rules` (loaded with `tide.LoadRules`) decide which request and response headers are kept per route and which values are captured into variables, from response JSON paths, headers, redirect query strings or form fields.
- Variables: `tide.Store` holds captured values such as tokens and IDs in a mode-0600 file, `tide.Store.Expand` substitutes `{{name}}` placeholders before a request is sent, and `tide.ScrubStep` puts placeholders back into fixtures. Scrubbing fails when a step still holds an unclassified token- or password-shaped value, so credentials do not leak into committed fixtures.
- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte.
- Replay and diff: `tide.ReplayFlow` re-sends each step, compares status, a fixed set of contract headers and the body, and returns `tide.Result` with per-step `tide.Diff` entries. JSON bodies are compared structurally after masking `id`, `*_id` and `*_ids` values and `*_at` timestamps; other bodies are compared byte for byte. Uploaded-file URLs under `url` and `thumb_url` keys are compared by shape: under the uploads prefix (`tide.ReplayConfig` `UploadPrefix`, default `tide.DefaultUploadPrefix`, `/storage/app/uploads/public`) an original must be `<prefix>/xxx/yyy/zzz/<disk_name>` with the partition taken from the disk name, and a thumbnail `<prefix>/xxx/yyy/zzz/thumb_<id>_<w>_<h>_<ox>_<oy>_<mode>.<ext>`. The partition, disk name and file id are masked; the prefix, size, offsets, mode and extension stay, so a thumbnail of another size or an upload URL under another prefix is still a difference.
- Fake Centrifugo: `tide.NewCentrifugoRecorder` returns an `http.Handler` that records every POST to a path ending in `/publish` or `/broadcast` as a `tide.Publication` (method, path, whether `Authorization: apikey <key>` carried the configured key, JSON body) and answers `{"result":{}}`. Paths ending in `/presence` answer `{"result":{"presence":{}}}`, `/unsubscribe` and `/info` answer `{"result":{}}`, anything else is 404. Bodies are capped at `tide.MaxPublicationBody` (1 MiB). The API key is only compared, never stored. `tide.CentrifugoRecorder.ListenAndServe` binds loopback addresses only, like the recording proxy.
- Broadcast goldens: `tide.RecordBroadcasts` runs a flow against a loopback reference backend whose Centrifugo API URL points at a recorder on `tide.DefaultCentrifugoListen` (`127.0.0.1:8424`). With `tide.BroadcastConfig` `Step` set, earlier steps run as setup and only that step's publications are kept. The result is a `tide.BroadcastGolden`, written with `tide.WriteBroadcastGolden` (which refuses token-shaped bodies) and read strictly with `tide.LoadBroadcastGolden`. A golden with `pending` set is recorded but not yet asserted.
- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`.
- Broadcast normalisation: `tide.NormalizePublications` masks only `$.data.timestamp` and `$.data.payload.timestamp` (ISO 8601 with an offset) as `"{{timestamp}}"`, `$.data.payload.actor` (an object of exactly `user_id` and `name`) as `"{{actor}}"`, and values equal to an `id:*` variable of a `tide.Store`: numbers or strings under `id`, `*_id` or `*_ids` keys, and the numeric last segment of a channel name such as `room:12`. Carbon `+00:00` values of `*_at` keys anywhere under `$.data.payload.album` become `"{{datetime}}"`; a date of another shape is left as it is, so a format change shows as a difference. A masked number is written as a bare `{{id:name}}`, so a number that becomes a string still differs. A value matching two id variables is an error. `tide.DiffPublications` compares the count, method, path, authorization flag and body (structurally, key order ignored) and reports paths such as `$[0].body.data.payload.id`.
- Manifests: `tide.Manifest` lists routes with auth groups, a pending or ported status, cases and fixture paths; `tide.RecordManifest` records missing cases in batches of at most `tide.MaxBatch`, and `tide.ReplayManifest` replays every recorded case into a `tide.Coverage` table.
## Usage
@@ -88,6 +89,12 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) {
| `tide.LoadFlow` | Reads and validates a flow file. |
| `tide.SaveFlow` | Writes a validated flow atomically. |
| `tide.RecordFlow` | Executes a spec flow against a target and returns the recorded flow. |
| `tide.Request` | The outbound call of a step: method, path, query, headers and a `Body` or multipart `Parts`. |
| `tide.Part` | One multipart field: a text value, or a file beside the fixture pinned by its SHA-256. |
| `tide.MultipartBoundary` | The fixed boundary multipart request bodies are encoded with. |
| `tide.RecordConfig` | Target, client, body cap, vars store, capture rules and part-file directory for `tide.RecordFlow`. |
| `tide.ReplayConfig` | Target, client, body cap, vars store, fixture directory and upload URL prefix for `tide.ReplayFlow`. |
| `tide.DefaultUploadPrefix` | The default uploads URL prefix, `/storage/app/uploads/public`. |
| `tide.ReplayFlow` | Replays a recorded flow against a target and diffs every step. |
| `tide.Result` | Replay outcome: overall status and per-step `tide.StepResult` values. |
| `tide.Diff` | One structural JSON or byte-level mismatch. |
@@ -109,7 +116,7 @@ for _, d := range tide.DiffPublications(golden.Publications, norm) {
| `tide.BroadcastGolden` | Versioned broadcast golden: name, flow, optional pending reason, publications. |
| `tide.LoadBroadcastGolden` | Reads a golden strictly and checks every body parses. |
| `tide.WriteBroadcastGolden` | Writes a golden atomically, refusing token-shaped bodies. |
| `tide.NormalizePublications` | Masks timestamps, the actor and captured ids in publication bodies. |
| `tide.NormalizePublications` | Masks timestamps, the actor, album dates and captured ids in publication bodies. |
| `tide.DiffPublications` | Structural diff of two publication lists. |
| `tide.DefaultCentrifugoListen` | Default recorder address, `127.0.0.1:8424`. |
| `tide.Coverage` | Recorded, passing, failing and unrecorded counts, with table rows and a summary line. |

View File

@@ -164,6 +164,9 @@ var channelIDRe = regexp.MustCompile(`^((?:[a-z][a-z0-9_.-]*:)+)([0-9]+)$`)
// when they have the ISO 8601 offset shape;
// - $.data.payload.actor becomes "{{actor}}" when it is an object with
// exactly user_id and name;
// - a *_at value anywhere under $.data.payload.album becomes
// "{{datetime}}" when it has Carbon's +00:00 shape; any other shape is
// left as it is, so a format change shows up as a diff;
// - a number or string under an id, *_id or *_ids key that equals an id:*
// variable of store becomes that {{id:name}} placeholder (bare for a
// number, quoted for a string), and so does the numeric last segment of
@@ -304,6 +307,9 @@ func (n *normalizer) walk(path, key string, v *onode) *onode {
}
return v
case kindString:
if strings.HasPrefix(path, "$.data.payload.album.") && strings.HasSuffix(key, "_at") && carbonOffsetRe.MatchString(v.str) {
return &onode{kind: kindString, str: "{{datetime}}"}
}
if key != "" && isIDKey(key) {
if name, ok := n.lookup(v.str); ok {
return &onode{kind: kindString, str: "{{" + name + "}}"}

View File

@@ -13,11 +13,15 @@ import (
)
func compareBodies(want, got Response, step Step) []Diff {
return compareBodiesWith(want, got, step, maskOptions{})
}
func compareBodiesWith(want, got Response, step Step, opts maskOptions) []Diff {
wantJSON := isJSONContentType(want.Headers)
gotJSON := isJSONContentType(got.Headers)
if wantJSON && gotJSON {
wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step)
gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step)
wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step, opts)
gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step, opts)
diffs := append([]Diff{}, wantDiffs...)
diffs = append(diffs, gotDiffs...)
diffs = append(diffs, diffJSON(wantRaw, gotRaw)...)

View File

@@ -25,13 +25,21 @@ func (b *Body) UnmarshalYAML(data []byte) error {
return nil
}
// LoadFlow reads and validates a version-1 YAML flow from path.
// LoadFlow reads and validates a version-1 YAML flow from path. Request
// part files are checked against their sha256, relative to path's directory.
func LoadFlow(path string) (Flow, error) {
raw, err := os.ReadFile(path)
if err != nil {
return Flow{}, fmt.Errorf("tide: read %s: %w", path, err)
}
return ParseFlow(raw)
flow, err := ParseFlow(raw)
if err != nil {
return Flow{}, err
}
if err := verifyPartFiles(filepath.Dir(path), flow); err != nil {
return Flow{}, err
}
return flow, nil
}
// ParseFlow decodes a version-1 YAML flow, rejecting unknown fields.
@@ -163,6 +171,30 @@ func writeRequest(b *strings.Builder, indent int, req Request) {
}
writeHeaders(b, indent, req.Headers)
writeBody(b, indent, string(req.Body))
writeParts(b, indent, req.Parts)
}
func writeParts(b *strings.Builder, indent int, parts []Part) {
if len(parts) == 0 {
return
}
pad := strings.Repeat(" ", indent)
fmt.Fprintf(b, "%sparts:\n", pad)
inner := strings.Repeat(" ", indent+2)
for _, p := range parts {
fmt.Fprintf(b, "%s- name: %s\n", inner, encodeScalar(p.Name))
for _, kv := range [][2]string{
{"value", p.Value},
{"file", p.File},
{"filename", p.Filename},
{"content_type", p.ContentType},
{"sha256", p.SHA256},
} {
if kv[1] != "" {
fmt.Fprintf(b, "%s %s: %s\n", inner, kv[0], encodeScalar(kv[1]))
}
}
}
}
func writeResponse(b *strings.Builder, indent int, resp Response) {

View File

@@ -42,6 +42,9 @@ type Request struct {
Query string `yaml:"query,omitempty"`
Headers map[string]string `yaml:"headers,omitempty"`
Body Body `yaml:"body,omitempty"`
// Parts is a multipart/form-data body, encoded with MultipartBoundary
// when the step runs. A request has Body or Parts, never both.
Parts []Part `yaml:"parts,omitempty"`
}
// Response is the recorded or expected HTTP reply.
@@ -73,21 +76,28 @@ type NormalizeRule struct {
type Body string
// RecordConfig injects the HTTP target, client and body bound for recording.
// BaseDir is the fixture directory that request part files are read from.
type RecordConfig struct {
Target string
Client *http.Client
MaxBody int64
Store *Store
Rules Rules
BaseDir string
}
// ReplayConfig injects the HTTP target, client and body bound for replay.
// BaseDir is the fixture directory that body_file sidecars and request part
// files are read from.
// UploadPrefix is the public URL prefix of uploaded files whose url and
// thumb_url values are compared by shape; empty means DefaultUploadPrefix.
type ReplayConfig struct {
Target string
Client *http.Client
MaxBody int64
Store *Store
BaseDir string
Target string
Client *http.Client
MaxBody int64
Store *Store
BaseDir string
UploadPrefix string
}
// Result is the outcome of replaying a flow.
@@ -169,6 +179,9 @@ func validateFlow(flow Flow) error {
if err := validateSidecar(step.Response.BodyFile); err != nil {
return fmt.Errorf("tide: step %s: %w", step.ID, err)
}
if err := validateParts(step.Request); err != nil {
return fmt.Errorf("tide: step %s: %w", step.ID, err)
}
}
return nil
}

201
modules/tide/multipart.go Normal file
View File

@@ -0,0 +1,201 @@
package tide
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"mime/multipart"
"net/textproto"
"os"
"path"
"path/filepath"
"regexp"
"strings"
)
// MultipartBoundary is the fixed boundary every multipart request body is
// encoded with, so a recording against one backend and a replay against
// another send byte-identical bodies.
const MultipartBoundary = "SummerTideMultipartBoundary7MA4YWxkTrZu0gW"
// Part is one field of a multipart/form-data request. A text field sets
// Value; a file field sets File, a path relative to the fixture directory
// (for example files/cover.png), whose bytes are never inlined into the
// YAML, with SHA256 pinning its content. Filename defaults to the base name
// of File and ContentType to application/octet-stream.
type Part struct {
Name string `yaml:"name"`
Value string `yaml:"value,omitempty"`
File string `yaml:"file,omitempty"`
Filename string `yaml:"filename,omitempty"`
ContentType string `yaml:"content_type,omitempty"`
SHA256 string `yaml:"sha256,omitempty"`
}
var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
// validateParts checks the shape of a request's parts without reading files.
func validateParts(req Request) error {
if len(req.Parts) == 0 {
return nil
}
if req.Body != "" {
return fmt.Errorf("request has both body and parts")
}
for i, p := range req.Parts {
if strings.TrimSpace(p.Name) == "" {
return fmt.Errorf("parts[%d] is missing name", i)
}
if p.File == "" {
if p.Filename != "" || p.ContentType != "" || p.SHA256 != "" {
return fmt.Errorf("part %q has file attributes but no file", p.Name)
}
continue
}
if p.Value != "" {
return fmt.Errorf("part %q has both value and file", p.Name)
}
if err := validateSidecar(p.File); err != nil {
return fmt.Errorf("part %q: %w", p.Name, err)
}
if !sha256Hex.MatchString(p.SHA256) {
return fmt.Errorf("part %q needs the sha256 of its file", p.Name)
}
}
return nil
}
// verifyPartFiles checks that every part file of flow exists under base and
// matches its sha256.
func verifyPartFiles(base string, flow Flow) error {
for _, step := range flow.Steps {
for _, p := range step.Request.Parts {
if p.File == "" {
continue
}
if _, err := readPartFile(base, p); err != nil {
return fmt.Errorf("tide: step %s: %w", step.ID, err)
}
}
}
return nil
}
func readPartFile(base string, p Part) ([]byte, error) {
if err := validateSidecar(p.File); err != nil {
return nil, fmt.Errorf("part %q: %w", p.Name, err)
}
full := p.File
if base != "" {
full = filepath.Join(base, p.File)
}
resolved, err := resolvePath(full)
if err != nil {
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
}
if base != "" {
root, err := resolvePath(base)
if err != nil {
return nil, fmt.Errorf("fixture dir: %w", err)
}
if resolved != root && !strings.HasPrefix(resolved, root+string(os.PathSeparator)) {
return nil, fmt.Errorf("part %q file %q escapes the fixture directory", p.Name, p.File)
}
}
raw, err := os.ReadFile(resolved)
if err != nil {
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
}
sum := sha256.Sum256(raw)
if !strings.EqualFold(hex.EncodeToString(sum[:]), p.SHA256) {
return nil, fmt.Errorf("part %q file %s sha256 mismatch", p.Name, p.File)
}
return raw, nil
}
var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
// encodeParts writes parts in declaration order as a multipart/form-data
// body with MultipartBoundary and returns the body and its Content-Type.
func encodeParts(base string, parts []Part) ([]byte, string, error) {
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
if err := w.SetBoundary(MultipartBoundary); err != nil {
return nil, "", err
}
marker := []byte(MultipartBoundary)
for _, p := range parts {
h := make(textproto.MIMEHeader)
var content []byte
if p.File == "" {
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, quoteEscaper.Replace(p.Name)))
content = []byte(p.Value)
} else {
raw, err := readPartFile(base, p)
if err != nil {
return nil, "", err
}
filename := p.Filename
if filename == "" {
filename = path.Base(filepath.ToSlash(p.File))
}
ct := p.ContentType
if ct == "" {
ct = "application/octet-stream"
}
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, quoteEscaper.Replace(p.Name), quoteEscaper.Replace(filename)))
h.Set("Content-Type", ct)
content = raw
}
if bytes.Contains(content, marker) {
return nil, "", fmt.Errorf("part %q contains the multipart boundary", p.Name)
}
pw, err := w.CreatePart(h)
if err != nil {
return nil, "", err
}
if _, err := pw.Write(content); err != nil {
return nil, "", err
}
}
if err := w.Close(); err != nil {
return nil, "", err
}
return buf.Bytes(), w.FormDataContentType(), nil
}
// prepareRequest encodes a request's parts into its body. The multipart
// Content-Type replaces a recorded multipart Content-Type (whose boundary
// would be stale) and is added when none is recorded; a recorded non-
// multipart Content-Type is kept as it is.
func prepareRequest(req Request, base string) (Request, error) {
if len(req.Parts) == 0 {
return req, nil
}
if err := validateParts(req); err != nil {
return Request{}, err
}
body, ct, err := encodeParts(base, req.Parts)
if err != nil {
return Request{}, err
}
out := req
out.Body = Body(body)
out.Headers = make(map[string]string, len(req.Headers)+1)
set := false
for k, v := range req.Headers {
if strings.EqualFold(k, "Content-Type") {
set = true
if media, _, err := mime.ParseMediaType(v); err != nil || strings.HasPrefix(media, "multipart/") {
v = ct
}
}
out.Headers[k] = v
}
if !set {
out.Headers["Content-Type"] = ct
}
return out, nil
}

View File

@@ -0,0 +1,296 @@
package tide
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"mime"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
// multipartFixture writes files/cover.png under a temp fixture directory and
// returns the directory and a one-step upload flow that references it.
func multipartFixture(t *testing.T) (string, Flow) {
t.Helper()
dir := t.TempDir()
content := []byte("\x89PNG\r\n\x1a\nfake image bytes")
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), content, 0o644); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(content)
flow := Flow{
Version: CurrentVersion,
Name: "upload",
Steps: []Step{{
ID: "upload",
Request: Request{
Method: "POST",
Path: "/posts/1/photos",
Headers: map[string]string{"Content-Type": "multipart/form-data; boundary=recorded-elsewhere"},
Parts: []Part{
{Name: "title", Value: "Cover {{id:post}}"},
{Name: "file", File: "files/cover.png", ContentType: "image/png", SHA256: hex.EncodeToString(sum[:])},
},
},
}},
}
return dir, flow
}
type rawCapture struct {
mu sync.Mutex
bodies [][]byte
types []string
}
func (c *rawCapture) handler(t *testing.T) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
raw, err := io.ReadAll(r.Body)
if err != nil {
t.Error(err)
}
c.mu.Lock()
c.bodies = append(c.bodies, raw)
c.types = append(c.types, r.Header.Get("Content-Type"))
c.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok":true}`))
}
}
func TestMultipartRecordReplaySendsIdenticalBytes(t *testing.T) {
dir, flow := multipartFixture(t)
capture := &rawCapture{}
srv := httptest.NewServer(capture.handler(t))
t.Cleanup(srv.Close)
store := mustMemoryStore()
store.Set("id:post", "7")
recorded, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir})
if err != nil {
t.Fatalf("record: %v", err)
}
if _, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
t.Fatalf("second record: %v", err)
}
if _, err := ReplayFlow(t.Context(), recorded, ReplayConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
t.Fatalf("replay: %v", err)
}
if len(capture.bodies) != 3 {
t.Fatalf("requests = %d", len(capture.bodies))
}
for i := 1; i < 3; i++ {
if !bytes.Equal(capture.bodies[0], capture.bodies[i]) {
t.Fatalf("body %d differs:\n%q\n%q", i, capture.bodies[0], capture.bodies[i])
}
if capture.types[i] != capture.types[0] {
t.Fatalf("content type %d = %q", i, capture.types[i])
}
}
media, params, err := mime.ParseMediaType(capture.types[0])
if err != nil || media != "multipart/form-data" || params["boundary"] != MultipartBoundary {
t.Fatalf("content type = %q", capture.types[0])
}
form, err := multipart.NewReader(bytes.NewReader(capture.bodies[0]), MultipartBoundary).ReadForm(1 << 20)
if err != nil {
t.Fatal(err)
}
if got := form.Value["title"]; len(got) != 1 || got[0] != "Cover 7" {
t.Fatalf("title = %q (variables expand in values)", got)
}
fh := form.File["file"]
if len(fh) != 1 || fh[0].Filename != "cover.png" || fh[0].Header.Get("Content-Type") != "image/png" {
t.Fatalf("file part = %+v", fh)
}
// The recording keeps the parts, never the file bytes.
raw, err := marshalFlow(recorded)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "fake image bytes") || !strings.Contains(string(raw), "file: files/cover.png") {
t.Fatalf("recorded flow:\n%s", raw)
}
parsed, err := ParseFlow(raw)
if err != nil {
t.Fatalf("parse recorded flow: %v", err)
}
if len(parsed.Steps[0].Request.Parts) != 2 || parsed.Steps[0].Request.Parts[1].SHA256 != flow.Steps[0].Request.Parts[1].SHA256 {
t.Fatalf("parts after round trip = %+v", parsed.Steps[0].Request.Parts)
}
}
func TestMultipartTamperedPartFileFailsLoad(t *testing.T) {
dir, flow := multipartFixture(t)
raw, err := marshalFlow(flow)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, "upload.yaml")
if err := os.WriteFile(path, raw, 0o644); err != nil {
t.Fatal(err)
}
if _, err := LoadFlow(path); err != nil {
t.Fatalf("intact fixture: %v", err)
}
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), []byte("tampered"), 0o644); err != nil {
t.Fatal(err)
}
_, err = LoadFlow(path)
if err == nil || !strings.Contains(err.Error(), `part "file"`) || !strings.Contains(err.Error(), "sha256") {
t.Fatalf("tampered file: %v", err)
}
if err := os.Remove(filepath.Join(dir, "files", "cover.png")); err != nil {
t.Fatal(err)
}
if _, err := LoadFlow(path); err == nil {
t.Fatal("missing part file must fail the load")
}
}
func TestMultipartRejectsInvalidParts(t *testing.T) {
_, flow := multipartFixture(t)
cases := map[string]func(*Request){
"body and parts": func(r *Request) { r.Body = "x=1" },
"escaping file": func(r *Request) { r.Parts[1].File = "../secret.png" },
"missing sha256": func(r *Request) { r.Parts[1].SHA256 = "" },
"value and file": func(r *Request) { r.Parts[1].Value = "x" },
"missing name": func(r *Request) { r.Parts[0].Name = "" },
}
for name, mutate := range cases {
f := flow
f.Steps = []Step{flow.Steps[0]}
f.Steps[0].Request.Parts = append([]Part(nil), flow.Steps[0].Request.Parts...)
mutate(&f.Steps[0].Request)
if err := validateFlow(f); err == nil {
t.Fatalf("%s: validateFlow accepted the flow", name)
}
}
}
func TestMultipartKeepsNonMultipartContentType(t *testing.T) {
dir, flow := multipartFixture(t)
req := flow.Steps[0].Request
req.Headers = map[string]string{"Content-Type": "text/plain"}
out, err := prepareRequest(req, dir)
if err != nil {
t.Fatal(err)
}
if out.Headers["Content-Type"] != "text/plain" {
t.Fatalf("content type = %q", out.Headers["Content-Type"])
}
req.Headers = nil
out, err = prepareRequest(req, dir)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out.Headers["Content-Type"], MultipartBoundary) {
t.Fatalf("content type = %q", out.Headers["Content-Type"])
}
}
func uploadBody(url, thumb string) []byte {
b, _ := json.Marshal(map[string]any{"data": map[string]any{"photos": []any{map[string]any{"id": 1, "url": url, "thumb_url": thumb}}}})
return b
}
func TestNormalizeUploadURLMasksRandomParts(t *testing.T) {
step := Step{ID: "u"}
php := uploadBody(
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
goBody := uploadBody(
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(goBody)}, step); len(diffs) != 0 {
t.Fatalf("diffs = %+v", diffs)
}
// An external URL under url is not an upload and is compared as is.
ext := uploadBody("https://img.example.com/a.jpg", "https://img.example.com/b.jpg")
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(ext)}, Response{Headers: jsonCT(), Body: Body(ext)}, step); len(diffs) != 0 {
t.Fatalf("external diffs = %+v", diffs)
}
}
func TestNormalizeUploadURLReportsWrongShape(t *testing.T) {
step := Step{ID: "u"}
php := uploadBody(
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
cases := map[string][]byte{
"thumb size": uploadBody(
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_100_100_0_0_crop.png"),
"prefix": uploadBody(
"/storage/uploads/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
"partition": uploadBody(
"/storage/app/uploads/public/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
"partition not from disk name": uploadBody(
"/storage/app/uploads/public/aaa/bbb/ccc/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
"extension": uploadBody(
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.jpg",
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
}
for name, got := range cases {
diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(got)}, step)
if len(diffs) == 0 {
t.Fatalf("%s: masking hid the mismatch", name)
}
}
// A configured prefix replaces the WinterCMS default.
alt := uploadBody("/files/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", "/files/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
alt2 := uploadBody("/files/123/456/789/1234567890abcdef.png", "/files/123/456/789/thumb_9_200_200_0_0_crop.png")
if diffs := compareBodiesWith(Response{Headers: jsonCT(), Body: Body(alt)}, Response{Headers: jsonCT(), Body: Body(alt2)}, step, maskOptions{uploadPrefix: "/files"}); len(diffs) != 0 {
t.Fatalf("custom prefix diffs = %+v", diffs)
}
}
func TestNormalizePublicationAlbumDates(t *testing.T) {
store := mustMemoryStore()
pub := func(created, updated string) []Publication {
return []Publication{{Method: "POST", Path: "/api/publish", Body: json.RawMessage(
`{"channel":"c","data":{"payload":{"album":{"name":"x","created_at":"` + created + `","updated_at":"` + updated +
`","market_price_checked_at":null,"photos":[{"created_at":"` + created + `"}]},"created_at":"2026-01-01T00:00:00+00:00"}}}`)}}
}
a, err := NormalizePublications(pub("2026-09-30T11:21:55+00:00", "2026-09-30T11:21:56+00:00"), store)
if err != nil {
t.Fatal(err)
}
b, err := NormalizePublications(pub("2026-10-02T08:00:00+00:00", "2026-10-02T08:00:01+00:00"), store)
if err != nil {
t.Fatal(err)
}
if diffs := DiffPublications(a, b); len(diffs) != 0 {
t.Fatalf("diffs = %+v", diffs)
}
if !strings.Contains(string(a[0].Body), `"updated_at":"{{datetime}}"`) || !strings.Contains(string(a[0].Body), `"market_price_checked_at":null`) {
t.Fatalf("album dates not masked: %s", a[0].Body)
}
// Only the album subtree is masked; a payload date outside it stays.
if !strings.Contains(string(a[0].Body), `},"created_at":"2026-01-01T00:00:00+00:00"`) {
t.Fatalf("over-normalised: %s", a[0].Body)
}
// A Z-suffixed album date keeps its value, so a format change is a diff.
z, err := NormalizePublications(pub("2026-10-02T08:00:00Z", "2026-10-02T08:00:01+00:00"), store)
if err != nil {
t.Fatal(err)
}
if diffs := DiffPublications(a, z); len(diffs) == 0 {
t.Fatal("a Z album date must show as a diff")
}
}

View File

@@ -14,7 +14,25 @@ const (
maskID = "<id>"
)
func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
// DefaultUploadPrefix is the WinterCMS public uploads URL prefix
// (cms.storage.uploads.path plus /public) that url and thumb_url values are
// checked against when ReplayConfig.UploadPrefix is empty.
const DefaultUploadPrefix = "/storage/app/uploads/public"
// maskOptions configures the response-body normalizer.
type maskOptions struct {
uploadPrefix string
}
func (o maskOptions) prefix() string {
p := strings.TrimRight(o.uploadPrefix, "/")
if p == "" {
return DefaultUploadPrefix
}
return p
}
func normalizeJSON(raw []byte, step Step, opts maskOptions) ([]byte, []Diff) {
if len(strings.TrimSpace(string(raw))) == 0 {
return raw, nil
}
@@ -23,7 +41,7 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
return raw, nil
}
var diffs []Diff
masked := maskValue("$", val, step, &diffs)
masked := maskValue("$", val, step, opts, &diffs)
out, err := json.Marshal(masked)
if err != nil {
return raw, diffs
@@ -31,30 +49,36 @@ func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) {
return out, diffs
}
func maskValue(path string, val any, step Step, diffs *[]Diff) any {
func maskValue(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any {
switch v := val.(type) {
case map[string]any:
out := make(map[string]any, len(v))
for k, child := range v {
out[k] = maskValue(pathJoin(path, k), child, step, diffs)
out[k] = maskValue(pathJoin(path, k), child, step, opts, diffs)
}
return out
case []any:
out := make([]any, len(v))
for i, child := range v {
out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, diffs)
out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, opts, diffs)
}
return out
default:
return maskLeaf(path, val, step, diffs)
return maskLeaf(path, val, step, opts, diffs)
}
}
func maskLeaf(path string, val any, step Step, diffs *[]Diff) any {
func maskLeaf(path string, val any, step Step, opts maskOptions, diffs *[]Diff) any {
key := lastPathKey(path)
if key == "slug" || disabledPath(step, path, key) {
return val
}
if key == "url" || key == "thumb_url" {
if s, ok := val.(string); ok {
return maskUploadURL(path, s, opts.prefix(), diffs)
}
return val
}
if key == "collection_key" || key == "client_id" {
if val == nil {
return nil
@@ -154,3 +178,42 @@ func disabledPath(step Step, jsonPath, key string) bool {
func strconvQuote(s string) string {
return `"` + s + `"`
}
var (
// uploadOriginalRe is <partition>/<disk_name>: Winter's partition
// directory (the first nine characters of the disk name in three groups)
// and a hex disk name with its extension.
uploadOriginalRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{9,})(\.[a-z0-9]+)?$`)
// uploadThumbRe is <partition>/thumb_<id>_<w>_<h>_<ox>_<oy>_<mode>.<ext>
// (Winter getThumbFilename).
uploadThumbRe = regexp.MustCompile(`^/([0-9a-f]{3})/([0-9a-f]{3})/([0-9a-f]{3})/thumb_([0-9]+)_([0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+)$`)
// uploadShapeRe recognises an upload URL under any prefix.
uploadShapeRe = regexp.MustCompile(`/[0-9a-f]{3}/[0-9a-f]{3}/[0-9a-f]{3}/(?:thumb_[0-9]+_[0-9]+_[0-9]+_-?[0-9]+_-?[0-9]+_[a-z0-9]+\.[a-z0-9]+|[0-9a-f]{9,}(?:\.[a-z0-9]+)?)$`)
)
// maskUploadURL masks the random parts of an uploaded file's URL, the
// partition and disk name of an original and the partition and file id of a
// thumbnail, after checking the shape. The prefix, thumbnail size, offsets,
// mode and extension stay visible, so a different size or extension still
// shows as a mismatch. A URL under another prefix that looks like an upload
// is a Diff; any other value is left as it is.
func maskUploadURL(path, s, prefix string, diffs *[]Diff) any {
if rest, ok := strings.CutPrefix(s, prefix); ok && strings.HasPrefix(rest, "/") {
if m := uploadOriginalRe.FindStringSubmatch(rest); m != nil {
if m[1]+m[2]+m[3] != m[4][:9] {
*diffs = append(*diffs, Diff{Path: path, Expected: "partition from the disk name", Actual: strconvQuote(s)})
return s
}
return prefix + "/<partition>/<disk_name>" + m[5]
}
if m := uploadThumbRe.FindStringSubmatch(rest); m != nil {
return prefix + "/<partition>/thumb_<id>_" + m[5]
}
*diffs = append(*diffs, Diff{Path: path, Expected: "upload URL " + prefix + "/xxx/yyy/zzz/<disk_name>", Actual: strconvQuote(s)})
return s
}
if uploadShapeRe.MatchString(s) && !strings.Contains(s, "://") {
*diffs = append(*diffs, Diff{Path: path, Expected: "upload URL under " + prefix, Actual: strconvQuote(s)})
}
return s
}

View File

@@ -37,6 +37,10 @@ func RecordFlow(ctx context.Context, spec Flow, cfg RecordConfig) (Flow, error)
if err != nil {
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)
}
req, err = prepareRequest(req, cfg.BaseDir)
if err != nil {
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)
}
resp, err := doStep(ctx, client, cfg.Target, req, limit)
if err != nil {
return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err)

View File

@@ -44,6 +44,10 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
skipRest = true
continue
}
req, err = prepareRequest(req, cfg.BaseDir)
if err != nil {
return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err)
}
got, err := doStep(ctx, client, cfg.Target, req, limit)
if err != nil {
return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err)
@@ -71,7 +75,7 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
skipRest = true
continue
}
sr.Diffs = append(sr.Diffs, compareStep(want, live.Response)...)
sr.Diffs = append(sr.Diffs, compareStepWith(want, live.Response, maskOptions{uploadPrefix: cfg.UploadPrefix})...)
if len(sr.Diffs) > 0 {
sr.OK = false
result.OK = false
@@ -90,6 +94,10 @@ func mustMemoryStore() *Store {
}
func compareStep(want Step, got Response) []Diff {
return compareStepWith(want, got, maskOptions{})
}
func compareStepWith(want Step, got Response, opts maskOptions) []Diff {
var diffs []Diff
if want.Response.Status != 0 && got.Status != want.Response.Status {
diffs = append(diffs, Diff{
@@ -99,6 +107,6 @@ func compareStep(want Step, got Response) []Diff {
})
}
diffs = append(diffs, compareHeaders(want.Response.Headers, got.Headers, want.Headers)...)
diffs = append(diffs, compareBodies(want.Response, got, want)...)
diffs = append(diffs, compareBodiesWith(want.Response, got, want, opts)...)
return diffs
}

View File

@@ -207,6 +207,19 @@ func expandRequest(req Request, store *Store) (Request, error) {
return Request{}, err
}
out.Body = Body(body)
if len(req.Parts) > 0 {
// Variables expand in part values only, never in file bytes.
out.Parts = make([]Part, len(req.Parts))
for i, p := range req.Parts {
out.Parts[i] = p
if p.File == "" {
out.Parts[i].Value, err = store.Expand(p.Value)
if err != nil {
return Request{}, err
}
}
}
}
return out, nil
}
@@ -461,6 +474,14 @@ func ScrubStep(store *Store, step *Step) error {
step.Request.Query = replaceAll(step.Request.Query, pairs, true)
step.Request.Headers = scrubMap(step.Request.Headers, pairs, true)
step.Request.Body = Body(replaceAll(string(step.Request.Body), pairs, false))
if len(step.Request.Parts) > 0 {
parts := make([]Part, len(step.Request.Parts))
for i, p := range step.Request.Parts {
parts[i] = p
parts[i].Value = replaceAll(p.Value, pairs, false)
}
step.Request.Parts = parts
}
for _, rule := range step.Capture {
if strings.TrimSpace(rule.From) != "request.form" {
continue
@@ -606,6 +627,11 @@ func rejectUnclassifiedCredentials(step Step) error {
if err := check("request body", string(step.Request.Body)); err != nil {
return err
}
for _, p := range step.Request.Parts {
if err := check("request part "+p.Name, p.Value); err != nil {
return err
}
}
for k, v := range step.Response.Headers {
if err := check("response header "+k, v); err != nil {
return err