feat(12-01): record multipart uploads and match Winter upload URLs
- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the image size from the header before decoding - tide requests carry multipart parts (files beside the fixture pinned by sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive byte-identical bodies - tide masks the random partition, disk name and file id of url/thumb_url upload URLs while still diffing prefix, size, mode and extension, and NormalizePublications masks Carbon dates in the published album
This commit is contained in:
Jakub Zych
296
modules/tide/multipart_test.go
Normal file
296
modules/tide/multipart_test.go
Normal file
@@ -0,0 +1,296 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// multipartFixture writes files/cover.png under a temp fixture directory and
|
||||
// returns the directory and a one-step upload flow that references it.
|
||||
func multipartFixture(t *testing.T) (string, Flow) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
content := []byte("\x89PNG\r\n\x1a\nfake image bytes")
|
||||
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(content)
|
||||
flow := Flow{
|
||||
Version: CurrentVersion,
|
||||
Name: "upload",
|
||||
Steps: []Step{{
|
||||
ID: "upload",
|
||||
Request: Request{
|
||||
Method: "POST",
|
||||
Path: "/posts/1/photos",
|
||||
Headers: map[string]string{"Content-Type": "multipart/form-data; boundary=recorded-elsewhere"},
|
||||
Parts: []Part{
|
||||
{Name: "title", Value: "Cover {{id:post}}"},
|
||||
{Name: "file", File: "files/cover.png", ContentType: "image/png", SHA256: hex.EncodeToString(sum[:])},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
return dir, flow
|
||||
}
|
||||
|
||||
type rawCapture struct {
|
||||
mu sync.Mutex
|
||||
bodies [][]byte
|
||||
types []string
|
||||
}
|
||||
|
||||
func (c *rawCapture) handler(t *testing.T) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.bodies = append(c.bodies, raw)
|
||||
c.types = append(c.types, r.Header.Get("Content-Type"))
|
||||
c.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartRecordReplaySendsIdenticalBytes(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
capture := &rawCapture{}
|
||||
srv := httptest.NewServer(capture.handler(t))
|
||||
t.Cleanup(srv.Close)
|
||||
store := mustMemoryStore()
|
||||
store.Set("id:post", "7")
|
||||
|
||||
recorded, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("record: %v", err)
|
||||
}
|
||||
if _, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
||||
t.Fatalf("second record: %v", err)
|
||||
}
|
||||
if _, err := ReplayFlow(t.Context(), recorded, ReplayConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
||||
t.Fatalf("replay: %v", err)
|
||||
}
|
||||
if len(capture.bodies) != 3 {
|
||||
t.Fatalf("requests = %d", len(capture.bodies))
|
||||
}
|
||||
for i := 1; i < 3; i++ {
|
||||
if !bytes.Equal(capture.bodies[0], capture.bodies[i]) {
|
||||
t.Fatalf("body %d differs:\n%q\n%q", i, capture.bodies[0], capture.bodies[i])
|
||||
}
|
||||
if capture.types[i] != capture.types[0] {
|
||||
t.Fatalf("content type %d = %q", i, capture.types[i])
|
||||
}
|
||||
}
|
||||
media, params, err := mime.ParseMediaType(capture.types[0])
|
||||
if err != nil || media != "multipart/form-data" || params["boundary"] != MultipartBoundary {
|
||||
t.Fatalf("content type = %q", capture.types[0])
|
||||
}
|
||||
form, err := multipart.NewReader(bytes.NewReader(capture.bodies[0]), MultipartBoundary).ReadForm(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := form.Value["title"]; len(got) != 1 || got[0] != "Cover 7" {
|
||||
t.Fatalf("title = %q (variables expand in values)", got)
|
||||
}
|
||||
fh := form.File["file"]
|
||||
if len(fh) != 1 || fh[0].Filename != "cover.png" || fh[0].Header.Get("Content-Type") != "image/png" {
|
||||
t.Fatalf("file part = %+v", fh)
|
||||
}
|
||||
|
||||
// The recording keeps the parts, never the file bytes.
|
||||
raw, err := marshalFlow(recorded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), "fake image bytes") || !strings.Contains(string(raw), "file: files/cover.png") {
|
||||
t.Fatalf("recorded flow:\n%s", raw)
|
||||
}
|
||||
parsed, err := ParseFlow(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse recorded flow: %v", err)
|
||||
}
|
||||
if len(parsed.Steps[0].Request.Parts) != 2 || parsed.Steps[0].Request.Parts[1].SHA256 != flow.Steps[0].Request.Parts[1].SHA256 {
|
||||
t.Fatalf("parts after round trip = %+v", parsed.Steps[0].Request.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartTamperedPartFileFailsLoad(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
raw, err := marshalFlow(flow)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(dir, "upload.yaml")
|
||||
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(path); err != nil {
|
||||
t.Fatalf("intact fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), []byte("tampered"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = LoadFlow(path)
|
||||
if err == nil || !strings.Contains(err.Error(), `part "file"`) || !strings.Contains(err.Error(), "sha256") {
|
||||
t.Fatalf("tampered file: %v", err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(dir, "files", "cover.png")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(path); err == nil {
|
||||
t.Fatal("missing part file must fail the load")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartRejectsInvalidParts(t *testing.T) {
|
||||
_, flow := multipartFixture(t)
|
||||
cases := map[string]func(*Request){
|
||||
"body and parts": func(r *Request) { r.Body = "x=1" },
|
||||
"escaping file": func(r *Request) { r.Parts[1].File = "../secret.png" },
|
||||
"missing sha256": func(r *Request) { r.Parts[1].SHA256 = "" },
|
||||
"value and file": func(r *Request) { r.Parts[1].Value = "x" },
|
||||
"missing name": func(r *Request) { r.Parts[0].Name = "" },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
f := flow
|
||||
f.Steps = []Step{flow.Steps[0]}
|
||||
f.Steps[0].Request.Parts = append([]Part(nil), flow.Steps[0].Request.Parts...)
|
||||
mutate(&f.Steps[0].Request)
|
||||
if err := validateFlow(f); err == nil {
|
||||
t.Fatalf("%s: validateFlow accepted the flow", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipartKeepsNonMultipartContentType(t *testing.T) {
|
||||
dir, flow := multipartFixture(t)
|
||||
req := flow.Steps[0].Request
|
||||
req.Headers = map[string]string{"Content-Type": "text/plain"}
|
||||
out, err := prepareRequest(req, dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Headers["Content-Type"] != "text/plain" {
|
||||
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
||||
}
|
||||
req.Headers = nil
|
||||
out, err = prepareRequest(req, dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.Headers["Content-Type"], MultipartBoundary) {
|
||||
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
||||
}
|
||||
}
|
||||
|
||||
func uploadBody(url, thumb string) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"data": map[string]any{"photos": []any{map[string]any{"id": 1, "url": url, "thumb_url": thumb}}}})
|
||||
return b
|
||||
}
|
||||
|
||||
func TestNormalizeUploadURLMasksRandomParts(t *testing.T) {
|
||||
step := Step{ID: "u"}
|
||||
php := uploadBody(
|
||||
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
||||
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
||||
goBody := uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
||||
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(goBody)}, step); len(diffs) != 0 {
|
||||
t.Fatalf("diffs = %+v", diffs)
|
||||
}
|
||||
// An external URL under url is not an upload and is compared as is.
|
||||
ext := uploadBody("https://img.example.com/a.jpg", "https://img.example.com/b.jpg")
|
||||
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(ext)}, Response{Headers: jsonCT(), Body: Body(ext)}, step); len(diffs) != 0 {
|
||||
t.Fatalf("external diffs = %+v", diffs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeUploadURLReportsWrongShape(t *testing.T) {
|
||||
step := Step{ID: "u"}
|
||||
php := uploadBody(
|
||||
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
||||
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
||||
cases := map[string][]byte{
|
||||
"thumb size": uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_100_100_0_0_crop.png"),
|
||||
"prefix": uploadBody(
|
||||
"/storage/uploads/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"partition": uploadBody(
|
||||
"/storage/app/uploads/public/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"partition not from disk name": uploadBody(
|
||||
"/storage/app/uploads/public/aaa/bbb/ccc/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
"extension": uploadBody(
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.jpg",
|
||||
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
||||
}
|
||||
for name, got := range cases {
|
||||
diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(got)}, step)
|
||||
if len(diffs) == 0 {
|
||||
t.Fatalf("%s: masking hid the mismatch", name)
|
||||
}
|
||||
}
|
||||
// A configured prefix replaces the WinterCMS default.
|
||||
alt := uploadBody("/files/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", "/files/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
||||
alt2 := uploadBody("/files/123/456/789/1234567890abcdef.png", "/files/123/456/789/thumb_9_200_200_0_0_crop.png")
|
||||
if diffs := compareBodiesWith(Response{Headers: jsonCT(), Body: Body(alt)}, Response{Headers: jsonCT(), Body: Body(alt2)}, step, maskOptions{uploadPrefix: "/files"}); len(diffs) != 0 {
|
||||
t.Fatalf("custom prefix diffs = %+v", diffs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePublicationAlbumDates(t *testing.T) {
|
||||
store := mustMemoryStore()
|
||||
pub := func(created, updated string) []Publication {
|
||||
return []Publication{{Method: "POST", Path: "/api/publish", Body: json.RawMessage(
|
||||
`{"channel":"c","data":{"payload":{"album":{"name":"x","created_at":"` + created + `","updated_at":"` + updated +
|
||||
`","market_price_checked_at":null,"photos":[{"created_at":"` + created + `"}]},"created_at":"2026-01-01T00:00:00+00:00"}}}`)}}
|
||||
}
|
||||
a, err := NormalizePublications(pub("2026-09-30T11:21:55+00:00", "2026-09-30T11:21:56+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := NormalizePublications(pub("2026-10-02T08:00:00+00:00", "2026-10-02T08:00:01+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diffs := DiffPublications(a, b); len(diffs) != 0 {
|
||||
t.Fatalf("diffs = %+v", diffs)
|
||||
}
|
||||
if !strings.Contains(string(a[0].Body), `"updated_at":"{{datetime}}"`) || !strings.Contains(string(a[0].Body), `"market_price_checked_at":null`) {
|
||||
t.Fatalf("album dates not masked: %s", a[0].Body)
|
||||
}
|
||||
// Only the album subtree is masked; a payload date outside it stays.
|
||||
if !strings.Contains(string(a[0].Body), `},"created_at":"2026-01-01T00:00:00+00:00"`) {
|
||||
t.Fatalf("over-normalised: %s", a[0].Body)
|
||||
}
|
||||
// A Z-suffixed album date keeps its value, so a format change is a diff.
|
||||
z, err := NormalizePublications(pub("2026-10-02T08:00:00Z", "2026-10-02T08:00:01+00:00"), store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diffs := DiffPublications(a, z); len(diffs) == 0 {
|
||||
t.Fatal("a Z album date must show as a diff")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user