feat(12-01): record multipart uploads and match Winter upload URLs
- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the image size from the header before decoding - tide requests carry multipart parts (files beside the fixture pinned by sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive byte-identical bodies - tide masks the random partition, disk name and file id of url/thumb_url upload URLs while still diffing prefix, size, mode and extension, and NormalizePublications masks Carbon dates in the published album
This commit is contained in:
Jakub Zych
201
modules/tide/multipart.go
Normal file
201
modules/tide/multipart.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MultipartBoundary is the fixed boundary every multipart request body is
|
||||
// encoded with, so a recording against one backend and a replay against
|
||||
// another send byte-identical bodies.
|
||||
const MultipartBoundary = "SummerTideMultipartBoundary7MA4YWxkTrZu0gW"
|
||||
|
||||
// Part is one field of a multipart/form-data request. A text field sets
|
||||
// Value; a file field sets File, a path relative to the fixture directory
|
||||
// (for example files/cover.png), whose bytes are never inlined into the
|
||||
// YAML, with SHA256 pinning its content. Filename defaults to the base name
|
||||
// of File and ContentType to application/octet-stream.
|
||||
type Part struct {
|
||||
Name string `yaml:"name"`
|
||||
Value string `yaml:"value,omitempty"`
|
||||
File string `yaml:"file,omitempty"`
|
||||
Filename string `yaml:"filename,omitempty"`
|
||||
ContentType string `yaml:"content_type,omitempty"`
|
||||
SHA256 string `yaml:"sha256,omitempty"`
|
||||
}
|
||||
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-fA-F]{64}$`)
|
||||
|
||||
// validateParts checks the shape of a request's parts without reading files.
|
||||
func validateParts(req Request) error {
|
||||
if len(req.Parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
if req.Body != "" {
|
||||
return fmt.Errorf("request has both body and parts")
|
||||
}
|
||||
for i, p := range req.Parts {
|
||||
if strings.TrimSpace(p.Name) == "" {
|
||||
return fmt.Errorf("parts[%d] is missing name", i)
|
||||
}
|
||||
if p.File == "" {
|
||||
if p.Filename != "" || p.ContentType != "" || p.SHA256 != "" {
|
||||
return fmt.Errorf("part %q has file attributes but no file", p.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if p.Value != "" {
|
||||
return fmt.Errorf("part %q has both value and file", p.Name)
|
||||
}
|
||||
if err := validateSidecar(p.File); err != nil {
|
||||
return fmt.Errorf("part %q: %w", p.Name, err)
|
||||
}
|
||||
if !sha256Hex.MatchString(p.SHA256) {
|
||||
return fmt.Errorf("part %q needs the sha256 of its file", p.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyPartFiles checks that every part file of flow exists under base and
|
||||
// matches its sha256.
|
||||
func verifyPartFiles(base string, flow Flow) error {
|
||||
for _, step := range flow.Steps {
|
||||
for _, p := range step.Request.Parts {
|
||||
if p.File == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := readPartFile(base, p); err != nil {
|
||||
return fmt.Errorf("tide: step %s: %w", step.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readPartFile(base string, p Part) ([]byte, error) {
|
||||
if err := validateSidecar(p.File); err != nil {
|
||||
return nil, fmt.Errorf("part %q: %w", p.Name, err)
|
||||
}
|
||||
full := p.File
|
||||
if base != "" {
|
||||
full = filepath.Join(base, p.File)
|
||||
}
|
||||
resolved, err := resolvePath(full)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
|
||||
}
|
||||
if base != "" {
|
||||
root, err := resolvePath(base)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fixture dir: %w", err)
|
||||
}
|
||||
if resolved != root && !strings.HasPrefix(resolved, root+string(os.PathSeparator)) {
|
||||
return nil, fmt.Errorf("part %q file %q escapes the fixture directory", p.Name, p.File)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("part %q file %s: %w", p.Name, p.File, err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if !strings.EqualFold(hex.EncodeToString(sum[:]), p.SHA256) {
|
||||
return nil, fmt.Errorf("part %q file %s sha256 mismatch", p.Name, p.File)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||
|
||||
// encodeParts writes parts in declaration order as a multipart/form-data
|
||||
// body with MultipartBoundary and returns the body and its Content-Type.
|
||||
func encodeParts(base string, parts []Part) ([]byte, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
if err := w.SetBoundary(MultipartBoundary); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
marker := []byte(MultipartBoundary)
|
||||
for _, p := range parts {
|
||||
h := make(textproto.MIMEHeader)
|
||||
var content []byte
|
||||
if p.File == "" {
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, quoteEscaper.Replace(p.Name)))
|
||||
content = []byte(p.Value)
|
||||
} else {
|
||||
raw, err := readPartFile(base, p)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
filename := p.Filename
|
||||
if filename == "" {
|
||||
filename = path.Base(filepath.ToSlash(p.File))
|
||||
}
|
||||
ct := p.ContentType
|
||||
if ct == "" {
|
||||
ct = "application/octet-stream"
|
||||
}
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, quoteEscaper.Replace(p.Name), quoteEscaper.Replace(filename)))
|
||||
h.Set("Content-Type", ct)
|
||||
content = raw
|
||||
}
|
||||
if bytes.Contains(content, marker) {
|
||||
return nil, "", fmt.Errorf("part %q contains the multipart boundary", p.Name)
|
||||
}
|
||||
pw, err := w.CreatePart(h)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if _, err := pw.Write(content); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf.Bytes(), w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// prepareRequest encodes a request's parts into its body. The multipart
|
||||
// Content-Type replaces a recorded multipart Content-Type (whose boundary
|
||||
// would be stale) and is added when none is recorded; a recorded non-
|
||||
// multipart Content-Type is kept as it is.
|
||||
func prepareRequest(req Request, base string) (Request, error) {
|
||||
if len(req.Parts) == 0 {
|
||||
return req, nil
|
||||
}
|
||||
if err := validateParts(req); err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
body, ct, err := encodeParts(base, req.Parts)
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
out := req
|
||||
out.Body = Body(body)
|
||||
out.Headers = make(map[string]string, len(req.Headers)+1)
|
||||
set := false
|
||||
for k, v := range req.Headers {
|
||||
if strings.EqualFold(k, "Content-Type") {
|
||||
set = true
|
||||
if media, _, err := mime.ParseMediaType(v); err != nil || strings.HasPrefix(media, "multipart/") {
|
||||
v = ct
|
||||
}
|
||||
}
|
||||
out.Headers[k] = v
|
||||
}
|
||||
if !set {
|
||||
out.Headers["Content-Type"] = ct
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user