6e30624ece
- lagoon: Go-typed request values (typed slices and maps, sized integers, floats, file pointers, typed path lookups), regex delimiters, mimes sniffing (jpg/jpeg, SVG, PHP names, unreadable content), UploadedFileFromHeader, the rule builders, custom catalog lines with :input/:index/:position, and exists: against Postgres (text compare, inferred and NULL columns, arrays, unsafe identifiers, missing tables) - lagoon/attach: thumbnails in every mode from PNG, GIF and JPEG originals, bucket URL normalisation, OpenBucket/Publish refusals, URL helpers and static prefix stripping - tide: part validation and encoding edges, symlinks out of the fixture directory, Content-Type handling, every response mask and the coverage report helpers Coverage: lagoon 84.4%, lagoon/attach 87.7%, tide 81.4%, beachcomber 84.3%, beachcomber/typesense 95.9%.
417 lines
16 KiB
Go
417 lines
16 KiB
Go
package tide
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"io"
|
|
"mime"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// multipartFixture writes files/cover.png under a temp fixture directory and
|
|
// returns the directory and a one-step upload flow that references it.
|
|
func multipartFixture(t *testing.T) (string, Flow) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
content := []byte("\x89PNG\r\n\x1a\nfake image bytes")
|
|
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), content, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(content)
|
|
flow := Flow{
|
|
Version: CurrentVersion,
|
|
Name: "upload",
|
|
Steps: []Step{{
|
|
ID: "upload",
|
|
Request: Request{
|
|
Method: "POST",
|
|
Path: "/posts/1/photos",
|
|
Headers: map[string]string{"Content-Type": "multipart/form-data; boundary=recorded-elsewhere"},
|
|
Parts: []Part{
|
|
{Name: "title", Value: "Cover {{id:post}}"},
|
|
{Name: "file", File: "files/cover.png", ContentType: "image/png", SHA256: hex.EncodeToString(sum[:])},
|
|
},
|
|
},
|
|
}},
|
|
}
|
|
return dir, flow
|
|
}
|
|
|
|
type rawCapture struct {
|
|
mu sync.Mutex
|
|
bodies [][]byte
|
|
types []string
|
|
}
|
|
|
|
func (c *rawCapture) handler(t *testing.T) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
raw, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
c.mu.Lock()
|
|
c.bodies = append(c.bodies, raw)
|
|
c.types = append(c.types, r.Header.Get("Content-Type"))
|
|
c.mu.Unlock()
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"ok":true}`))
|
|
}
|
|
}
|
|
|
|
func TestMultipartRecordReplaySendsIdenticalBytes(t *testing.T) {
|
|
dir, flow := multipartFixture(t)
|
|
capture := &rawCapture{}
|
|
srv := httptest.NewServer(capture.handler(t))
|
|
t.Cleanup(srv.Close)
|
|
store := mustMemoryStore()
|
|
store.Set("id:post", "7")
|
|
|
|
recorded, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir})
|
|
if err != nil {
|
|
t.Fatalf("record: %v", err)
|
|
}
|
|
if _, err := RecordFlow(t.Context(), flow, RecordConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
|
t.Fatalf("second record: %v", err)
|
|
}
|
|
if _, err := ReplayFlow(t.Context(), recorded, ReplayConfig{Target: srv.URL, Store: store, BaseDir: dir}); err != nil {
|
|
t.Fatalf("replay: %v", err)
|
|
}
|
|
if len(capture.bodies) != 3 {
|
|
t.Fatalf("requests = %d", len(capture.bodies))
|
|
}
|
|
for i := 1; i < 3; i++ {
|
|
if !bytes.Equal(capture.bodies[0], capture.bodies[i]) {
|
|
t.Fatalf("body %d differs:\n%q\n%q", i, capture.bodies[0], capture.bodies[i])
|
|
}
|
|
if capture.types[i] != capture.types[0] {
|
|
t.Fatalf("content type %d = %q", i, capture.types[i])
|
|
}
|
|
}
|
|
media, params, err := mime.ParseMediaType(capture.types[0])
|
|
if err != nil || media != "multipart/form-data" || params["boundary"] != MultipartBoundary {
|
|
t.Fatalf("content type = %q", capture.types[0])
|
|
}
|
|
form, err := multipart.NewReader(bytes.NewReader(capture.bodies[0]), MultipartBoundary).ReadForm(1 << 20)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := form.Value["title"]; len(got) != 1 || got[0] != "Cover 7" {
|
|
t.Fatalf("title = %q (variables expand in values)", got)
|
|
}
|
|
fh := form.File["file"]
|
|
if len(fh) != 1 || fh[0].Filename != "cover.png" || fh[0].Header.Get("Content-Type") != "image/png" {
|
|
t.Fatalf("file part = %+v", fh)
|
|
}
|
|
|
|
// The recording keeps the parts, never the file bytes.
|
|
raw, err := marshalFlow(recorded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(raw), "fake image bytes") || !strings.Contains(string(raw), "file: files/cover.png") {
|
|
t.Fatalf("recorded flow:\n%s", raw)
|
|
}
|
|
parsed, err := ParseFlow(raw)
|
|
if err != nil {
|
|
t.Fatalf("parse recorded flow: %v", err)
|
|
}
|
|
if len(parsed.Steps[0].Request.Parts) != 2 || parsed.Steps[0].Request.Parts[1].SHA256 != flow.Steps[0].Request.Parts[1].SHA256 {
|
|
t.Fatalf("parts after round trip = %+v", parsed.Steps[0].Request.Parts)
|
|
}
|
|
}
|
|
|
|
func TestMultipartTamperedPartFileFailsLoad(t *testing.T) {
|
|
dir, flow := multipartFixture(t)
|
|
raw, err := marshalFlow(flow)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(dir, "upload.yaml")
|
|
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadFlow(path); err != nil {
|
|
t.Fatalf("intact fixture: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "files", "cover.png"), []byte("tampered"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = LoadFlow(path)
|
|
if err == nil || !strings.Contains(err.Error(), `part "file"`) || !strings.Contains(err.Error(), "sha256") {
|
|
t.Fatalf("tampered file: %v", err)
|
|
}
|
|
if err := os.Remove(filepath.Join(dir, "files", "cover.png")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadFlow(path); err == nil {
|
|
t.Fatal("missing part file must fail the load")
|
|
}
|
|
}
|
|
|
|
func TestMultipartRejectsInvalidParts(t *testing.T) {
|
|
_, flow := multipartFixture(t)
|
|
cases := map[string]func(*Request){
|
|
"body and parts": func(r *Request) { r.Body = "x=1" },
|
|
"escaping file": func(r *Request) { r.Parts[1].File = "../secret.png" },
|
|
"missing sha256": func(r *Request) { r.Parts[1].SHA256 = "" },
|
|
"value and file": func(r *Request) { r.Parts[1].Value = "x" },
|
|
"missing name": func(r *Request) { r.Parts[0].Name = "" },
|
|
}
|
|
for name, mutate := range cases {
|
|
f := flow
|
|
f.Steps = []Step{flow.Steps[0]}
|
|
f.Steps[0].Request.Parts = append([]Part(nil), flow.Steps[0].Request.Parts...)
|
|
mutate(&f.Steps[0].Request)
|
|
if err := validateFlow(f); err == nil {
|
|
t.Fatalf("%s: validateFlow accepted the flow", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMultipartKeepsNonMultipartContentType(t *testing.T) {
|
|
dir, flow := multipartFixture(t)
|
|
req := flow.Steps[0].Request
|
|
req.Headers = map[string]string{"Content-Type": "text/plain"}
|
|
out, err := prepareRequest(req, dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out.Headers["Content-Type"] != "text/plain" {
|
|
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
|
}
|
|
req.Headers = nil
|
|
out, err = prepareRequest(req, dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(out.Headers["Content-Type"], MultipartBoundary) {
|
|
t.Fatalf("content type = %q", out.Headers["Content-Type"])
|
|
}
|
|
}
|
|
|
|
func uploadBody(url, thumb string) []byte {
|
|
b, _ := json.Marshal(map[string]any{"data": map[string]any{"photos": []any{map[string]any{"id": 1, "url": url, "thumb_url": thumb}}}})
|
|
return b
|
|
}
|
|
|
|
func TestNormalizeUploadURLMasksRandomParts(t *testing.T) {
|
|
step := Step{ID: "u"}
|
|
php := uploadBody(
|
|
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
|
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
|
goBody := uploadBody(
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
|
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(goBody)}, step); len(diffs) != 0 {
|
|
t.Fatalf("diffs = %+v", diffs)
|
|
}
|
|
// An external URL under url is not an upload and is compared as is.
|
|
ext := uploadBody("https://img.example.com/a.jpg", "https://img.example.com/b.jpg")
|
|
if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(ext)}, Response{Headers: jsonCT(), Body: Body(ext)}, step); len(diffs) != 0 {
|
|
t.Fatalf("external diffs = %+v", diffs)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeUploadURLReportsWrongShape(t *testing.T) {
|
|
step := Step{ID: "u"}
|
|
php := uploadBody(
|
|
"/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png",
|
|
"/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png")
|
|
cases := map[string][]byte{
|
|
"thumb size": uploadBody(
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_100_100_0_0_crop.png"),
|
|
"prefix": uploadBody(
|
|
"/storage/uploads/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
|
"partition": uploadBody(
|
|
"/storage/app/uploads/public/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
|
"partition not from disk name": uploadBody(
|
|
"/storage/app/uploads/public/aaa/bbb/ccc/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
|
"extension": uploadBody(
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.jpg",
|
|
"/storage/app/uploads/public/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png"),
|
|
}
|
|
for name, got := range cases {
|
|
diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(php)}, Response{Headers: jsonCT(), Body: Body(got)}, step)
|
|
if len(diffs) == 0 {
|
|
t.Fatalf("%s: masking hid the mismatch", name)
|
|
}
|
|
}
|
|
// A configured prefix replaces the WinterCMS default.
|
|
alt := uploadBody("/files/9f8/e7d/6c5/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png", "/files/9f8/e7d/6c5/thumb_3_200_200_0_0_crop.png")
|
|
alt2 := uploadBody("/files/123/456/789/1234567890abcdef.png", "/files/123/456/789/thumb_9_200_200_0_0_crop.png")
|
|
if diffs := compareBodiesWith(Response{Headers: jsonCT(), Body: Body(alt)}, Response{Headers: jsonCT(), Body: Body(alt2)}, step, maskOptions{uploadPrefix: "/files"}); len(diffs) != 0 {
|
|
t.Fatalf("custom prefix diffs = %+v", diffs)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePublicationAlbumDates(t *testing.T) {
|
|
store := mustMemoryStore()
|
|
pub := func(created, updated string) []Publication {
|
|
return []Publication{{Method: "POST", Path: "/api/publish", Body: json.RawMessage(
|
|
`{"channel":"c","data":{"payload":{"album":{"name":"x","created_at":"` + created + `","updated_at":"` + updated +
|
|
`","market_price_checked_at":null,"photos":[{"created_at":"` + created + `"}]},"created_at":"2026-01-01T00:00:00+00:00"}}}`)}}
|
|
}
|
|
a, err := NormalizePublications(pub("2026-09-30T11:21:55+00:00", "2026-09-30T11:21:56+00:00"), store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := NormalizePublications(pub("2026-10-02T08:00:00+00:00", "2026-10-02T08:00:01+00:00"), store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if diffs := DiffPublications(a, b); len(diffs) != 0 {
|
|
t.Fatalf("diffs = %+v", diffs)
|
|
}
|
|
if !strings.Contains(string(a[0].Body), `"updated_at":"{{datetime}}"`) || !strings.Contains(string(a[0].Body), `"market_price_checked_at":null`) {
|
|
t.Fatalf("album dates not masked: %s", a[0].Body)
|
|
}
|
|
// Only the album subtree is masked; a payload date outside it stays.
|
|
if !strings.Contains(string(a[0].Body), `},"created_at":"2026-01-01T00:00:00+00:00"`) {
|
|
t.Fatalf("over-normalised: %s", a[0].Body)
|
|
}
|
|
// A Z-suffixed album date keeps its value, so a format change is a diff.
|
|
z, err := NormalizePublications(pub("2026-10-02T08:00:00Z", "2026-10-02T08:00:01+00:00"), store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if diffs := DiffPublications(a, z); len(diffs) == 0 {
|
|
t.Fatal("a Z album date must show as a diff")
|
|
}
|
|
}
|
|
|
|
// TestMultipartPartEdges covers part validation, file reading and encoding
|
|
// edges: file attributes without a file, empty and value-only parts,
|
|
// default file names and content types, unicode and quoted names, a part
|
|
// that contains the boundary, a missing or tampered file, a file outside the
|
|
// fixture directory and the Content-Type handling of prepareRequest.
|
|
func TestMultipartPartEdges(t *testing.T) {
|
|
dir := t.TempDir()
|
|
png := []byte("\x89PNG\r\n\x1a\nfixture")
|
|
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "files", "okładka.png"), png, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sum := sha256.Sum256(png)
|
|
pin := hex.EncodeToString(sum[:])
|
|
|
|
for name, req := range map[string]Request{
|
|
"file attributes without a file": {Parts: []Part{{Name: "a", Filename: "x.png"}}},
|
|
"content type without a file": {Parts: []Part{{Name: "a", ContentType: "image/png"}}},
|
|
"sha without a file": {Parts: []Part{{Name: "a", SHA256: pin}}},
|
|
"blank name": {Parts: []Part{{Name: " ", Value: "x"}}},
|
|
"absolute file": {Parts: []Part{{Name: "a", File: "/etc/passwd", SHA256: pin}}},
|
|
"short sha": {Parts: []Part{{Name: "a", File: "files/okładka.png", SHA256: "abc"}}},
|
|
} {
|
|
if err := validateParts(req); err == nil {
|
|
t.Errorf("%s: validateParts accepted it", name)
|
|
}
|
|
}
|
|
if err := validateParts(Request{}); err != nil {
|
|
t.Fatalf("no parts: %v", err)
|
|
}
|
|
|
|
parts := []Part{
|
|
{Name: "empty", Value: ""},
|
|
{Name: `tit"le`, Value: "Łódź"},
|
|
{Name: "file", File: "files/okładka.png", SHA256: strings.ToUpper(pin)},
|
|
}
|
|
body, ct, err := encodeParts(dir, parts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ct != "multipart/form-data; boundary="+MultipartBoundary {
|
|
t.Fatalf("content type %q", ct)
|
|
}
|
|
text := string(body)
|
|
for _, want := range []string{
|
|
`Content-Disposition: form-data; name="empty"`,
|
|
`Content-Disposition: form-data; name="tit\"le"` + "\r\n\r\nŁódź",
|
|
`Content-Disposition: form-data; name="file"; filename="okładka.png"`,
|
|
"Content-Type: application/octet-stream",
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Errorf("encoded body lacks %q:\n%s", want, text)
|
|
}
|
|
}
|
|
again, _, err := encodeParts(dir, parts)
|
|
if err != nil || !bytes.Equal(body, again) {
|
|
t.Fatal("encoding is not deterministic")
|
|
}
|
|
if _, _, err := encodeParts(dir, []Part{{Name: "x", Value: "--" + MultipartBoundary}}); err == nil {
|
|
t.Error("a value holding the boundary was encoded")
|
|
}
|
|
for name, p := range map[string]Part{
|
|
"missing": {Name: "f", File: "files/nope.png", SHA256: pin},
|
|
"tampered": {Name: "f", File: "files/okładka.png", SHA256: strings.Repeat("0", 64)},
|
|
"escaping": {Name: "f", File: "../okładka.png", SHA256: pin},
|
|
} {
|
|
if _, err := readPartFile(dir, p); err == nil {
|
|
t.Errorf("%s part file was read", name)
|
|
}
|
|
}
|
|
outside := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(outside, "x.png"), png, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(filepath.Join(outside, "x.png"), filepath.Join(dir, "files", "link.png")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := readPartFile(dir, Part{Name: "f", File: "files/link.png", SHA256: pin}); err == nil {
|
|
t.Error("a symlink leaving the fixture directory was read")
|
|
}
|
|
if raw, err := readPartFile("", Part{Name: "f", File: filepath.Join(dir, "files", "okładka.png")[len(dir)+1:], SHA256: pin}); err == nil && raw != nil {
|
|
t.Error("a part file resolved without its fixture directory")
|
|
}
|
|
|
|
for name, tc := range map[string]struct {
|
|
headers map[string]string
|
|
want string
|
|
}{
|
|
"none recorded": {nil, "multipart/form-data; boundary=" + MultipartBoundary},
|
|
"stale boundary": {map[string]string{"content-type": "multipart/form-data; boundary=old"}, "multipart/form-data; boundary=" + MultipartBoundary},
|
|
"unparsable": {map[string]string{"Content-Type": ";;;"}, "multipart/form-data; boundary=" + MultipartBoundary},
|
|
"kept non-multipart": {map[string]string{"Content-Type": "text/plain"}, "text/plain"},
|
|
} {
|
|
out, err := prepareRequest(Request{Method: "POST", Headers: tc.headers, Parts: parts[:1]}, dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := ""
|
|
for k, v := range out.Headers {
|
|
if strings.EqualFold(k, "Content-Type") {
|
|
got = v
|
|
}
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("%s: Content-Type %q, want %q", name, got, tc.want)
|
|
}
|
|
}
|
|
if out, err := prepareRequest(Request{Body: "a=1"}, dir); err != nil || out.Body != "a=1" {
|
|
t.Fatalf("no parts: %+v %v", out, err)
|
|
}
|
|
if _, err := prepareRequest(Request{Body: "a=1", Parts: parts[:1]}, dir); err == nil {
|
|
t.Fatal("body and parts were prepared")
|
|
}
|
|
if _, err := prepareRequest(Request{Parts: []Part{{Name: "f", File: "files/nope.png", SHA256: pin}}}, dir); err == nil {
|
|
t.Fatal("a missing part file was prepared")
|
|
}
|
|
}
|