From 59b5276cba41df9e8935a30be9aa4b91834e36df Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Thu, 17 Sep 2026 14:03:55 +0200 Subject: [PATCH] test(02-05): lock down every response parity class with negative tests - Record/replay baselines plus mutated fixtures for nil vs [], dates, tri-state bools, envelopes, money and ids - Header, CSV/image byte, sidecar digest and two-flow continuation cases fail with path diagnostics - Manifest coverage still reports later flows after a comparison failure Co-authored-by: Cursor --- tide/diff_contract_test.go | 189 +++++++++++++++++ tide/flow_contract_test.go | 364 +++++++++++++++++++++++++++++++++ tide/manifest_contract_test.go | 116 +++++++++++ 3 files changed, 669 insertions(+) create mode 100644 tide/diff_contract_test.go create mode 100644 tide/flow_contract_test.go create mode 100644 tide/manifest_contract_test.go diff --git a/tide/diff_contract_test.go b/tide/diff_contract_test.go new file mode 100644 index 0000000..557df4d --- /dev/null +++ b/tide/diff_contract_test.go @@ -0,0 +1,189 @@ +package tide + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" +) + +func TestDiffContract(t *testing.T) { + ctx := context.Background() + baseline := `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}` + reordered := `{"meta":{"total":1,"per_page":15,"last_page":1,"current_page":1},"data":[{"deleted_at":null,"created_at":"2026-01-01T00:00:00+00:00","is_owner":true,"tracklist":[],"price":"1.5000","slug":"keep-me","id":9}]}` + + spec := Flow{Version: 1, Name: "diff-contract", Steps: []Step{{ + ID: "get", + Request: Request{Method: http.MethodGet, Path: "/item"}, + }}} + orig := jsonServer(t, baseline) + recorded, err := RecordFlow(ctx, spec, RecordConfig{Target: orig.URL}) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(ctx, recorded, ReplayConfig{Target: orig.URL}); err != nil { + t.Fatalf("baseline must pass: %v", err) + } + if _, err := ReplayFlow(ctx, recorded, ReplayConfig{Target: jsonServer(t, reordered).URL}); err != nil { + t.Fatalf("object key reorder must pass: %v", err) + } + + type mutation struct { + name string + live string + path string + } + cases := []mutation{ + {"null vs array", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":null,"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "tracklist"}, + {"empty object vs array", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":{},"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "tracklist"}, + {"carbon Z vs +00:00", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00Z","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "created_at"}, + {"non-date text", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"yesterday","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "created_at"}, + {"date present-null vs absent", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00"}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "deleted_at"}, + {"tri-state true vs false", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":false,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "is_owner"}, + {"tri-state true vs null", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":null,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "is_owner"}, + {"missing meta envelope", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}]}`, "meta"}, + {"extra links envelope key", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1},"links":{}}`, "links"}, + {"conditional reservation key", `{"data":[{"id":1,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null,"reservation":{"id":2}}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "reservation"}, + {"money string vs number", `{"data":[{"id":1,"slug":"keep-me","price":1.5,"tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "price"}, + {"integer id vs string", `{"data":[{"id":"1","slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "id"}, + {"integer id vs fraction", `{"data":[{"id":1.5,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "id"}, + {"exact slug mismatch", `{"data":[{"id":1,"slug":"other","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-01-01T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`, "slug"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mut := cloneRecorded(t, recorded) + mut.Steps[0].Response.Body = Body(tc.live) + _, err := ReplayFlow(ctx, mut, ReplayConfig{Target: orig.URL}) + if err == nil { + t.Fatal("mutated fixture must fail") + } + assertPathMismatch(t, err, tc.path) + }) + } + + t.Run("csv exact-byte mismatch", func(t *testing.T) { + want := csvServer(t, "a,b\n1,2\n", `attachment; filename="albums.csv"`) + got := csvServer(t, "a,b\n1,3\n", `attachment; filename="albums.csv"`) + spec := Flow{Version: 1, Name: "csv", Steps: []Step{{ + ID: "export", + Request: Request{Method: http.MethodGet, Path: "/export"}, + }}} + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: want.URL}) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(ctx, rec, ReplayConfig{Target: want.URL}); err != nil { + t.Fatalf("csv baseline must pass: %v", err) + } + _, err = ReplayFlow(ctx, rec, ReplayConfig{Target: got.URL}) + if err == nil { + t.Fatal("csv byte mismatch must fail") + } + msg := err.Error() + if !strings.Contains(msg, "byte") && !strings.Contains(msg, "body[") { + t.Fatalf("csv mismatch missing byte diagnostic: %s", msg) + } + }) + + t.Run("image exact-byte mismatch", func(t *testing.T) { + want := binaryServer(t, "image/png", []byte{0x89, 'P', 'N', 'G', 1}) + got := binaryServer(t, "image/png", []byte{0x89, 'P', 'N', 'G', 2}) + spec := Flow{Version: 1, Name: "png", Steps: []Step{{ + ID: "cover", + Request: Request{Method: http.MethodGet, Path: "/cover"}, + }}} + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: want.URL}) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(ctx, rec, ReplayConfig{Target: want.URL}); err != nil { + t.Fatalf("image baseline must pass: %v", err) + } + _, err = ReplayFlow(ctx, rec, ReplayConfig{Target: got.URL}) + if err == nil { + t.Fatal("image byte mismatch must fail") + } + if !strings.Contains(err.Error(), "byte") && !strings.Contains(err.Error(), "body[") { + t.Fatalf("image mismatch missing byte diagnostic: %v", err) + } + }) + + t.Run("per-step normalization disable", func(t *testing.T) { + later := jsonServer(t, `{"data":[{"id":9,"slug":"keep-me","price":"1.5000","tracklist":[],"is_owner":true,"created_at":"2026-02-02T00:00:00+00:00","deleted_at":null}],"meta":{"current_page":1,"last_page":1,"per_page":15,"total":1}}`) + if _, err := ReplayFlow(ctx, recorded, ReplayConfig{Target: later.URL}); err != nil { + t.Fatalf("masked dates/ids must pass: %v", err) + } + disabled := cloneRecorded(t, recorded) + disabled.Steps[0].Normalize = []NormalizeRule{{Path: "created_at", Disable: true}} + _, err := ReplayFlow(ctx, disabled, ReplayConfig{Target: later.URL}) + if err == nil { + t.Fatal("disabled date mask must fail") + } + assertPathMismatch(t, err, "created_at") + }) +} + +func cloneRecorded(t *testing.T, flow Flow) Flow { + t.Helper() + path := filepath.Join(t.TempDir(), "clone.yaml") + if err := SaveFlow(path, flow); err != nil { + t.Fatal(err) + } + cloned, err := LoadFlow(path) + if err != nil { + t.Fatal(err) + } + return cloned +} + +func assertPathMismatch(t *testing.T, err error, path string) { + t.Helper() + msg := err.Error() + if !strings.Contains(msg, path) { + t.Fatalf("want path %s in %s", path, msg) + } + if !strings.Contains(msg, "expected") || !strings.Contains(msg, "actual") { + t.Fatalf("want expected/actual diagnostic in %s", msg) + } + var mis *MismatchError + if !errors.As(err, &mis) { + return + } + found := false + for _, step := range mis.Result.Steps { + for _, d := range step.Diffs { + if strings.Contains(d.Path, path) && d.Expected != d.Actual { + found = true + } + } + } + if !found { + t.Fatalf("result diffs missing path %s: %+v", path, mis.Result.Steps) + } +} + +func csvServer(t *testing.T, body, disposition string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", disposition) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func binaryServer(t *testing.T, ct string, body []byte) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", ct) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} diff --git a/tide/flow_contract_test.go b/tide/flow_contract_test.go new file mode 100644 index 0000000..53e5505 --- /dev/null +++ b/tide/flow_contract_test.go @@ -0,0 +1,364 @@ +package tide + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func TestFlowContract(t *testing.T) { + ctx := context.Background() + + t.Run("record replay baseline", func(t *testing.T) { + srv := jsonServer(t, `{"ok":true}`) + spec := Flow{Version: 1, Name: "baseline", Steps: []Step{{ + ID: "a", + Request: Request{Method: http.MethodGet, Path: "/ok"}, + }}} + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: srv.URL}) + if err != nil { + t.Fatal(err) + } + if rec.Steps[0].Response.Status != http.StatusOK { + t.Fatalf("status %d", rec.Steps[0].Response.Status) + } + if _, err := ReplayFlow(ctx, rec, ReplayConfig{Target: srv.URL}); err != nil { + t.Fatalf("baseline replay: %v", err) + } + }) + + t.Run("oauth and 401 headers plus ignored date server request id", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("WWW-Authenticate", `Bearer error="invalid_token", resource_metadata="http://127.0.0.1/.well-known/oauth-protected-resource"`) + w.Header().Set("Server", "php") + w.Header().Set("X-Request-Id", "live-req") + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"invalid_token"}`)) + })) + t.Cleanup(srv.Close) + spec := Flow{Version: 1, Name: "oauth-401", Steps: []Step{{ + ID: "token", + Request: Request{Method: http.MethodGet, Path: "/token"}, + }}} + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: srv.URL}) + if err != nil { + t.Fatal(err) + } + rec.Steps[0].Response.Headers["Date"] = "Wed, 01 Jan 2020 00:00:00 GMT" + rec.Steps[0].Response.Headers["Server"] = "old" + rec.Steps[0].Response.Headers["X-Request-Id"] = "fixture-req" + if _, err := ReplayFlow(ctx, rec, ReplayConfig{Target: srv.URL}); err != nil { + t.Fatalf("Date/Server/request id must be ignored: %v", err) + } + + badCache := cloneRecorded(t, rec) + badCache.Steps[0].Response.Headers["Cache-Control"] = "public" + _, err = ReplayFlow(ctx, badCache, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "cache-control") { + t.Fatalf("Cache-Control mismatch: %v", err) + } + + badPragma := cloneRecorded(t, rec) + badPragma.Steps[0].Response.Headers["Pragma"] = "public" + _, err = ReplayFlow(ctx, badPragma, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "pragma") { + t.Fatalf("Pragma mismatch: %v", err) + } + + badWWW := cloneRecorded(t, rec) + badWWW.Steps[0].Response.Headers["WWW-Authenticate"] = `Bearer error="other"` + _, err = ReplayFlow(ctx, badWWW, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "www-authenticate") { + t.Fatalf("WWW-Authenticate mismatch: %v", err) + } + + badStatus := cloneRecorded(t, rec) + badStatus.Steps[0].Response.Status = http.StatusOK + _, err = ReplayFlow(ctx, badStatus, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(err.Error(), "status") { + t.Fatalf("status mismatch: %v", err) + } + }) + + t.Run("csv content-disposition mismatch", func(t *testing.T) { + srv := csvServer(t, "a,b\n1,2\n", `attachment; filename="albums.csv"`) + spec := Flow{Version: 1, Name: "csv-disp", Steps: []Step{{ + ID: "export", + Request: Request{Method: http.MethodGet, Path: "/export"}, + }}} + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: srv.URL}) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(ctx, rec, ReplayConfig{Target: srv.URL}); err != nil { + t.Fatalf("csv header baseline: %v", err) + } + bad := cloneRecorded(t, rec) + bad.Steps[0].Response.Headers["Content-Disposition"] = `attachment; filename="other.csv"` + _, err = ReplayFlow(ctx, bad, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "content-disposition") { + t.Fatalf("Content-Disposition mismatch: %v", err) + } + }) + + t.Run("sidecar digest and path", func(t *testing.T) { + dir := t.TempDir() + payload := []byte("hello-bin") + sum := sha256.Sum256(payload) + if err := os.WriteFile(filepath.Join(dir, "data.bin"), payload, 0o644); err != nil { + t.Fatal(err) + } + unsafe := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n body_file: ../secret.bin\n" + if err := os.WriteFile(filepath.Join(dir, "unsafe.yaml"), []byte(unsafe), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadFlow(filepath.Join(dir, "unsafe.yaml")); err == nil || !strings.Contains(err.Error(), "body_file") { + t.Fatalf("parent sidecar must fail: %v", err) + } + abs := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n body_file: /tmp/secret.bin\n" + if err := os.WriteFile(filepath.Join(dir, "abs.yaml"), []byte(abs), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadFlow(filepath.Join(dir, "abs.yaml")); err == nil || !strings.Contains(err.Error(), "body_file") { + t.Fatalf("absolute sidecar must fail: %v", err) + } + + srv := binaryServer(t, "application/octet-stream", payload) + good := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n status: 200\n headers:\n Content-Type: application/octet-stream\n body_file: data.bin\n sha256: " + hex.EncodeToString(sum[:]) + "\n" + gp := filepath.Join(dir, "good.yaml") + if err := os.WriteFile(gp, []byte(good), 0o644); err != nil { + t.Fatal(err) + } + flow, err := LoadFlow(gp) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(ctx, flow, ReplayConfig{Target: srv.URL, BaseDir: dir}); err != nil { + t.Fatalf("matching sidecar digest must pass: %v", err) + } + bad := strings.Replace(good, hex.EncodeToString(sum[:]), strings.Repeat("ab", 32), 1) + bp := filepath.Join(dir, "bad-digest.yaml") + if err := os.WriteFile(bp, []byte(bad), 0o644); err != nil { + t.Fatal(err) + } + flow, err = LoadFlow(bp) + if err != nil { + t.Fatal(err) + } + _, err = ReplayFlow(ctx, flow, ReplayConfig{Target: srv.URL, BaseDir: dir}) + if err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("digest mismatch: %v", err) + } + }) + + t.Run("comparison failure continues later steps", func(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/one": + _, _ = w.Write([]byte(`{"data":"no"}`)) + default: + _, _ = w.Write([]byte(`{"data":"ok"}`)) + } + })) + t.Cleanup(srv.Close) + flow := Flow{ + Version: 1, + Name: "mismatch-continue", + Steps: []Step{ + {ID: "a", Request: Request{Method: http.MethodGet, Path: "/one"}, Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"yes"}`)}}, + {ID: "b", Request: Request{Method: http.MethodGet, Path: "/two"}, Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}}, + }, + } + res, err := ReplayFlow(ctx, flow, ReplayConfig{Target: srv.URL}) + if err == nil { + t.Fatal("mismatch must error") + } + if hits.Load() != 2 { + t.Fatalf("later step must run, hits=%d", hits.Load()) + } + if len(res.Steps) != 2 || res.Steps[1].Skipped || !res.Steps[1].OK { + t.Fatalf("step b should pass: %+v", res.Steps) + } + assertPathMismatch(t, err, "$.data") + }) + + t.Run("capture failure skips remainder", func(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":"ok"}`)) + })) + t.Cleanup(srv.Close) + flow := Flow{ + Version: 1, + Name: "capture-skip", + Steps: []Step{ + { + ID: "a", + Request: Request{Method: http.MethodGet, Path: "/one"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}, + Capture: []CaptureRule{{From: "response.json", Path: "$.token", As: "jwt:alice"}}, + }, + {ID: "b", Request: Request{Method: http.MethodGet, Path: "/two"}, Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}}, + }, + } + res, err := ReplayFlow(ctx, flow, ReplayConfig{Target: srv.URL}) + if err == nil { + t.Fatal("failed capture must error") + } + if hits.Load() != 1 { + t.Fatalf("capture fail should skip rest, hits=%d", hits.Load()) + } + if len(res.Steps) != 2 || !res.Steps[1].Skipped { + t.Fatalf("step b should skip: %+v", res.Steps) + } + if !strings.Contains(err.Error(), "capture") { + t.Fatalf("capture diagnostic: %v", err) + } + }) + + t.Run("missing variable fails before send", func(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(srv.Close) + flow := Flow{Version: 1, Name: "missing-var", Steps: []Step{ + {ID: "a", Request: Request{Method: http.MethodGet, Path: "/items/{{missing}}"}, Response: Response{Status: 200}}, + {ID: "b", Request: Request{Method: http.MethodGet, Path: "/later"}, Response: Response{Status: 200}}, + }} + res, err := ReplayFlow(ctx, flow, ReplayConfig{Target: srv.URL}) + if err == nil || !strings.Contains(err.Error(), "unresolved") && !strings.Contains(err.Error(), "placeholder") { + t.Fatalf("missing variable: %v", err) + } + if hits.Load() != 0 { + t.Fatalf("must not send unresolved placeholder, hits=%d", hits.Load()) + } + if len(res.Steps) != 2 || !res.Steps[1].Skipped { + t.Fatalf("remainder must skip: %+v", res.Steps) + } + }) + + t.Run("unknown capture source", func(t *testing.T) { + store, err := OpenStore("") + if err != nil { + t.Fatal(err) + } + srv := jsonServer(t, `{"ok":true}`) + spec := Flow{Version: 1, Name: "bad-capture", Steps: []Step{{ + ID: "a", + Request: Request{Method: http.MethodGet, Path: "/ok"}, + Capture: []CaptureRule{{From: "response.unknown", Path: "$.ok", As: "x"}}, + }}} + _, err = RecordFlow(ctx, spec, RecordConfig{Target: srv.URL, Store: store}) + if err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("unknown capture from: %v", err) + } + }) + + t.Run("capture-by-reference mismatch", func(t *testing.T) { + store, err := OpenStore("") + if err != nil { + t.Fatal(err) + } + n := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/create": + _, _ = w.Write([]byte(`{"token":"shareTokValue99"}`)) + case "/show": + n++ + if n == 1 { + _, _ = w.Write([]byte(`{"token":"shareTokValue99"}`)) + } else { + _, _ = w.Write([]byte(`{"token":"shareTokOther00"}`)) + } + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + spec := Flow{ + Version: 1, + Name: "capture-ref", + Steps: []Step{ + { + ID: "create", + Request: Request{Method: http.MethodGet, Path: "/create"}, + Capture: []CaptureRule{{From: "response.json", Path: "$.token", As: "share:item"}}, + }, + {ID: "show", Request: Request{Method: http.MethodGet, Path: "/show"}}, + }, + } + rec, err := RecordFlow(ctx, spec, RecordConfig{Target: srv.URL, Store: store}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(rec.Steps[0].Response.Body), "{{share:item}}") { + t.Fatalf("create not scrubbed: %s", rec.Steps[0].Response.Body) + } + if !strings.Contains(string(rec.Steps[1].Response.Body), "{{share:item}}") { + t.Fatalf("show not scrubbed by reference: %s", rec.Steps[1].Response.Body) + } + replayStore, err := OpenStore("") + if err != nil { + t.Fatal(err) + } + _, err = ReplayFlow(ctx, rec, ReplayConfig{Target: srv.URL, Store: replayStore}) + if err == nil { + t.Fatal("capture-by-reference mismatch must fail") + } + assertPathMismatch(t, err, "token") + }) +} + +func TestFlowContractTwoFlowsContinue(t *testing.T) { + ctx := context.Background() + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"v":2}`)) + })) + t.Cleanup(srv.Close) + + first := Flow{Version: 1, Name: "first", Steps: []Step{{ + ID: "a", Request: Request{Method: http.MethodGet, Path: "/a"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"v":1}`)}, + }}} + second := Flow{Version: 1, Name: "second", Steps: []Step{{ + ID: "b", Request: Request{Method: http.MethodGet, Path: "/b"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"v":2}`)}, + }}} + _, err := ReplayFlow(ctx, first, ReplayConfig{Target: srv.URL}) + if err == nil { + t.Fatal("first flow must fail") + } + if _, err := ReplayFlow(ctx, second, ReplayConfig{Target: srv.URL}); err != nil { + t.Fatalf("later flow must still run: %v", err) + } + if hits.Load() != 2 { + t.Fatalf("both flows must execute, hits=%d", hits.Load()) + } + var mis *MismatchError + if !errors.As(err, &mis) || mis.Result.OK { + t.Fatalf("first flow mismatch result: %v", err) + } +} diff --git a/tide/manifest_contract_test.go b/tide/manifest_contract_test.go new file mode 100644 index 0000000..e937cf4 --- /dev/null +++ b/tide/manifest_contract_test.go @@ -0,0 +1,116 @@ +package tide + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func TestManifestContract(t *testing.T) { + ctx := context.Background() + fixtures := t.TempDir() + if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil { + t.Fatal(err) + } + + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/pass": + _, _ = w.Write([]byte(`{"v":1}`)) + case "/fail": + _, _ = w.Write([]byte(`{"v":2}`)) + default: + _, _ = w.Write([]byte(`{"v":0}`)) + } + })) + t.Cleanup(srv.Close) + + writeRouteFixture(t, srv.URL, fixtures, "pass", "/pass", `{"v":1}`) + writeRouteFixture(t, srv.URL, fixtures, "fail", "/fail", `{"v":1}`) + + m := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{ + routeEntry("GET /pass public", "/pass", StatusPorted, "routes/pass.yaml"), + routeEntry("GET /fail public", "/fail", StatusPorted, "routes/fail.yaml"), + {ID: "GET /none public", Method: http.MethodGet, Path: "/none", AuthGroup: "public", Status: StatusPending}, + }, + } + before := hits.Load() + cov, err := ReplayManifest(ctx, m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Mode: ModeRequireRecorded}) + if err == nil { + t.Fatal("ported fail plus unrecorded required must be nonzero") + } + if cov.Passing != 1 || cov.Failing != 1 || cov.Unrecorded != 1 || cov.Recorded != 2 { + t.Fatalf("coverage %+v", cov) + } + if cov.Pending != 1 || cov.Ported != 2 { + t.Fatalf("pending/ported counts %+v", cov) + } + if !strings.Contains(cov.SummaryLine(), "recorded 2/3") { + t.Fatalf("summary %s", cov.SummaryLine()) + } + if hits.Load()-before != 2 { + t.Fatalf("comparison failure must still replay later recorded flows, extra hits=%d", hits.Load()-before) + } + foundFailPath := false + for _, d := range cov.Diffs { + if strings.Contains(d, "GET /fail public") && (strings.Contains(d, "$.v") || strings.Contains(d, ".v")) { + foundFailPath = true + } + } + if !foundFailPath { + t.Fatalf("coverage diffs missing path for failing route: %v", cov.Diffs) + } + + t.Run("pending mismatch is measured not fatal", func(t *testing.T) { + pending := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{routeEntry("GET /fail public", "/fail", StatusPending, "routes/fail.yaml")}, + } + cov, err := ReplayManifest(ctx, pending, ManifestConfig{Target: srv.URL, Fixtures: fixtures}) + if err != nil { + t.Fatalf("pending mismatch must not fail: %v", err) + } + if cov.Failing != 1 || cov.Passing != 0 { + t.Fatalf("pending failing %+v", cov) + } + if _, err := ReplayManifest(ctx, pending, ManifestConfig{Target: srv.URL, Fixtures: fixtures, SelfCheck: true}); err == nil { + t.Fatal("self-check must fail pending mismatches") + } + }) + + t.Run("require-recorded reports unrecorded", func(t *testing.T) { + empty := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{{ + ID: "GET /ghost public", Method: http.MethodGet, Path: "/ghost", + AuthGroup: "public", Status: StatusPending, + }}, + } + if err := ValidateManifest(empty, fixtures, ModeAllowIncomplete); err != nil { + t.Fatal(err) + } + if err := ValidateManifest(empty, fixtures, ModeRequireRecorded); err == nil { + t.Fatal("missing cases must fail require-recorded") + } + cov, err := ReplayManifest(ctx, empty, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Mode: ModeRequireRecorded}) + if err == nil { + t.Fatal("unrecorded required must error") + } + if cov.Unrecorded != 1 || cov.Passing != 0 { + t.Fatalf("unrecorded coverage %+v", cov) + } + }) +}