From aa165fe3d0eb95350348df0a2fc81a321eaead36 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Thu, 17 Sep 2026 12:32:49 +0200 Subject: [PATCH] feat(02-02): capture, scrub, and strictly diff stateful flows Resolve named placeholders from a private variable store, mask dates and ids after shape checks, and keep comparing independent steps. Co-authored-by: Cursor --- cmd/summer/parity.go | 30 ++- tide/capture_test.go | 273 ++++++++++++++++++++ tide/diff.go | 68 ++++- tide/diff_test.go | 34 +++ tide/flow.go | 10 +- tide/flow_test.go | 80 ++++++ tide/headers_test.go | 77 ++++++ tide/normalize.go | 133 ++++++++++ tide/normalize_test.go | 44 ++++ tide/proxy.go | 194 ++------------- tide/record.go | 44 +++- tide/replay.go | 59 ++++- tide/variables.go | 547 +++++++++++++++++++++++++++++++++++++++++ 13 files changed, 1390 insertions(+), 203 deletions(-) create mode 100644 tide/capture_test.go create mode 100644 tide/diff_test.go create mode 100644 tide/flow_test.go create mode 100644 tide/headers_test.go create mode 100644 tide/normalize.go create mode 100644 tide/normalize_test.go create mode 100644 tide/variables.go diff --git a/cmd/summer/parity.go b/cmd/summer/parity.go index e6237e2..740a81a 100644 --- a/cmd/summer/parity.go +++ b/cmd/summer/parity.go @@ -33,6 +33,8 @@ func parityRecordCommand() bonfire.Command { {Name: "spec", Description: "YAML request spec path"}, {Name: "target", Description: "Base URL of the HTTP backend"}, {Name: "output", Description: "Destination fixture path"}, + {Name: "rules", Description: "Committed YAML capture rules"}, + {Name: "vars", Description: "Private mode-0600 variable store outside fixtures"}, }, Run: runParityRecord, } @@ -45,6 +47,7 @@ func parityReplayCommand() bonfire.Command { Flags: []bonfire.Flag{ {Name: "fixtures", Description: "Recorded YAML fixture path"}, {Name: "target", Description: "Base URL of the HTTP backend"}, + {Name: "vars", Description: "Private mode-0600 variable store"}, }, Run: runParityReplay, } @@ -99,7 +102,22 @@ func runParityRecord(ctx context.Context, in bonfire.Input, out bonfire.Output) if err != nil { return err } - flow, err := tide.RecordFlow(ctx, spec, tide.RecordConfig{Target: target}) + cfg := tide.RecordConfig{Target: target} + if rulesPath, ok := in.Flag("rules"); ok && strings.TrimSpace(rulesPath) != "" { + rules, err := tide.LoadRules(rulesPath) + if err != nil { + return err + } + cfg.Rules = rules + } + if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { + store, err := tide.OpenStore(varsPath) + if err != nil { + return err + } + cfg.Store = store + } + flow, err := tide.RecordFlow(ctx, spec, cfg) if err != nil { return err } @@ -123,7 +141,15 @@ func runParityReplay(ctx context.Context, in bonfire.Input, out bonfire.Output) if err != nil { return err } - result, err := tide.ReplayFlow(ctx, flow, tide.ReplayConfig{Target: target}) + cfg := tide.ReplayConfig{Target: target} + if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { + store, err := tide.OpenStore(varsPath) + if err != nil { + return err + } + cfg.Store = store + } + result, err := tide.ReplayFlow(ctx, flow, cfg) if err != nil { return err } diff --git a/tide/capture_test.go b/tide/capture_test.go new file mode 100644 index 0000000..7148a83 --- /dev/null +++ b/tide/capture_test.go @@ -0,0 +1,273 @@ +package tide + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +const testJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhbGljZSJ9.signaturehere123456" + +func TestCaptureAndPlaceholderResolution(t *testing.T) { + store, err := OpenStore("") + if err != nil { + t.Fatal(err) + } + step := Step{ + ID: "login", + Response: Response{ + Headers: map[string]string{"Content-Type": "application/json"}, + Body: Body(`{"token":"` + testJWT + `"}`), + }, + Capture: []CaptureRule{{ + From: "response.json", + Path: "$.token", + As: "jwt:alice", + Identity: "alice", + Category: "jwt", + }}, + } + if err := CaptureStep(store, &step); err != nil { + t.Fatal(err) + } + if err := ScrubStep(store, &step); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(step.Response.Body), "{{jwt:alice}}") { + t.Fatalf("body not scrubbed: %s", step.Response.Body) + } + if strings.Contains(string(step.Response.Body), testJWT) { + t.Fatal("jwt left in body") + } + got, err := store.Expand("Bearer {{jwt:alice}}") + if err != nil { + t.Fatal(err) + } + if got != "Bearer "+testJWT { + t.Fatalf("expand: %q", got) + } + if _, err := store.Expand("{{missing}}"); err == nil || !strings.Contains(err.Error(), "unresolved") { + t.Fatalf("missing placeholder: %v", err) + } + + step2 := Step{ + ID: "pkce", + Request: Request{ + Method: "POST", + Path: "/authorize", + Body: Body("code_verifier=pkceVerifierValue1&client_id=x"), + }, + Response: Response{ + Status: 302, + Headers: map[string]string{"Location": "/cb?code=oauthCode99"}, + }, + Capture: []CaptureRule{ + {From: "request.form", Name: "code_verifier", As: "pkce:alice", Category: "pkce"}, + {From: "response.location.query", Name: "code", As: "oauth:alice-code", Category: "oauth_code"}, + }, + } + if err := CaptureStep(store, &step2); err != nil { + t.Fatal(err) + } + if err := ScrubStep(store, &step2); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(step2.Request.Body), "{{pkce:alice}}") || strings.Contains(string(step2.Request.Body), "pkceVerifierValue1") { + t.Fatalf("pkce not scrubbed: %s", step2.Request.Body) + } + if !strings.Contains(step2.Response.Headers["Location"], "{{oauth:alice-code}}") { + t.Fatalf("code not scrubbed: %v", step2.Response.Headers) + } +} + +func TestScrubRejectsUnclassifiedAndPersistsVars(t *testing.T) { + dir := t.TempDir() + varsPath := filepath.Join(dir, "vars.yaml") + store, err := OpenStore(varsPath) + if err != nil { + t.Fatal(err) + } + step := Step{ + ID: "login", + Response: Response{ + Headers: map[string]string{"Content-Type": "application/json"}, + Body: Body(`{"token":"` + testJWT + `"}`), + }, + Capture: []CaptureRule{{From: "response.json", Path: "$.token", As: "jwt:alice", Category: "jwt"}}, + } + if err := CaptureStep(store, &step); err != nil { + t.Fatal(err) + } + if err := ScrubStep(store, &step); err != nil { + t.Fatal(err) + } + if err := store.Save(); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(varsPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), testJWT) { + t.Fatalf("vars missing secret:\n%s", raw) + } + st, err := os.Stat(varsPath) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != 0o600 { + t.Fatalf("mode %o", st.Mode().Perm()) + } + unknown := Step{ + ID: "bad", + Response: Response{Body: Body(`{"token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJib2IifQ.otherSignatureValue99"}`)}, + } + if err := ScrubStep(store, &unknown); err == nil { + t.Fatal("unclassified leftover jwt must fail") + } +} + +func TestProxyLoginTokenAndPKCESession(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/login": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"` + testJWT + `"}`)) + case r.URL.Path == "/me": + if r.Header.Get("Authorization") != "Bearer "+testJWT { + http.Error(w, "no", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + case r.URL.Path == "/authorize": + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "code_verifier=pkceVerifierValue1") { + http.Error(w, "missing verifier", http.StatusBadRequest) + return + } + w.Header().Set("Location", "/cb?code=oauthCode99") + w.WriteHeader(http.StatusFound) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + + fixtures := t.TempDir() + varsPath := filepath.Join(t.TempDir(), "vars.yaml") + rules := mustParseRules(t, ""+ + "client: nuxt\n"+ + "keep_request_headers:\n"+ + " - Authorization\n"+ + " - Content-Type\n"+ + "keep_response_headers:\n"+ + " - Content-Type\n"+ + " - Location\n"+ + "routes:\n"+ + " - method: POST\n"+ + " path: /login\n"+ + " capture:\n"+ + " - from: response.json\n"+ + " path: $.token\n"+ + " as: jwt:alice\n"+ + " identity: alice\n"+ + " category: jwt\n"+ + " - method: GET\n"+ + " path: /me\n"+ + " - method: POST\n"+ + " path: /authorize\n"+ + " capture:\n"+ + " - from: request.form\n"+ + " name: code_verifier\n"+ + " as: pkce:alice\n"+ + " category: pkce\n"+ + " - from: response.location.query\n"+ + " name: code\n"+ + " as: oauth:alice-code\n"+ + " category: oauth_code\n") + proxy, err := NewProxy(ProxyConfig{ + Listen: "127.0.0.1:0", + Upstream: upstream.URL, + Fixtures: fixtures, + VarsPath: varsPath, + Rules: rules, + }) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(proxy.Handler()) + t.Cleanup(srv.Close) + + post(t, srv.URL+"/login", "sess", "application/json", `{}`) + req, err := http.NewRequest(http.MethodGet, srv.URL+"/me", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(SessionHeader, "sess") + req.Header.Set("Authorization", "Bearer "+testJWT) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("me: %d", resp.StatusCode) + } + post(t, srv.URL+"/authorize", "sess", "application/x-www-form-urlencoded", "code_verifier=pkceVerifierValue1") + if err := proxy.Flush(); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(filepath.Join(fixtures, "nuxt", "sess.yaml")) + if err != nil { + t.Fatal(err) + } + text := string(raw) + if strings.Contains(text, testJWT) || strings.Contains(text, "pkceVerifierValue1") || strings.Contains(text, "oauthCode99") { + t.Fatalf("secrets in fixture:\n%s", text) + } + if !strings.Contains(text, "{{jwt:alice}}") || !strings.Contains(text, "{{pkce:alice}}") || !strings.Contains(text, "{{oauth:alice-code}}") { + t.Fatalf("placeholders missing:\n%s", text) + } + + flow, err := LoadFlow(filepath.Join(fixtures, "nuxt", "sess.yaml")) + if err != nil { + t.Fatal(err) + } + store, err := OpenStore(varsPath) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(context.Background(), flow, ReplayConfig{Target: upstream.URL, Store: store}); err != nil { + t.Fatalf("replay captured session: %v", err) + } +} + +func post(t *testing.T, url, session, ct, body string) { + t.Helper() + req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set(SessionHeader, session) + req.Header.Set("Content-Type", ct) + client := &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode >= 400 { + t.Fatalf("%s: %d", url, resp.StatusCode) + } +} diff --git a/tide/diff.go b/tide/diff.go index 1f90872..26263f8 100644 --- a/tide/diff.go +++ b/tide/diff.go @@ -5,21 +5,85 @@ import ( "encoding/json" "fmt" "mime" + "net/http" "strconv" "strings" "unicode" "unicode/utf8" ) -func compareBodies(want, got Response) []Diff { +func compareBodies(want, got Response, step Step) []Diff { wantJSON := isJSONContentType(want.Headers) gotJSON := isJSONContentType(got.Headers) if wantJSON && gotJSON { - return diffJSON([]byte(want.Body), []byte(got.Body)) + wantRaw, wantDiffs := normalizeJSON([]byte(want.Body), step) + gotRaw, gotDiffs := normalizeJSON([]byte(got.Body), step) + diffs := append([]Diff{}, wantDiffs...) + diffs = append(diffs, gotDiffs...) + diffs = append(diffs, diffJSON(wantRaw, gotRaw)...) + return diffs } return diffBytes([]byte(want.Body), []byte(got.Body)) } +var globalCompareHeaders = []string{ + "Content-Type", + "X-Total-Count", + "Access-Control-Allow-Origin", + "Access-Control-Allow-Credentials", + "Access-Control-Allow-Headers", + "Access-Control-Allow-Methods", + "Access-Control-Expose-Headers", +} + +var extraCompareHeaders = []string{ + "Cache-Control", + "Pragma", + "WWW-Authenticate", + "Content-Disposition", + "Link", +} + +var neverCompareHeaders = []string{ + "Date", + "Server", + "X-Request-Id", + "X-Request-ID", + "X-Correlation-Id", +} + +func compareHeaders(want, got, extra map[string]string) []Diff { + allow := map[string]bool{} + for _, n := range globalCompareHeaders { + allow[http.CanonicalHeaderKey(n)] = true + } + for _, n := range extraCompareHeaders { + allow[http.CanonicalHeaderKey(n)] = true + } + for n := range extra { + allow[http.CanonicalHeaderKey(n)] = true + } + for _, n := range neverCompareHeaders { + delete(allow, http.CanonicalHeaderKey(n)) + } + var diffs []Diff + for k, wv := range want { + ck := http.CanonicalHeaderKey(k) + if !allow[ck] { + continue + } + gv := headerValue(got, k) + if gv == wv { + continue + } + if gv == "" { + gv = "" + } + diffs = append(diffs, Diff{Path: "header." + ck, Expected: wv, Actual: gv}) + } + return diffs +} + func isJSONContentType(headers map[string]string) bool { ct := headerValue(headers, "Content-Type") if ct == "" { diff --git a/tide/diff_test.go b/tide/diff_test.go new file mode 100644 index 0000000..54941cb --- /dev/null +++ b/tide/diff_test.go @@ -0,0 +1,34 @@ +package tide + +import ( + "strings" + "testing" +) + +func TestDiffParityClasses(t *testing.T) { + step := Step{ID: "d"} + ct := jsonCT() + check := func(name, want, got, path string) { + t.Helper() + diffs := compareBodies(Response{Headers: ct, Body: Body(want)}, Response{Headers: ct, Body: Body(got)}, step) + if len(diffs) == 0 { + t.Fatalf("%s: expected mismatch", name) + } + found := false + for _, d := range diffs { + if strings.Contains(d.Path, path) && d.Expected != d.Actual { + found = true + break + } + } + if !found { + t.Fatalf("%s: want path %s in %+v", name, path, diffs) + } + } + check("null vs array", `{"tracklist":[]}`, `{"tracklist":null}`, "tracklist") + check("money string vs number", `{"price":"1.5000"}`, `{"price":1.5}`, "price") + check("null vs absent", `{"deleted_at":null}`, `{}`, "deleted_at") + check("tri-state bool", `{"is_owner":null}`, `{"is_owner":false}`, "is_owner") + check("conditional key", `{"data":{"name":"a"}}`, `{"data":{"name":"a","extra":1}}`, "extra") + check("date vs Z", `{"created_at":"2026-01-01T00:00:00+00:00"}`, `{"created_at":"2026-01-01T00:00:00Z"}`, "created_at") +} diff --git a/tide/flow.go b/tide/flow.go index e54db04..6a958a3 100644 --- a/tide/flow.go +++ b/tide/flow.go @@ -74,6 +74,8 @@ type RecordConfig struct { Target string Client *http.Client MaxBody int64 + Store *Store + Rules Rules } // ReplayConfig injects the HTTP target, client and body bound for replay. @@ -81,6 +83,7 @@ type ReplayConfig struct { Target string Client *http.Client MaxBody int64 + Store *Store } // Result is the outcome of replaying a flow. @@ -91,9 +94,10 @@ type Result struct { // StepResult is the outcome of one replayed step. type StepResult struct { - ID string - OK bool - Diffs []Diff + ID string + OK bool + Skipped bool + Diffs []Diff } // Diff is one structural JSON or raw-byte mismatch. diff --git a/tide/flow_test.go b/tide/flow_test.go new file mode 100644 index 0000000..e85e2a3 --- /dev/null +++ b/tide/flow_test.go @@ -0,0 +1,80 @@ +package tide + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestFlowMismatchContinuesAndCaptureSkipsRest(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/one": + _, _ = w.Write([]byte(`{"data":"no"}`)) + default: + _, _ = w.Write([]byte(`{"data":"ok"}`)) + } + })) + t.Cleanup(srv.Close) + + mismatch := Flow{ + Version: 1, + Name: "mismatch-continue", + Steps: []Step{ + { + ID: "a", + Request: Request{Method: http.MethodGet, Path: "/one"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"yes"}`)}, + }, + { + ID: "b", + Request: Request{Method: http.MethodGet, Path: "/two"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}, + }, + }, + } + res, err := ReplayFlow(context.Background(), mismatch, ReplayConfig{Target: srv.URL}) + if err == nil { + t.Fatal("mismatch must error") + } + if hits.Load() != 2 { + t.Fatalf("mismatch should continue, hits=%d", hits.Load()) + } + if len(res.Steps) != 2 || res.Steps[1].Skipped || !res.Steps[1].OK { + t.Fatalf("step b should run and pass: %+v", res.Steps) + } + + hits.Store(0) + captureFail := Flow{ + Version: 1, + Name: "capture-skip", + Steps: []Step{ + { + ID: "a", + Request: Request{Method: http.MethodGet, Path: "/two"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}, + Capture: []CaptureRule{{From: "response.json", Path: "$.token", As: "jwt:alice"}}, + }, + { + ID: "b", + Request: Request{Method: http.MethodGet, Path: "/two"}, + Response: Response{Status: 200, Headers: jsonCT(), Body: Body(`{"data":"ok"}`)}, + }, + }, + } + res, err = ReplayFlow(context.Background(), captureFail, ReplayConfig{Target: srv.URL}) + if err == nil { + t.Fatal("failed capture must error") + } + if hits.Load() != 1 { + t.Fatalf("capture fail should skip rest, hits=%d", hits.Load()) + } + if len(res.Steps) != 2 || !res.Steps[1].Skipped { + t.Fatalf("step b should skip: %+v", res.Steps) + } +} diff --git a/tide/headers_test.go b/tide/headers_test.go new file mode 100644 index 0000000..c0896b6 --- /dev/null +++ b/tide/headers_test.go @@ -0,0 +1,77 @@ +package tide + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHeadersAllowListIgnoresDate(t *testing.T) { + want := Response{ + Status: 401, + Headers: map[string]string{ + "Content-Type": "application/json", + "WWW-Authenticate": `Bearer error="invalid_token"`, + "Cache-Control": "no-store", + "Date": "Wed, 01 Jan 2020 00:00:00 GMT", + }, + Body: Body(`{"ok":false}`), + } + got := Response{ + Status: 401, + Headers: map[string]string{ + "Content-Type": "application/json", + "WWW-Authenticate": `Bearer error="invalid_token"`, + "Cache-Control": "no-store", + "Date": "Thu, 02 Jan 2020 00:00:00 GMT", + "Server": "php", + }, + Body: Body(`{"ok":false}`), + } + diffs := compareStep(Step{ID: "h", Response: want, Headers: map[string]string{"WWW-Authenticate": "", "Cache-Control": ""}}, got) + if len(diffs) != 0 { + t.Fatalf("Date/Server must be ignored: %+v", diffs) + } + + got.Headers["WWW-Authenticate"] = `Bearer error="other"` + diffs = compareStep(Step{ID: "h", Response: want}, got) + if len(diffs) == 0 { + t.Fatal("WWW-Authenticate mismatch must fail") + } + if !strings.Contains(strings.ToLower(diffs[0].Path), "www-authenticate") { + t.Fatalf("path %s", diffs[0].Path) + } +} + +func TestHeadersReplayAgainstServer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Pragma", "no-cache") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + flow := Flow{ + Version: 1, + Name: "headers", + Steps: []Step{{ + ID: "h", + Request: Request{Method: http.MethodGet, Path: "/"}, + Response: Response{ + Status: 200, + Headers: map[string]string{ + "Content-Type": "application/json", + "Cache-Control": "no-store", + "Pragma": "no-cache", + }, + Body: Body(`{"ok":true}`), + }, + }}, + } + if _, err := ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL}); err != nil { + t.Fatal(err) + } +} diff --git a/tide/normalize.go b/tide/normalize.go new file mode 100644 index 0000000..1c44729 --- /dev/null +++ b/tide/normalize.go @@ -0,0 +1,133 @@ +package tide + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" +) + +var carbonOffsetRe = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$`) + +const ( + maskDatetime = "" + maskID = "" +) + +func normalizeJSON(raw []byte, step Step) ([]byte, []Diff) { + if len(strings.TrimSpace(string(raw))) == 0 { + return raw, nil + } + val, err := decodeJSON(raw) + if err != nil { + return raw, nil + } + var diffs []Diff + masked := maskValue("$", val, step, &diffs) + out, err := json.Marshal(masked) + if err != nil { + return raw, diffs + } + return out, diffs +} + +func maskValue(path string, val any, step Step, diffs *[]Diff) any { + switch v := val.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for k, child := range v { + out[k] = maskValue(pathJoin(path, k), child, step, diffs) + } + return out + case []any: + out := make([]any, len(v)) + for i, child := range v { + out[i] = maskValue(fmt.Sprintf("%s[%d]", path, i), child, step, diffs) + } + return out + default: + return maskLeaf(path, val, step, diffs) + } +} + +func maskLeaf(path string, val any, step Step, diffs *[]Diff) any { + key := lastPathKey(path) + if key == "slug" || disabledPath(step, path, key) { + return val + } + if isDateKey(key) { + return maskDate(path, val, diffs) + } + if isIDKey(key) { + return maskIDValue(path, val, diffs) + } + return val +} + +func maskDate(path string, val any, diffs *[]Diff) any { + if val == nil { + return nil + } + s, ok := val.(string) + if !ok { + *diffs = append(*diffs, Diff{Path: path, Expected: "Carbon +00:00 string or null", Actual: formatValue(val)}) + return val + } + if !carbonOffsetRe.MatchString(s) { + *diffs = append(*diffs, Diff{Path: path, Expected: "Carbon +00:00", Actual: strconvQuote(s)}) + return val + } + return maskDatetime +} + +func maskIDValue(path string, val any, diffs *[]Diff) any { + n, ok := val.(json.Number) + if !ok { + *diffs = append(*diffs, Diff{Path: path, Expected: "integer id", Actual: formatValue(val)}) + return val + } + if strings.Contains(string(n), ".") { + *diffs = append(*diffs, Diff{Path: path, Expected: "integer id", Actual: "number " + string(n)}) + return val + } + return maskID +} + +func isDateKey(key string) bool { + return strings.HasSuffix(key, "_at") +} + +func isIDKey(key string) bool { + if key == "id" { + return true + } + return strings.HasSuffix(key, "_id") && !strings.HasSuffix(key, "_at") +} + +func lastPathKey(path string) string { + path = strings.TrimPrefix(path, "$.") + if i := strings.LastIndex(path, "."); i >= 0 { + path = path[i+1:] + } + if i := strings.IndexByte(path, '['); i >= 0 { + path = path[:i] + } + return path +} + +func disabledPath(step Step, jsonPath, key string) bool { + for _, rule := range step.Normalize { + if !rule.Disable { + continue + } + p := strings.TrimSpace(rule.Path) + if p == jsonPath || p == key || strings.TrimPrefix(p, "$.") == strings.TrimPrefix(jsonPath, "$.") { + return true + } + } + return false +} + +func strconvQuote(s string) string { + return `"` + s + `"` +} diff --git a/tide/normalize_test.go b/tide/normalize_test.go new file mode 100644 index 0000000..00156e6 --- /dev/null +++ b/tide/normalize_test.go @@ -0,0 +1,44 @@ +package tide + +import ( + "strings" + "testing" +) + +func TestNormalizeDateIDAndDisable(t *testing.T) { + step := Step{ID: "n"} + want := []byte(`{"id":1,"created_at":"2026-01-01T00:00:00+00:00","slug":"keep-me"}`) + gotOK := []byte(`{"id":9,"created_at":"2026-02-02T00:00:00+00:00","slug":"keep-me"}`) + gotZ := []byte(`{"id":9,"created_at":"2026-02-02T00:00:00Z","slug":"keep-me"}`) + gotStrID := []byte(`{"id":"9","created_at":"2026-02-02T00:00:00+00:00","slug":"keep-me"}`) + gotSlug := []byte(`{"id":9,"created_at":"2026-02-02T00:00:00+00:00","slug":"other"}`) + + if diffs := compareBodies(Response{Headers: jsonCT(), Body: Body(want)}, Response{Headers: jsonCT(), Body: Body(gotOK)}, step); len(diffs) != 0 { + t.Fatalf("masked date/id should pass: %+v", diffs) + } + zdiffs := compareBodies(Response{Headers: jsonCT(), Body: Body(want)}, Response{Headers: jsonCT(), Body: Body(gotZ)}, step) + if len(zdiffs) == 0 { + t.Fatal("Z date must fail") + } + if !strings.Contains(zdiffs[0].Path, "created_at") { + t.Fatalf("Z path %s", zdiffs[0].Path) + } + idDiffs := compareBodies(Response{Headers: jsonCT(), Body: Body(want)}, Response{Headers: jsonCT(), Body: Body(gotStrID)}, step) + if len(idDiffs) == 0 || !strings.Contains(idDiffs[0].Path, "id") { + t.Fatalf("string id: %+v", idDiffs) + } + slugDiffs := compareBodies(Response{Headers: jsonCT(), Body: Body(want)}, Response{Headers: jsonCT(), Body: Body(gotSlug)}, step) + if len(slugDiffs) == 0 || !strings.Contains(slugDiffs[0].Path, "slug") { + t.Fatalf("slug must stay exact: %+v", slugDiffs) + } + + disabled := Step{ID: "n", Normalize: []NormalizeRule{{Path: "created_at", Disable: true}}} + dDiffs := compareBodies(Response{Headers: jsonCT(), Body: Body(want)}, Response{Headers: jsonCT(), Body: Body(gotOK)}, disabled) + if len(dDiffs) == 0 { + t.Fatal("disabled date mask must compare raw timestamps") + } +} + +func jsonCT() map[string]string { + return map[string]string{"Content-Type": "application/json"} +} diff --git a/tide/proxy.go b/tide/proxy.go index 0632986..19bac45 100644 --- a/tide/proxy.go +++ b/tide/proxy.go @@ -56,6 +56,7 @@ type Proxy struct { upstream *url.URL rp *httputil.ReverseProxy limit int64 + store *Store mu sync.Mutex sessions map[string]*sessionBuf @@ -83,15 +84,18 @@ func NewProxy(cfg ProxyConfig) (*Proxy, error) { if err := validateRules(cfg.Rules); err != nil { return nil, err } - if cfg.VarsPath != "" { - if err := prepareVarsFile(cfg.VarsPath, cfg.Fixtures); err != nil { - return nil, err - } + if err := varsOutsideFixtures(cfg.VarsPath, cfg.Fixtures); err != nil { + return nil, err + } + store, err := OpenStore(cfg.VarsPath) + if err != nil { + return nil, err } p := &Proxy{ cfg: cfg, upstream: upstream, limit: maxBody(cfg.MaxBody), + store: store, sessions: make(map[string]*sessionBuf), failed: make(map[string]error), } @@ -268,9 +272,15 @@ func (p *Proxy) recordStep(state *captureState, resp *http.Response, respBody [] if route != nil { step.Capture = append([]CaptureRule(nil), route.Capture...) } - if err := rejectUnclassifiedCredentials(step); err != nil { + if err := CaptureStep(p.store, &step); err != nil { return fmt.Errorf("tide: session %q: %w", state.session, err) } + if err := ScrubStep(p.store, &step); err != nil { + return fmt.Errorf("tide: session %q: %w", state.session, err) + } + if err := p.store.Save(); err != nil { + return err + } buf.steps = append(buf.steps, step) return p.writeSessionLocked(buf) } @@ -365,180 +375,6 @@ func isLoopbackHost(host string) bool { return ip != nil && ip.IsLoopback() } -func prepareVarsFile(path, fixtures string) error { - absVars, err := filepath.Abs(path) - if err != nil { - return fmt.Errorf("tide: vars path: %w", err) - } - absFix, err := filepath.Abs(fixtures) - if err != nil { - return fmt.Errorf("tide: fixtures path: %w", err) - } - if absVars == absFix || strings.HasPrefix(absVars, absFix+string(os.PathSeparator)) { - return fmt.Errorf("tide: vars file %q must be outside fixtures %q", path, fixtures) - } - if st, err := os.Stat(absVars); err == nil { - if st.IsDir() { - return fmt.Errorf("tide: vars %q is a directory", path) - } - if err := os.Chmod(absVars, 0o600); err != nil { - return fmt.Errorf("tide: chmod vars: %w", err) - } - return nil - } else if !os.IsNotExist(err) { - return fmt.Errorf("tide: stat vars: %w", err) - } - if err := os.MkdirAll(filepath.Dir(absVars), 0o755); err != nil { - return fmt.Errorf("tide: create vars dir: %w", err) - } - if err := os.WriteFile(absVars, []byte("{}\n"), 0o600); err != nil { - return fmt.Errorf("tide: create vars: %w", err) - } - return os.Chmod(absVars, 0o600) -} - func isTruncated(err error) bool { return err != nil && strings.Contains(err.Error(), "exceeds") } - -func rejectUnclassifiedCredentials(step Step) error { - var parts []string - for _, v := range step.Request.Headers { - parts = append(parts, v) - } - parts = append(parts, step.Request.Query, string(step.Request.Body)) - for _, v := range step.Response.Headers { - parts = append(parts, v) - } - parts = append(parts, string(step.Response.Body)) - classified := classifiedNames(step.Capture) - for _, part := range parts { - if hit := firstCredential(part); hit != "" && !classified[hit] { - return fmt.Errorf("unclassified credential-shaped value in step %s", step.ID) - } - } - return nil -} - -func classifiedNames(rules []CaptureRule) map[string]bool { - out := make(map[string]bool) - for _, rule := range rules { - if rule.Category != "" { - out[rule.Category] = true - } - out[rule.As] = true - } - return out -} - -func firstCredential(s string) string { - if s == "" { - return "" - } - if jwtRe.MatchString(s) { - return "jwt" - } - if invRe.MatchString(s) { - return "token" - } - lower := strings.ToLower(s) - if strings.Contains(lower, "auth_token=") { - return "cookie" - } - if strings.Contains(lower, "client_secret=") { - return "oauth_secret" - } - if strings.Contains(lower, "code_verifier=") { - return "pkce" - } - return "" -} - -var ( - jwtRe = mustCompileJWT() - invRe = mustCompileInv() -) - -func mustCompileJWT() *regexpJWT { - return ®expJWT{} -} - -func mustCompileInv() *regexpInv { - return ®expInv{} -} - -// tiny wrappers keep the credential regexes local without extra files in task 1. -type regexpJWT struct{} - -func (regexpJWT) MatchString(s string) bool { - return jwtLooksLike(s) -} - -type regexpInv struct{} - -func (regexpInv) MatchString(s string) bool { - return strings.Contains(s, "inv_") && invLooksLike(s) -} - -func jwtLooksLike(s string) bool { - const prefix = "eyJ" - for i := 0; i < len(s); i++ { - j := strings.Index(s[i:], prefix) - if j < 0 { - return false - } - i += j - if token := jwtAt(s[i:]); token != "" { - return true - } - i++ - } - return false -} - -func jwtAt(s string) string { - parts := 0 - n := 0 - for n < len(s) { - c := s[n] - if isJWTByte(c) { - n++ - continue - } - if c == '.' { - parts++ - n++ - if parts > 2 { - return "" - } - continue - } - break - } - if parts == 2 && n >= 20 { - return s[:n] - } - return "" -} - -func isJWTByte(c byte) bool { - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' -} - -func invLooksLike(s string) bool { - for { - i := strings.Index(s, "inv_") - if i < 0 { - return false - } - rest := s[i+4:] - n := 0 - for n < len(rest) && isJWTByte(rest[n]) { - n++ - } - if n >= 8 { - return true - } - s = rest - } -} diff --git a/tide/record.go b/tide/record.go index eb774af..22387c7 100644 --- a/tide/record.go +++ b/tide/record.go @@ -30,12 +30,27 @@ func RecordFlow(ctx context.Context, spec Flow, cfg RecordConfig) (Flow, error) out.Version = CurrentVersion out.Steps = make([]Step, len(spec.Steps)) copy(out.Steps, spec.Steps) - for i, step := range spec.Steps { - resp, err := doStep(ctx, client, cfg.Target, step.Request, limit) + for i := range out.Steps { + step := &out.Steps[i] + mergeRouteCaptures(step, cfg.Rules) + req, err := expandRequest(step.Request, cfg.Store) if err != nil { return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) } - out.Steps[i].Response = resp + resp, err := doStep(ctx, client, cfg.Target, req, limit) + if err != nil { + return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) + } + step.Response = resp + if err := CaptureStep(cfg.Store, step); err != nil { + return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) + } + if err := ScrubStep(cfg.Store, step); err != nil { + return Flow{}, fmt.Errorf("tide: record step %s: %w", step.ID, err) + } + if err := cfg.Store.Save(); err != nil { + return Flow{}, err + } } if err := validateFlow(out); err != nil { return Flow{}, err @@ -100,11 +115,24 @@ func joinURL(target, path, rawQuery string) (string, error) { } func keepResponseHeaders(h http.Header) map[string]string { - ct := h.Get("Content-Type") - if ct == "" { - return nil - } - return map[string]string{"Content-Type": ct} + return filterHeaders(h, recordedHeaderNames) +} + +var recordedHeaderNames = []string{ + "Content-Type", + "Location", + "Set-Cookie", + "Cache-Control", + "Pragma", + "WWW-Authenticate", + "Content-Disposition", + "Access-Control-Allow-Origin", + "Access-Control-Allow-Credentials", + "Access-Control-Allow-Headers", + "Access-Control-Allow-Methods", + "Access-Control-Expose-Headers", + "X-Total-Count", + "Link", } func readBounded(r io.Reader, max int64) ([]byte, error) { diff --git a/tide/replay.go b/tide/replay.go index 194c894..2d950ee 100644 --- a/tide/replay.go +++ b/tide/replay.go @@ -19,22 +19,44 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error client = defaultClient() } limit := maxBody(cfg.MaxBody) + store := cfg.Store + if store == nil { + store = mustMemoryStore() + } result := Result{OK: true, Steps: make([]StepResult, 0, len(flow.Steps))} + skipRest := false for _, step := range flow.Steps { - got, err := doStep(ctx, client, cfg.Target, step.Request, limit) + if skipRest { + result.Steps = append(result.Steps, StepResult{ID: step.ID, Skipped: true}) + continue + } + sr := StepResult{ID: step.ID, OK: true} + req, err := expandRequest(step.Request, store) + if err != nil { + sr.OK = false + result.OK = false + sr.Diffs = append(sr.Diffs, Diff{Path: "request", Expected: "resolved placeholders", Actual: err.Error()}) + result.Steps = append(result.Steps, sr) + skipRest = true + continue + } + got, err := doStep(ctx, client, cfg.Target, req, limit) if err != nil { return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err) } - sr := StepResult{ID: step.ID, OK: true} - if step.Response.Status != 0 && got.Status != step.Response.Status { + live := step + live.Request = req + live.Response = got + if err := CaptureStep(store, &live); err != nil { sr.OK = false - sr.Diffs = append(sr.Diffs, Diff{ - Path: "status", - Expected: fmt.Sprintf("%d", step.Response.Status), - Actual: fmt.Sprintf("%d", got.Status), - }) + result.OK = false + sr.Diffs = append(sr.Diffs, Diff{Path: "capture", Expected: "captured value", Actual: err.Error()}) + result.Steps = append(result.Steps, sr) + skipRest = true + continue } - sr.Diffs = append(sr.Diffs, compareBodies(step.Response, got)...) + _ = ScrubStep(store, &live) + sr.Diffs = append(sr.Diffs, compareStep(step, live.Response)...) if len(sr.Diffs) > 0 { sr.OK = false result.OK = false @@ -46,3 +68,22 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error } return result, nil } + +func mustMemoryStore() *Store { + s, _ := OpenStore("") + return s +} + +func compareStep(want Step, got Response) []Diff { + var diffs []Diff + if want.Response.Status != 0 && got.Status != want.Response.Status { + diffs = append(diffs, Diff{ + Path: "status", + Expected: fmt.Sprintf("%d", want.Response.Status), + Actual: fmt.Sprintf("%d", got.Status), + }) + } + diffs = append(diffs, compareHeaders(want.Response.Headers, got.Headers, want.Headers)...) + diffs = append(diffs, compareBodies(want.Response, got, want)...) + return diffs +} diff --git a/tide/variables.go b/tide/variables.go new file mode 100644 index 0000000..951a487 --- /dev/null +++ b/tide/variables.go @@ -0,0 +1,547 @@ +package tide + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + "github.com/goccy/go-yaml" +) + +var ( + placeholderRe = regexp.MustCompile(`\{\{([^{}]+)\}\}`) + jwtShapeRe = regexp.MustCompile(`eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+`) + invShapeRe = regexp.MustCompile(`inv_[A-Za-z0-9]{8,}`) + cookieRe = regexp.MustCompile(`(?i)auth_token=([^;]+)`) + secretFormRe = regexp.MustCompile(`(?i)client_secret=([^&\s]+)`) + pkceFormRe = regexp.MustCompile(`(?i)code_verifier=([^&\s]+)`) +) + +// Store holds named capture values. When Path is set it is a mode-0600 private file. +type Store struct { + mu sync.Mutex + path string + vals map[string]string +} + +// OpenStore loads or creates a private variable map. Empty path is memory-only. +func OpenStore(path string) (*Store, error) { + s := &Store{path: path, vals: make(map[string]string)} + if strings.TrimSpace(path) == "" { + return s, nil + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("tide: vars path: %w", err) + } + s.path = abs + st, err := os.Stat(abs) + if err == nil { + if st.IsDir() { + return nil, fmt.Errorf("tide: vars %q is a directory", path) + } + raw, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("tide: read vars: %w", err) + } + if len(strings.TrimSpace(string(raw))) > 0 { + if err := yaml.Unmarshal(raw, &s.vals); err != nil { + return nil, fmt.Errorf("tide: parse vars: %w", err) + } + if s.vals == nil { + s.vals = make(map[string]string) + } + } + if err := os.Chmod(abs, 0o600); err != nil { + return nil, fmt.Errorf("tide: chmod vars: %w", err) + } + return s, nil + } + if !os.IsNotExist(err) { + return nil, fmt.Errorf("tide: stat vars: %w", err) + } + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return nil, fmt.Errorf("tide: create vars dir: %w", err) + } + if err := os.WriteFile(abs, []byte("{}\n"), 0o600); err != nil { + return nil, fmt.Errorf("tide: create vars: %w", err) + } + _ = os.Chmod(abs, 0o600) + return s, nil +} + +// Save writes the map as YAML with mode 0600. Memory-only stores are a no-op. +func (s *Store) Save() error { + if s == nil || s.path == "" { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.vals)) + for k := range s.vals { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + if len(keys) == 0 { + b.WriteString("{}\n") + } + for _, k := range keys { + fmt.Fprintf(&b, "%s: %s\n", strconv.Quote(k), strconv.Quote(s.vals[k])) + } + raw := []byte(b.String()) + tmp, err := os.CreateTemp(filepath.Dir(s.path), ".vars-*.tmp") + if err != nil { + return fmt.Errorf("tide: vars temp: %w", err) + } + tmpName := tmp.Name() + if _, err := tmp.Write(raw); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + _ = tmp.Chmod(0o600) + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, s.path); err != nil { + _ = os.Remove(tmpName) + return err + } + return os.Chmod(s.path, 0o600) +} + +// Get returns a stored value. +func (s *Store) Get(name string) (string, bool) { + if s == nil { + return "", false + } + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.vals[name] + return v, ok +} + +// Set stores a named value. +func (s *Store) Set(name, value string) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.vals[name] = value +} + +// Expand replaces {{name}} placeholders. Unresolved names fail before HTTP send. +func (s *Store) Expand(text string) (string, error) { + if !strings.Contains(text, "{{") { + return text, nil + } + if s == nil { + m := placeholderRe.FindStringSubmatch(text) + if len(m) > 1 { + return "", fmt.Errorf("tide: unresolved placeholder %q", m[1]) + } + return "", fmt.Errorf("tide: unresolved placeholder") + } + var missing []string + s.mu.Lock() + out := placeholderRe.ReplaceAllStringFunc(text, func(m string) string { + name := m[2 : len(m)-2] + v, ok := s.vals[name] + if !ok { + missing = append(missing, name) + return m + } + return v + }) + s.mu.Unlock() + if len(missing) > 0 { + return "", fmt.Errorf("tide: unresolved placeholder %q", missing[0]) + } + return out, nil +} + +func expandRequest(req Request, store *Store) (Request, error) { + out := req + var err error + out.Path, err = store.Expand(req.Path) + if err != nil { + return Request{}, err + } + out.Query, err = store.Expand(req.Query) + if err != nil { + return Request{}, err + } + if req.Headers != nil { + out.Headers = make(map[string]string, len(req.Headers)) + for k, v := range req.Headers { + out.Headers[k], err = store.Expand(v) + if err != nil { + return Request{}, err + } + } + } + body, err := store.Expand(string(req.Body)) + if err != nil { + return Request{}, err + } + out.Body = Body(body) + return out, nil +} + +// CaptureStep writes named values from the step into the store. +func CaptureStep(store *Store, step *Step) error { + if store == nil || step == nil || len(step.Capture) == 0 { + return nil + } + for _, rule := range step.Capture { + val, err := extractCapture(rule, step.Request, step.Response) + if err != nil { + return fmt.Errorf("tide: capture %q: %w", rule.As, err) + } + if strings.TrimSpace(val) == "" { + return fmt.Errorf("tide: capture %q was empty", rule.As) + } + store.Set(rule.As, val) + } + return nil +} + +func extractCapture(rule CaptureRule, req Request, resp Response) (string, error) { + from := strings.TrimSpace(rule.From) + if from == "" { + from = "response.json" + } + switch from { + case "response.json": + v, err := jsonPathValue([]byte(resp.Body), rule.Path) + if err != nil { + return "", err + } + return scalarString(v), nil + case "response.header": + v := headerValue(resp.Headers, rule.Name) + if v == "" { + return "", fmt.Errorf("missing response header %s", rule.Name) + } + return v, nil + case "response.query": + loc := headerValue(resp.Headers, "Location") + if loc == "" { + return "", fmt.Errorf("missing Location header") + } + u, err := url.Parse(loc) + if err != nil { + return "", err + } + v := u.Query().Get(rule.Name) + if v == "" { + return "", fmt.Errorf("missing response query %s", rule.Name) + } + return v, nil + case "response.location.query": + loc := headerValue(resp.Headers, "Location") + if loc == "" { + return "", fmt.Errorf("missing Location header") + } + u, err := url.Parse(loc) + if err != nil { + return "", err + } + v := u.Query().Get(rule.Name) + if v == "" { + return "", fmt.Errorf("missing Location query %s", rule.Name) + } + return v, nil + case "request.form": + v, err := formValue(string(req.Body), rule.Name) + if err != nil { + return "", err + } + if v == "" { + return "", fmt.Errorf("missing form field %s", rule.Name) + } + return v, nil + case "request.header": + v := headerValue(req.Headers, rule.Name) + if v == "" { + return "", fmt.Errorf("missing request header %s", rule.Name) + } + return v, nil + case "request.query": + q, err := url.ParseQuery(req.Query) + if err != nil { + return "", err + } + v := q.Get(rule.Name) + if v == "" { + return "", fmt.Errorf("missing request query %s", rule.Name) + } + return v, nil + default: + return "", fmt.Errorf("unknown capture from %q", from) + } +} + +func formValue(body, name string) (string, error) { + vals, err := url.ParseQuery(body) + if err != nil { + return "", err + } + return vals.Get(name), nil +} + +func jsonPathValue(raw []byte, path string) (any, error) { + if strings.TrimSpace(path) == "" { + return nil, fmt.Errorf("json path is required") + } + root, err := decodeJSON(raw) + if err != nil { + return nil, err + } + v, err := walkJSONPath(root, path) + if err != nil { + return nil, err + } + return v, nil +} + +func walkJSONPath(root any, path string) (any, error) { + path = strings.TrimSpace(path) + if path == "$" || path == "" { + return root, nil + } + if !strings.HasPrefix(path, "$") { + path = "$." + path + } + cur := root + rest := strings.TrimPrefix(path, "$") + for rest != "" { + switch { + case strings.HasPrefix(rest, "."): + rest = rest[1:] + name, next := splitPathSeg(rest) + if name == "" { + return nil, fmt.Errorf("invalid json path %s", path) + } + obj, ok := cur.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s is not an object", path) + } + v, ok := obj[name] + if !ok { + return nil, fmt.Errorf("missing %s", "$."+name) + } + cur = v + rest = next + case strings.HasPrefix(rest, "["): + end := strings.IndexByte(rest, ']') + if end < 0 { + return nil, fmt.Errorf("invalid json path %s", path) + } + idx, err := atoi(rest[1:end]) + if err != nil { + return nil, err + } + arr, ok := cur.([]any) + if !ok || idx < 0 || idx >= len(arr) { + return nil, fmt.Errorf("missing %s[%d]", path, idx) + } + cur = arr[idx] + rest = rest[end+1:] + default: + return nil, fmt.Errorf("invalid json path %s", path) + } + } + return cur, nil +} + +func splitPathSeg(s string) (name, rest string) { + i := 0 + for i < len(s) && s[i] != '.' && s[i] != '[' { + i++ + } + return s[:i], s[i:] +} + +func atoi(s string) (int, error) { + n := 0 + if s == "" { + return 0, fmt.Errorf("empty index") + } + for _, c := range s { + if c < '0' || c > '9' { + return 0, fmt.Errorf("invalid index %q", s) + } + n = n*10 + int(c-'0') + } + return n, nil +} + +func scalarString(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + case json.Number: + return string(t) + case bool: + return fmt.Sprintf("%v", t) + default: + return fmt.Sprintf("%v", t) + } +} + +// ScrubStep replaces stored capture values with {{name}} in kept fields. +func ScrubStep(store *Store, step *Step) error { + if store == nil || step == nil { + return nil + } + pairs := store.replacements() + step.Request.Path = replaceAll(step.Request.Path, pairs) + step.Request.Query = replaceAll(step.Request.Query, pairs) + step.Request.Headers = scrubMap(step.Request.Headers, pairs) + step.Request.Body = Body(replaceAll(string(step.Request.Body), pairs)) + step.Response.Headers = scrubMap(step.Response.Headers, pairs) + step.Response.Body = Body(replaceAll(string(step.Response.Body), pairs)) + return rejectUnclassifiedCredentials(*step) +} + +func (s *Store) replacements() [][2]string { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + keys := make([]string, 0, len(s.vals)) + for k, v := range s.vals { + if v != "" { + keys = append(keys, k) + } + } + sort.Slice(keys, func(i, j int) bool { + return len(s.vals[keys[i]]) > len(s.vals[keys[j]]) + }) + out := make([][2]string, 0, len(keys)) + for _, k := range keys { + out = append(out, [2]string{s.vals[k], "{{" + k + "}}"}) + } + return out +} + +func scrubMap(in map[string]string, pairs [][2]string) map[string]string { + if in == nil { + return nil + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = replaceAll(v, pairs) + } + return out +} + +func replaceAll(s string, pairs [][2]string) string { + for _, p := range pairs { + if p[0] == "" { + continue + } + s = strings.ReplaceAll(s, p[0], p[1]) + } + return s +} + +func rejectUnclassifiedCredentials(step Step) error { + check := func(label, s string) error { + if hit := remainingCredential(s); hit != "" { + return fmt.Errorf("unclassified credential-shaped value (%s) in %s step %s", hit, label, step.ID) + } + return nil + } + for k, v := range step.Request.Headers { + if err := check("request header "+k, v); err != nil { + return err + } + } + if err := check("request query", step.Request.Query); err != nil { + return err + } + if err := check("request body", string(step.Request.Body)); err != nil { + return err + } + for k, v := range step.Response.Headers { + if err := check("response header "+k, v); err != nil { + return err + } + } + return check("response body", string(step.Response.Body)) +} + +func remainingCredential(s string) string { + s = placeholderRe.ReplaceAllString(s, "") + if s == "" { + return "" + } + if jwtShapeRe.MatchString(s) { + return "jwt" + } + if invShapeRe.MatchString(s) { + return "token" + } + if cookieRe.MatchString(s) { + return "cookie" + } + if secretFormRe.MatchString(s) { + return "oauth_secret" + } + if pkceFormRe.MatchString(s) { + return "pkce" + } + return "" +} + +func varsOutsideFixtures(varsPath, fixtures string) error { + if varsPath == "" || fixtures == "" { + return nil + } + absVars, err := filepath.Abs(varsPath) + if err != nil { + return fmt.Errorf("tide: vars path: %w", err) + } + absFix, err := filepath.Abs(fixtures) + if err != nil { + return fmt.Errorf("tide: fixtures path: %w", err) + } + if absVars == absFix || strings.HasPrefix(absVars, absFix+string(os.PathSeparator)) { + return fmt.Errorf("tide: vars file %q must be outside fixtures %q", varsPath, fixtures) + } + return nil +} + +func mergeRouteCaptures(step *Step, rules Rules) { + if step == nil || len(step.Capture) > 0 { + return + } + route := rules.Match(step.Request.Method, step.Request.Path) + if route != nil && len(route.Capture) > 0 { + step.Capture = append([]CaptureRule(nil), route.Capture...) + } +} + +func recordedResponseHeaders(h http.Header, rules Rules, method, path string) map[string]string { + route := rules.Match(method, path) + if keep := rules.responseHeaders(route); len(keep) > 0 { + return filterHeaders(h, keep) + } + return keepResponseHeaders(h) +}