diff --git a/cmd/summer/parity.go b/cmd/summer/parity.go index 740a81a..17c82a7 100644 --- a/cmd/summer/parity.go +++ b/cmd/summer/parity.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "path/filepath" "strings" "git.golem15.com/golem15/summercms/bonfire" @@ -35,6 +36,13 @@ func parityRecordCommand() bonfire.Command { {Name: "output", Description: "Destination fixture path"}, {Name: "rules", Description: "Committed YAML capture rules"}, {Name: "vars", Description: "Private mode-0600 variable store outside fixtures"}, + {Name: "manifest", Description: "Route manifest path"}, + {Name: "fixtures", Description: "Fixture directory for manifest recording"}, + {Name: "update", Description: "Overwrite existing fixtures (true/false)"}, + {Name: "next-batch", Description: "Max unrecorded routes to record (max 15)"}, + {Name: "resume", Description: "Skip valid existing fixtures (true/false)"}, + {Name: "allow-incomplete", Description: "Permit unfinished cases (true/false)"}, + {Name: "require-recorded", Description: "Require a fixture for every case (true/false)"}, }, Run: runParityRecord, } @@ -45,9 +53,12 @@ func parityReplayCommand() bonfire.Command { Name: "parity:replay", Description: "Replay recorded fixtures against an HTTP backend", Flags: []bonfire.Flag{ - {Name: "fixtures", Description: "Recorded YAML fixture path"}, + {Name: "fixtures", Description: "Recorded YAML fixture path or directory"}, {Name: "target", Description: "Base URL of the HTTP backend"}, {Name: "vars", Description: "Private mode-0600 variable store"}, + {Name: "manifest", Description: "Route manifest path"}, + {Name: "self-check", Description: "Fail on any recorded mismatch (true/false)"}, + {Name: "require-recorded", Description: "Fail unrecorded required cases (true/false)"}, }, Run: runParityReplay, } @@ -86,6 +97,9 @@ func runParityProxy(ctx context.Context, in bonfire.Input, out bonfire.Output) e } func runParityRecord(ctx context.Context, in bonfire.Input, out bonfire.Output) error { + if man, ok := in.Flag("manifest"); ok && strings.TrimSpace(man) != "" { + return runParityRecordManifest(ctx, in, out, strings.TrimSpace(man)) + } specPath, err := requireFlag(in, "spec", "parity:record") if err != nil { return err @@ -103,19 +117,8 @@ func runParityRecord(ctx context.Context, in bonfire.Input, out bonfire.Output) return err } cfg := tide.RecordConfig{Target: target} - if rulesPath, ok := in.Flag("rules"); ok && strings.TrimSpace(rulesPath) != "" { - rules, err := tide.LoadRules(rulesPath) - if err != nil { - return err - } - cfg.Rules = rules - } - if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { - store, err := tide.OpenStore(varsPath) - if err != nil { - return err - } - cfg.Store = store + if err := attachRulesAndVars(in, &cfg); err != nil { + return err } flow, err := tide.RecordFlow(ctx, spec, cfg) if err != nil { @@ -128,20 +131,71 @@ func runParityRecord(ctx context.Context, in bonfire.Input, out bonfire.Output) return nil } -func runParityReplay(ctx context.Context, in bonfire.Input, out bonfire.Output) error { - path, err := requireFlag(in, "fixtures", "parity:replay") +func runParityRecordManifest(ctx context.Context, in bonfire.Input, out bonfire.Output, manPath string) error { + target, err := requireFlag(in, "target", "parity:record") if err != nil { return err } + fixtures, err := requireFlag(in, "fixtures", "parity:record") + if err != nil { + return err + } + m, err := tide.LoadManifest(manPath) + if err != nil { + return err + } + mode := tide.ModeAllowIncomplete + if flagBool(in, "require-recorded") { + mode = tide.ModeRequireRecorded + } + if err := tide.ValidateManifest(m, fixtures, mode); err != nil && mode == tide.ModeRequireRecorded { + return err + } + if err := tide.ValidateManifest(m, fixtures, tide.ModeAllowIncomplete); err != nil { + return err + } + batch, err := tide.ParseNextBatch(flagValue(in, "next-batch")) + if err != nil { + return err + } + cfg := tide.ManifestConfig{ + Target: target, + Fixtures: fixtures, + Update: flagBool(in, "update"), + Resume: flagBool(in, "resume"), + NextBatch: batch, + Mode: mode, + } + if err := attachManifestStore(in, &cfg); err != nil { + return err + } + cov, err := tide.RecordManifest(ctx, m, cfg) + if err != nil { + return err + } + out.Table(tide.CoverageHeaders(), cov.CoverageRows()) + out.Success(cov.SummaryLine()) + out.Info(cov.ResumeLine()) + return nil +} + +func runParityReplay(ctx context.Context, in bonfire.Input, out bonfire.Output) error { target, err := requireFlag(in, "target", "parity:replay") if err != nil { return err } + if man, ok := in.Flag("manifest"); ok && strings.TrimSpace(man) != "" { + return runParityReplayManifest(ctx, in, out, strings.TrimSpace(man), target) + } + path, err := requireFlag(in, "fixtures", "parity:replay") + if err != nil { + return err + } flow, err := tide.LoadFlow(path) if err != nil { return err } - cfg := tide.ReplayConfig{Target: target} + cfg := tide.ReplayConfig{Target: target, BaseDir: filepath.Dir(path)} if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { store, err := tide.OpenStore(varsPath) if err != nil { @@ -159,6 +213,90 @@ func runParityReplay(ctx context.Context, in bonfire.Input, out bonfire.Output) return nil } +func runParityReplayManifest(ctx context.Context, in bonfire.Input, out bonfire.Output, manPath, target string) error { + fixtures, err := requireFlag(in, "fixtures", "parity:replay") + if err != nil { + return err + } + m, err := tide.LoadManifest(manPath) + if err != nil { + return err + } + cfg := tide.ManifestConfig{ + Target: target, + Fixtures: fixtures, + SelfCheck: flagBool(in, "self-check"), + Mode: tide.ModeAllowIncomplete, + } + if flagBool(in, "require-recorded") { + cfg.Mode = tide.ModeRequireRecorded + } + if err := attachManifestStore(in, &cfg); err != nil { + return err + } + cov, err := tide.ReplayManifest(ctx, m, cfg) + for _, d := range cov.Diffs { + out.Error(d) + } + out.Table(tide.CoverageHeaders(), cov.CoverageRows()) + out.Info(cov.SummaryLine()) + return err +} + +func attachRulesAndVars(in bonfire.Input, cfg *tide.RecordConfig) error { + if rulesPath, ok := in.Flag("rules"); ok && strings.TrimSpace(rulesPath) != "" { + rules, err := tide.LoadRules(rulesPath) + if err != nil { + return err + } + cfg.Rules = rules + } + if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { + store, err := tide.OpenStore(varsPath) + if err != nil { + return err + } + cfg.Store = store + } + return nil +} + +func attachManifestStore(in bonfire.Input, cfg *tide.ManifestConfig) error { + if rulesPath, ok := in.Flag("rules"); ok && strings.TrimSpace(rulesPath) != "" { + rules, err := tide.LoadRules(rulesPath) + if err != nil { + return err + } + cfg.Rules = rules + } + if varsPath, ok := in.Flag("vars"); ok && strings.TrimSpace(varsPath) != "" { + store, err := tide.OpenStore(varsPath) + if err != nil { + return err + } + cfg.Store = store + } + return nil +} + +func flagBool(in bonfire.Input, name string) bool { + v, ok := in.Flag(name) + if !ok { + return false + } + switch strings.ToLower(strings.TrimSpace(v)) { + case "", "true", "1", "yes": + return true + default: + return false + } +} + +func flagValue(in bonfire.Input, name string) string { + v, _ := in.Flag(name) + return strings.TrimSpace(v) +} + func requireFlag(in bonfire.Input, name, cmd string) (string, error) { v, ok := in.Flag(name) v = strings.TrimSpace(v) diff --git a/cmd/summer/parity_test.go b/cmd/summer/parity_test.go index e06ad9e..9d783b9 100644 --- a/cmd/summer/parity_test.go +++ b/cmd/summer/parity_test.go @@ -116,6 +116,22 @@ func TestParityCommands(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "loopback") { t.Fatalf("proxy non-loopback listen: %v", err) } + + manDir := t.TempDir() + fx := filepath.Join(manDir, "fx") + man := filepath.Join(manDir, "manifest.yaml") + if err := os.WriteFile(man, []byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: GET /cli public\n method: GET\n path: /sample\n auth_group: public\n status: ported\n cases:\n - id: ok\n status: 200\n fixture: routes/cli.yaml\n request:\n method: GET\n path: /sample\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := runParity("parity:record", "--manifest", man, "--target", origJSON.URL, "--fixtures", fx, "--resume", "true", "--next-batch", "15"); err != nil { + t.Fatalf("manifest record: %v", err) + } + if err := runParity("parity:record", "--manifest", man, "--target", origJSON.URL, "--fixtures", fx, "--next-batch", "16"); err == nil { + t.Fatal("batch 16 must fail") + } + if err := runParity("parity:replay", "--manifest", man, "--fixtures", fx, "--target", origJSON.URL); err != nil { + t.Fatalf("manifest replay: %v", err) + } } func commandNames() []string { diff --git a/tide/flow.go b/tide/flow.go index 6a958a3..7143734 100644 --- a/tide/flow.go +++ b/tide/flow.go @@ -1,8 +1,11 @@ package tide import ( + "crypto/sha256" + "encoding/hex" "fmt" "net/http" + "os" "path/filepath" "strings" ) @@ -84,6 +87,7 @@ type ReplayConfig struct { Client *http.Client MaxBody int64 Store *Store + BaseDir string } // Result is the outcome of replaying a flow. @@ -169,6 +173,30 @@ func validateFlow(flow Flow) error { return nil } +func materializeSidecar(base string, resp *Response) error { + if resp == nil || resp.BodyFile == "" { + return nil + } + if err := validateSidecar(resp.BodyFile); err != nil { + return err + } + path := resp.BodyFile + if base != "" { + path = filepath.Join(base, resp.BodyFile) + } + raw, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("tide: read body_file %s: %w", resp.BodyFile, err) + } + sum := sha256.Sum256(raw) + got := hex.EncodeToString(sum[:]) + if resp.SHA256 != "" && !strings.EqualFold(got, resp.SHA256) { + return fmt.Errorf("tide: body_file %s digest mismatch", resp.BodyFile) + } + resp.Body = Body(raw) + return nil +} + func validateSidecar(path string) error { if path == "" { return nil diff --git a/tide/manifest.go b/tide/manifest.go new file mode 100644 index 0000000..a673f1f --- /dev/null +++ b/tide/manifest.go @@ -0,0 +1,523 @@ +package tide + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/goccy/go-yaml" +) + +const MaxBatch = 15 + +const ( + StatusPending = "pending" + StatusPorted = "ported" +) + +const ( + ModeAllowIncomplete = "allow-incomplete" + ModeRequireRecorded = "require-recorded" +) + +// Manifest is a generic ordered list of route cases. It has no app-specific names. +type Manifest struct { + Version int `yaml:"version"` + AuthGroups []string `yaml:"auth_groups"` + Seed *Seed `yaml:"seed,omitempty"` + Routes []Route `yaml:"routes"` +} + +// Seed names an optional bootstrap flow recorded before route cases. +type Seed struct { + Hook string `yaml:"hook,omitempty"` + Spec string `yaml:"spec,omitempty"` + Fixture string `yaml:"fixture,omitempty"` +} + +// Route is one manifest entry with identity, cases and fixture path. +type Route struct { + ID string `yaml:"id"` + Method string `yaml:"method"` + Path string `yaml:"path"` + AuthGroup string `yaml:"auth_group"` + Status string `yaml:"status"` + Identities []string `yaml:"identities,omitempty"` + Cases []RouteCase `yaml:"cases,omitempty"` + Headers map[string]string `yaml:"headers,omitempty"` + Normalize []NormalizeRule `yaml:"normalize,omitempty"` + SeedHook string `yaml:"seed_hook,omitempty"` + Fixture string `yaml:"fixture,omitempty"` +} + +// RouteCase is one recorded request/response expectation for a route. +type RouteCase struct { + ID string `yaml:"id"` + Identity string `yaml:"identity,omitempty"` + Status int `yaml:"status,omitempty"` + Headers map[string]string `yaml:"headers,omitempty"` + Fixture string `yaml:"fixture,omitempty"` + Request *Request `yaml:"request,omitempty"` +} + +// ManifestConfig drives recording or replay of a manifest. +type ManifestConfig struct { + Target string + Fixtures string + Store *Store + Rules Rules + Update bool + Resume bool + NextBatch int + Mode string + SelfCheck bool +} + +// LoadManifest reads a strict YAML manifest. +func LoadManifest(path string) (Manifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return Manifest{}, fmt.Errorf("tide: read manifest %s: %w", path, err) + } + return ParseManifest(raw) +} + +// ParseManifest decodes a manifest, rejecting unknown fields. +func ParseManifest(raw []byte) (Manifest, error) { + var m Manifest + dec := yaml.NewDecoder(bytes.NewReader(raw), yaml.DisallowUnknownField()) + if err := dec.Decode(&m); err != nil { + return Manifest{}, fmt.Errorf("tide: parse manifest: %w", err) + } + if m.Version != CurrentVersion { + return Manifest{}, fmt.Errorf("tide: unsupported manifest version %d", m.Version) + } + return m, nil +} + +// ValidateManifest checks identities in both modes; case/fixture completeness depends on mode. +func ValidateManifest(m Manifest, fixtures, mode string) error { + if len(m.AuthGroups) == 0 { + return fmt.Errorf("tide: manifest auth_groups is required") + } + groups := make(map[string]struct{}, len(m.AuthGroups)) + for _, g := range m.AuthGroups { + if strings.TrimSpace(g) == "" { + return fmt.Errorf("tide: empty auth group") + } + groups[g] = struct{}{} + } + if len(m.Routes) == 0 { + return fmt.Errorf("tide: manifest has no routes") + } + seen := map[string]struct{}{} + for i, route := range m.Routes { + if strings.TrimSpace(route.ID) == "" { + return fmt.Errorf("tide: routes[%d] is missing id", i) + } + if _, dup := seen[route.ID]; dup { + return fmt.Errorf("tide: duplicate route id %q", route.ID) + } + seen[route.ID] = struct{}{} + if strings.TrimSpace(route.Method) == "" || strings.TrimSpace(route.Path) == "" { + return fmt.Errorf("tide: route %s is missing method or path", route.ID) + } + if _, ok := groups[route.AuthGroup]; !ok { + return fmt.Errorf("tide: route %s has unknown auth group %q", route.ID, route.AuthGroup) + } + if route.Status != StatusPending && route.Status != StatusPorted { + return fmt.Errorf("tide: route %s has unknown status %q", route.ID, route.Status) + } + if err := validateFixturePath(route.Fixture); err != nil { + return fmt.Errorf("tide: route %s: %w", route.ID, err) + } + for j, c := range route.Cases { + if err := validateFixturePath(c.Fixture); err != nil { + return fmt.Errorf("tide: route %s case[%d]: %w", route.ID, j, err) + } + if mode == ModeRequireRecorded { + if err := requireCase(route, c); err != nil { + return err + } + } + } + if mode == ModeRequireRecorded { + if len(route.Cases) == 0 { + return fmt.Errorf("tide: route %s is missing cases", route.ID) + } + for _, c := range routeCases(route) { + path := caseFixturePath(fixtures, route, c) + if _, err := LoadFlow(path); err != nil { + return fmt.Errorf("tide: route %s missing valid fixture %s: %w", route.ID, path, err) + } + } + } + } + return nil +} + +func requireCase(route Route, c RouteCase) error { + if strings.TrimSpace(c.ID) == "" { + return fmt.Errorf("tide: route %s has a case without id", route.ID) + } + if c.Status == 0 { + return fmt.Errorf("tide: route %s case %s is missing status", route.ID, c.ID) + } + req := caseRequest(route, c) + if strings.TrimSpace(req.Method) == "" || strings.TrimSpace(req.Path) == "" { + return fmt.Errorf("tide: route %s case %s is missing request", route.ID, c.ID) + } + if caseFixturePath("", route, c) == "" { + return fmt.Errorf("tide: route %s case %s is missing fixture", route.ID, c.ID) + } + return nil +} + +func validateFixturePath(p string) error { + if p == "" { + return nil + } + if filepath.IsAbs(p) { + return fmt.Errorf("fixture %q must be a relative path", p) + } + clean := filepath.ToSlash(filepath.Clean(p)) + if clean == ".." || strings.HasPrefix(clean, "../") { + return fmt.Errorf("fixture %q escapes the fixture directory", p) + } + return nil +} + +func routeCases(route Route) []RouteCase { + if len(route.Cases) > 0 { + return route.Cases + } + return nil +} + +func caseRequest(route Route, c RouteCase) Request { + if c.Request != nil { + return *c.Request + } + return Request{Method: route.Method, Path: route.Path} +} + +func caseFixturePath(root string, route Route, c RouteCase) string { + rel := c.Fixture + if rel == "" { + rel = route.Fixture + } + if rel == "" && c.ID != "" { + rel = filepath.ToSlash(filepath.Join("routes", sanitizeFile(route.ID)+"__"+sanitizeFile(c.ID)+".yaml")) + } + if rel == "" && route.ID != "" { + rel = filepath.ToSlash(filepath.Join("routes", sanitizeFile(route.ID)+".yaml")) + } + if root == "" { + return rel + } + return filepath.Join(root, rel) +} + +func sanitizeFile(s string) string { + s = strings.TrimSpace(s) + repl := strings.NewReplacer("/", "_", "\\", "_", " ", "_") + return repl.Replace(s) +} + +func fixtureHash(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + if len(bytes.TrimSpace(raw)) == 0 { + return "", fmt.Errorf("empty fixture") + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +func recordedCase(fixtures string, route Route, c RouteCase) (bool, error) { + path := caseFixturePath(fixtures, route, c) + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + if _, err := LoadFlow(path); err != nil { + return false, fmt.Errorf("tide: existing fixture %s is invalid: %w", path, err) + } + if _, err := fixtureHash(path); err != nil { + return false, fmt.Errorf("tide: existing fixture %s hash: %w", path, err) + } + return true, nil +} + +// RecordManifest records seed then route cases. NextBatch>15 is refused. +func RecordManifest(ctx context.Context, m Manifest, cfg ManifestConfig) (Coverage, error) { + if cfg.NextBatch > MaxBatch { + return Coverage{}, fmt.Errorf("tide: --next-batch %d exceeds %d", cfg.NextBatch, MaxBatch) + } + if err := ValidateManifest(m, cfg.Fixtures, ModeAllowIncomplete); err != nil { + return Coverage{}, err + } + cov := newCoverage(m) + if m.Seed != nil && strings.TrimSpace(m.Seed.Spec) != "" { + if err := recordSeed(ctx, m.Seed, cfg); err != nil { + return cov, err + } + } + limit := cfg.NextBatch + if limit <= 0 { + limit = len(m.Routes) + } + taken := 0 + for _, route := range m.Routes { + cases := route.Cases + if len(cases) == 0 { + cov.markUnrecorded(route) + continue + } + allRecorded := true + for _, c := range cases { + ok, err := recordedCase(cfg.Fixtures, route, c) + if err != nil { + return cov, err + } + if !ok { + allRecorded = false + break + } + } + if allRecorded { + cov.markRecorded(route) + continue + } + if taken >= limit { + cov.markUnrecorded(route) + continue + } + if err := recordRoute(ctx, route, cfg); err != nil { + return cov, err + } + cov.markRecorded(route) + taken++ + } + cov.ResumeRemaining = countUnrecorded(m, cfg.Fixtures) + return cov, nil +} + +func recordSeed(ctx context.Context, seed *Seed, cfg ManifestConfig) error { + spec, err := LoadFlow(seed.Spec) + if err != nil { + return err + } + flow, err := RecordFlow(ctx, spec, RecordConfig{Target: cfg.Target, Store: cfg.Store, Rules: cfg.Rules}) + if err != nil { + return err + } + dest := seed.Fixture + if dest == "" { + dest = seed.Spec + } + if cfg.Fixtures != "" && !filepath.IsAbs(dest) { + dest = filepath.Join(cfg.Fixtures, dest) + } + if _, err := os.Stat(dest); err == nil && !cfg.Update { + return nil + } + return SaveFlow(dest, flow) +} + +func recordRoute(ctx context.Context, route Route, cfg ManifestConfig) error { + for _, c := range route.Cases { + dest := caseFixturePath(cfg.Fixtures, route, c) + if _, err := os.Stat(dest); err == nil { + if cfg.Resume && !cfg.Update { + if _, err := LoadFlow(dest); err != nil { + return err + } + continue + } + if !cfg.Update { + return fmt.Errorf("tide: fixture %s exists; pass --update to overwrite", dest) + } + } + spec := Flow{ + Version: CurrentVersion, + Name: route.ID + "/" + c.ID, + SeedHook: route.SeedHook, + Steps: []Step{{ + ID: c.ID, + RouteID: route.ID, + Request: caseRequest(route, c), + Normalize: route.Normalize, + Headers: mergeHeaders(route.Headers, c.Headers), + }}, + } + flow, err := RecordFlow(ctx, spec, RecordConfig{Target: cfg.Target, Store: cfg.Store, Rules: cfg.Rules}) + if err != nil { + return err + } + if err := SaveFlow(dest, flow); err != nil { + return err + } + } + return nil +} + +func mergeHeaders(a, b map[string]string) map[string]string { + if len(a) == 0 && len(b) == 0 { + return nil + } + out := make(map[string]string) + for k, v := range a { + out[k] = v + } + for k, v := range b { + out[k] = v + } + return out +} + +func countUnrecorded(m Manifest, fixtures string) int { + n := 0 + for _, route := range m.Routes { + if len(route.Cases) == 0 { + n++ + continue + } + for _, c := range route.Cases { + ok, err := recordedCase(fixtures, route, c) + if err != nil || !ok { + n++ + break + } + } + } + return n +} + +// ReplayManifest replays recorded route fixtures and builds a coverage table. +func ReplayManifest(ctx context.Context, m Manifest, cfg ManifestConfig) (Coverage, error) { + if err := ValidateManifest(m, cfg.Fixtures, ModeAllowIncomplete); err != nil { + return Coverage{}, err + } + cov := newCoverage(m) + var fail error + for _, route := range m.Routes { + cases := route.Cases + if len(cases) == 0 { + cov.markUnrecorded(route) + if cfg.Mode == ModeRequireRecorded { + fail = firstErr(fail, fmt.Errorf("tide: unrecorded required route %s", route.ID)) + } + continue + } + routeFail := false + routeRecorded := true + for _, c := range cases { + path := caseFixturePath(cfg.Fixtures, route, c) + flow, err := LoadFlow(path) + if err != nil { + routeRecorded = false + if cfg.Mode == ModeRequireRecorded { + fail = firstErr(fail, fmt.Errorf("tide: unrecorded required route %s", route.ID)) + } + continue + } + res, err := ReplayFlow(ctx, flow, ReplayConfig{Target: cfg.Target, Store: cfg.Store, BaseDir: filepath.Dir(path)}) + if err != nil { + var mis *MismatchError + if !errors.As(err, &mis) { + return cov, err + } + res = mis.Result + } + if !res.OK { + routeFail = true + cov.addDiffs(route.ID, res) + if cfg.SelfCheck || route.Status == StatusPorted { + fail = firstErr(fail, fmt.Errorf("tide: route %s failed", route.ID)) + } + } + } + if !routeRecorded { + cov.markUnrecorded(route) + continue + } + if routeFail { + cov.markFailing(route) + continue + } + cov.markPassing(route) + } + return cov, fail +} + +func firstErr(cur, next error) error { + if cur == nil { + return next + } + return cur +} + +func newCoverage(m Manifest) Coverage { + c := Coverage{Total: len(m.Routes), Rows: make([]CoverageRow, 0, len(m.Routes))} + for _, route := range m.Routes { + if route.Status == StatusPending { + c.Pending++ + } else if route.Status == StatusPorted { + c.Ported++ + } + } + return c +} + +func (c *Coverage) markRecorded(route Route) { + c.Recorded++ + c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "recorded"}) +} + +func (c *Coverage) markPassing(route Route) { + c.Recorded++ + c.Passing++ + c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "passing"}) +} + +func (c *Coverage) markFailing(route Route) { + c.Recorded++ + c.Failing++ + c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "failing"}) +} + +func (c *Coverage) markUnrecorded(route Route) { + c.Unrecorded++ + c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "unrecorded"}) +} + +func (c *Coverage) addDiffs(id string, res Result) { + for _, sr := range res.Steps { + for _, d := range sr.Diffs { + c.Diffs = append(c.Diffs, fmt.Sprintf("%s %s: %s expected %s actual %s", id, sr.ID, d.Path, d.Expected, d.Actual)) + } + } +} + +func ParseNextBatch(s string) (int, error) { + if strings.TrimSpace(s) == "" { + return 0, nil + } + n, err := strconv.Atoi(s) + if err != nil { + return 0, fmt.Errorf("tide: --next-batch: %w", err) + } + return n, nil +} diff --git a/tide/manifest_test.go b/tide/manifest_test.go new file mode 100644 index 0000000..6ed0fa6 --- /dev/null +++ b/tide/manifest_test.go @@ -0,0 +1,367 @@ +package tide + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestManifestValidationAndCoverage(t *testing.T) { + dir := t.TempDir() + fixtures := filepath.Join(dir, "fixtures") + if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil { + t.Fatal(err) + } + manPath := filepath.Join(dir, "manifest.yaml") + manYAML := "" + + "version: 1\n" + + "auth_groups:\n" + + " - public\n" + + " - session\n" + + "routes:\n" + + " - id: GET /ok public\n" + + " method: GET\n" + + " path: /ok\n" + + " auth_group: public\n" + + " status: ported\n" + + " cases:\n" + + " - id: ok\n" + + " status: 200\n" + + " fixture: routes/ok.yaml\n" + + " request:\n" + + " method: GET\n" + + " path: /ok\n" + + " - id: GET /missing public\n" + + " method: GET\n" + + " path: /missing\n" + + " auth_group: public\n" + + " status: pending\n" + + " cases:\n" + + " - id: missing\n" + + " status: 200\n" + + " fixture: routes/missing.yaml\n" + + " request:\n" + + " method: GET\n" + + " path: /missing\n" + if err := os.WriteFile(manPath, []byte(manYAML), 0o644); err != nil { + t.Fatal(err) + } + m, err := LoadManifest(manPath) + if err != nil { + t.Fatal(err) + } + if err := ValidateManifest(m, fixtures, ModeAllowIncomplete); err != nil { + t.Fatal(err) + } + if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err == nil { + t.Fatal("require-recorded must fail with missing fixtures") + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + + cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Resume: true}) + if err != nil { + t.Fatal(err) + } + if cov.Recorded != 2 || cov.Unrecorded != 0 { + t.Fatalf("record coverage %+v", cov) + } + if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err != nil { + t.Fatal(err) + } +} + +func TestCoverageTwoRouteReport(t *testing.T) { + dir := t.TempDir() + fixtures := filepath.Join(dir, "fx") + if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/pass": + _, _ = w.Write([]byte(`{"v":1}`)) + default: + _, _ = w.Write([]byte(`{"v":2}`)) + } + })) + t.Cleanup(srv.Close) + + writeRouteFixture(t, srv.URL, fixtures, "pass", "/pass", `{"v":1}`) + writeRouteFixture(t, srv.URL, fixtures, "fail", "/fail", `{"v":1}`) + + m := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{ + routeEntry("GET /pass public", "/pass", "ported", "routes/pass.yaml"), + routeEntry("GET /fail public", "/fail", "ported", "routes/fail.yaml"), + {ID: "GET /none public", Method: "GET", Path: "/none", AuthGroup: "public", Status: StatusPending}, + }, + } + cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Mode: ModeRequireRecorded}) + if err == nil { + t.Fatal("ported fail + unrecorded required must be nonzero") + } + if cov.Passing != 1 || cov.Failing != 1 || cov.Unrecorded != 1 || cov.Recorded != 2 { + t.Fatalf("coverage %+v", cov) + } + if !strings.Contains(cov.SummaryLine(), "passing 1") { + t.Fatalf("summary %s", cov.SummaryLine()) + } +} + +func TestManifestResumeBatches(t *testing.T) { + dir := t.TempDir() + fixtures := filepath.Join(dir, "fx") + hits := map[string]int{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits[r.URL.Path]++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + + m := Manifest{Version: 1, AuthGroups: []string{"public"}} + for i := 1; i <= 16; i++ { + id := fmt.Sprintf("GET /r/%d public", i) + path := fmt.Sprintf("/r/%d", i) + m.Routes = append(m.Routes, Route{ + ID: id, + Method: http.MethodGet, + Path: path, + AuthGroup: "public", + Status: StatusPending, + Cases: []RouteCase{{ + ID: "ok", + Status: 200, + Fixture: fmt.Sprintf("routes/r%d.yaml", i), + Request: &Request{Method: http.MethodGet, Path: path}, + }}, + }) + } + if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 16}); err == nil { + t.Fatal("batch >15 must fail") + } + cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true}) + if err != nil { + t.Fatal(err) + } + if cov.Recorded != 15 || cov.ResumeRemaining != 1 { + t.Fatalf("first batch %+v remaining %d", cov, cov.ResumeRemaining) + } + cov, err = RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true}) + if err != nil { + t.Fatal(err) + } + if cov.Recorded != 16 || cov.ResumeRemaining != 0 { + t.Fatalf("second batch %+v remaining %d", cov, cov.ResumeRemaining) + } + for i := 1; i <= 16; i++ { + if hits[fmt.Sprintf("/r/%d", i)] != 1 { + t.Fatalf("route %d recaptured: %d", i, hits[fmt.Sprintf("/r/%d", i)]) + } + } +} + +func TestManifestAllowIncomplete154(t *testing.T) { + var b strings.Builder + b.WriteString("version: 1\nauth_groups:\n - public\nroutes:\n") + for i := 1; i <= 154; i++ { + fmt.Fprintf(&b, " - id: GET /r/%d public\n method: GET\n path: /r/%d\n auth_group: public\n status: pending\n", i, i) + } + m, err := ParseManifest([]byte(b.String())) + if err != nil { + t.Fatal(err) + } + if len(m.Routes) != 154 { + t.Fatalf("routes %d", len(m.Routes)) + } + if err := ValidateManifest(m, t.TempDir(), ModeAllowIncomplete); err != nil { + t.Fatal(err) + } + if err := ValidateManifest(m, t.TempDir(), ModeRequireRecorded); err == nil { + t.Fatal("154 empty cases must fail require-recorded") + } +} + +func TestManifestSidecarRejectAndDigest(t *testing.T) { + dir := t.TempDir() + base := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n body_file: ../secret.bin\n" + p := filepath.Join(dir, "bad.yaml") + if err := os.WriteFile(p, []byte(base), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadFlow(p); err == nil { + t.Fatal("parent sidecar must fail") + } + good := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n status: 200\n headers:\n Content-Type: application/octet-stream\n body_file: data.bin\n sha256: deadbeef\n" + gp := filepath.Join(dir, "good.yaml") + if err := os.WriteFile(gp, []byte(good), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data.bin"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + flow, err := LoadFlow(gp) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte("hello")) + })) + t.Cleanup(srv.Close) + _, err = ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir}) + if err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("digest mismatch: %v", err) + } + sum := sha256.Sum256([]byte("hello")) + good2 := strings.Replace(good, "deadbeef", hex.EncodeToString(sum[:]), 1) + if err := os.WriteFile(gp, []byte(good2), 0o644); err != nil { + t.Fatal(err) + } + flow, err = LoadFlow(gp) + if err != nil { + t.Fatal(err) + } + if _, err := ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir}); err != nil { + t.Fatal(err) + } +} + +func TestManifestUnknownGroupAndDuplicate(t *testing.T) { + m, err := ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: oauth\n status: pending\n")) + if err != nil { + t.Fatal(err) + } + if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "auth group") { + t.Fatalf("unknown group: %v", err) + } + m, err = ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: public\n status: pending\n - id: a\n method: GET\n path: /b\n auth_group: public\n status: pending\n")) + if err != nil { + t.Fatal(err) + } + if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate: %v", err) + } +} + +func routeEntry(id, path, status, fixture string) Route { + return Route{ + ID: id, + Method: http.MethodGet, + Path: path, + AuthGroup: "public", + Status: status, + Fixture: fixture, + Cases: []RouteCase{{ + ID: "ok", + Status: 200, + Fixture: fixture, + Request: &Request{Method: http.MethodGet, Path: path}, + }}, + } +} + +func writeRouteFixture(t *testing.T, target, fixtures, name, path, body string) { + t.Helper() + spec := Flow{Version: 1, Name: name, Steps: []Step{{ + ID: "ok", + Request: Request{Method: http.MethodGet, Path: path}, + }}} + orig := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(orig.Close) + rec, err := RecordFlow(context.Background(), spec, RecordConfig{Target: orig.URL}) + if err != nil { + t.Fatal(err) + } + if err := SaveFlow(filepath.Join(fixtures, "routes", name+".yaml"), rec); err != nil { + t.Fatal(err) + } +} + +func TestManifestNoOverwriteWithoutUpdate(t *testing.T) { + fixtures := t.TempDir() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + m := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{{ + ID: "GET /a public", + Method: http.MethodGet, + Path: "/a", + AuthGroup: "public", + Status: StatusPending, + Cases: []RouteCase{ + {ID: "one", Status: 200, Fixture: "routes/one.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}}, + {ID: "two", Status: 200, Fixture: "routes/two.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}}, + }, + }}, + } + if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil { + t.Fatal(err) + } + first := Flow{Version: 1, Name: "one", Steps: []Step{{ID: "one", Request: Request{Method: http.MethodGet, Path: "/a"}}}} + rec, err := RecordFlow(context.Background(), first, RecordConfig{Target: srv.URL}) + if err != nil { + t.Fatal(err) + } + if err := SaveFlow(filepath.Join(fixtures, "routes", "one.yaml"), rec); err != nil { + t.Fatal(err) + } + if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures}); err == nil || !strings.Contains(err.Error(), "--update") { + t.Fatalf("must refuse overwrite: %v", err) + } + if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Update: true}); err != nil { + t.Fatal(err) + } +} + +func TestCoveragePendingMismatchDoesNotFail(t *testing.T) { + fixtures := t.TempDir() + if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil { + t.Fatal(err) + } + writeRouteFixture(t, "", fixtures, "pend", "/pend", `{"v":1}`) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"v":9}`)) + })) + t.Cleanup(srv.Close) + m := Manifest{ + Version: 1, + AuthGroups: []string{"public"}, + Routes: []Route{routeEntry("GET /pend public", "/pend", StatusPending, "routes/pend.yaml")}, + } + cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures}) + if err != nil { + t.Fatalf("pending mismatch must not fail: %v", err) + } + if cov.Failing != 1 { + t.Fatalf("failing %d", cov.Failing) + } + if _, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, SelfCheck: true}); err == nil { + t.Fatal("self-check must fail pending mismatches") + } +} + diff --git a/tide/replay.go b/tide/replay.go index 2d950ee..5ef8524 100644 --- a/tide/replay.go +++ b/tide/replay.go @@ -31,6 +31,10 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error continue } sr := StepResult{ID: step.ID, OK: true} + want := step + if err := materializeSidecar(cfg.BaseDir, &want.Response); err != nil { + return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err) + } req, err := expandRequest(step.Request, store) if err != nil { sr.OK = false @@ -56,7 +60,7 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error continue } _ = ScrubStep(store, &live) - sr.Diffs = append(sr.Diffs, compareStep(step, live.Response)...) + sr.Diffs = append(sr.Diffs, compareStep(want, live.Response)...) if len(sr.Diffs) > 0 { sr.OK = false result.OK = false diff --git a/tide/report.go b/tide/report.go new file mode 100644 index 0000000..b1646c2 --- /dev/null +++ b/tide/report.go @@ -0,0 +1,53 @@ +package tide + +import ( + "fmt" + "strconv" +) + +// Coverage is the recorded/passing/failing/unrecorded table over manifest routes. +type Coverage struct { + Total int + Recorded int + Passing int + Failing int + Unrecorded int + Pending int + Ported int + ResumeRemaining int + Rows []CoverageRow + Diffs []string +} + +// CoverageRow is one manifest route's outcome. +type CoverageRow struct { + ID string + Status string + Outcome string +} + +// CoverageHeaders is the table header for CLI output. +func CoverageHeaders() []string { + return []string{"route", "status", "outcome"} +} + +// CoverageRows renders the coverage table. +func (c Coverage) CoverageRows() [][]string { + rows := make([][]string, 0, len(c.Rows)) + for _, r := range c.Rows { + rows = append(rows, []string{r.ID, r.Status, r.Outcome}) + } + return rows +} + +// SummaryLine is a one-line recorded/passing/failing/unrecorded report. +func (c Coverage) SummaryLine() string { + return fmt.Sprintf("recorded %d/%d passing %d failing %d unrecorded %d", c.Recorded, c.Total, c.Passing, c.Failing, c.Unrecorded) +} + +func (c Coverage) ResumeLine() string { + if c.ResumeRemaining <= 0 { + return "manifest recording complete" + } + return "resume remaining " + strconv.Itoa(c.ResumeRemaining) +}