feat(02-02): record manifest route cases and report coverage
Drive ordered route cases through RecordFlow, resume in batches of 15, and print recorded/passing/failing/unrecorded coverage. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jakub Zych
28
tide/flow.go
28
tide/flow.go
@@ -1,8 +1,11 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
@@ -84,6 +87,7 @@ type ReplayConfig struct {
|
||||
Client *http.Client
|
||||
MaxBody int64
|
||||
Store *Store
|
||||
BaseDir string
|
||||
}
|
||||
|
||||
// Result is the outcome of replaying a flow.
|
||||
@@ -169,6 +173,30 @@ func validateFlow(flow Flow) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func materializeSidecar(base string, resp *Response) error {
|
||||
if resp == nil || resp.BodyFile == "" {
|
||||
return nil
|
||||
}
|
||||
if err := validateSidecar(resp.BodyFile); err != nil {
|
||||
return err
|
||||
}
|
||||
path := resp.BodyFile
|
||||
if base != "" {
|
||||
path = filepath.Join(base, resp.BodyFile)
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tide: read body_file %s: %w", resp.BodyFile, err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
got := hex.EncodeToString(sum[:])
|
||||
if resp.SHA256 != "" && !strings.EqualFold(got, resp.SHA256) {
|
||||
return fmt.Errorf("tide: body_file %s digest mismatch", resp.BodyFile)
|
||||
}
|
||||
resp.Body = Body(raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSidecar(path string) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
|
||||
523
tide/manifest.go
Normal file
523
tide/manifest.go
Normal file
@@ -0,0 +1,523 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/goccy/go-yaml"
|
||||
)
|
||||
|
||||
const MaxBatch = 15
|
||||
|
||||
const (
|
||||
StatusPending = "pending"
|
||||
StatusPorted = "ported"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeAllowIncomplete = "allow-incomplete"
|
||||
ModeRequireRecorded = "require-recorded"
|
||||
)
|
||||
|
||||
// Manifest is a generic ordered list of route cases. It has no app-specific names.
|
||||
type Manifest struct {
|
||||
Version int `yaml:"version"`
|
||||
AuthGroups []string `yaml:"auth_groups"`
|
||||
Seed *Seed `yaml:"seed,omitempty"`
|
||||
Routes []Route `yaml:"routes"`
|
||||
}
|
||||
|
||||
// Seed names an optional bootstrap flow recorded before route cases.
|
||||
type Seed struct {
|
||||
Hook string `yaml:"hook,omitempty"`
|
||||
Spec string `yaml:"spec,omitempty"`
|
||||
Fixture string `yaml:"fixture,omitempty"`
|
||||
}
|
||||
|
||||
// Route is one manifest entry with identity, cases and fixture path.
|
||||
type Route struct {
|
||||
ID string `yaml:"id"`
|
||||
Method string `yaml:"method"`
|
||||
Path string `yaml:"path"`
|
||||
AuthGroup string `yaml:"auth_group"`
|
||||
Status string `yaml:"status"`
|
||||
Identities []string `yaml:"identities,omitempty"`
|
||||
Cases []RouteCase `yaml:"cases,omitempty"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Normalize []NormalizeRule `yaml:"normalize,omitempty"`
|
||||
SeedHook string `yaml:"seed_hook,omitempty"`
|
||||
Fixture string `yaml:"fixture,omitempty"`
|
||||
}
|
||||
|
||||
// RouteCase is one recorded request/response expectation for a route.
|
||||
type RouteCase struct {
|
||||
ID string `yaml:"id"`
|
||||
Identity string `yaml:"identity,omitempty"`
|
||||
Status int `yaml:"status,omitempty"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Fixture string `yaml:"fixture,omitempty"`
|
||||
Request *Request `yaml:"request,omitempty"`
|
||||
}
|
||||
|
||||
// ManifestConfig drives recording or replay of a manifest.
|
||||
type ManifestConfig struct {
|
||||
Target string
|
||||
Fixtures string
|
||||
Store *Store
|
||||
Rules Rules
|
||||
Update bool
|
||||
Resume bool
|
||||
NextBatch int
|
||||
Mode string
|
||||
SelfCheck bool
|
||||
}
|
||||
|
||||
// LoadManifest reads a strict YAML manifest.
|
||||
func LoadManifest(path string) (Manifest, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("tide: read manifest %s: %w", path, err)
|
||||
}
|
||||
return ParseManifest(raw)
|
||||
}
|
||||
|
||||
// ParseManifest decodes a manifest, rejecting unknown fields.
|
||||
func ParseManifest(raw []byte) (Manifest, error) {
|
||||
var m Manifest
|
||||
dec := yaml.NewDecoder(bytes.NewReader(raw), yaml.DisallowUnknownField())
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
return Manifest{}, fmt.Errorf("tide: parse manifest: %w", err)
|
||||
}
|
||||
if m.Version != CurrentVersion {
|
||||
return Manifest{}, fmt.Errorf("tide: unsupported manifest version %d", m.Version)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ValidateManifest checks identities in both modes; case/fixture completeness depends on mode.
|
||||
func ValidateManifest(m Manifest, fixtures, mode string) error {
|
||||
if len(m.AuthGroups) == 0 {
|
||||
return fmt.Errorf("tide: manifest auth_groups is required")
|
||||
}
|
||||
groups := make(map[string]struct{}, len(m.AuthGroups))
|
||||
for _, g := range m.AuthGroups {
|
||||
if strings.TrimSpace(g) == "" {
|
||||
return fmt.Errorf("tide: empty auth group")
|
||||
}
|
||||
groups[g] = struct{}{}
|
||||
}
|
||||
if len(m.Routes) == 0 {
|
||||
return fmt.Errorf("tide: manifest has no routes")
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for i, route := range m.Routes {
|
||||
if strings.TrimSpace(route.ID) == "" {
|
||||
return fmt.Errorf("tide: routes[%d] is missing id", i)
|
||||
}
|
||||
if _, dup := seen[route.ID]; dup {
|
||||
return fmt.Errorf("tide: duplicate route id %q", route.ID)
|
||||
}
|
||||
seen[route.ID] = struct{}{}
|
||||
if strings.TrimSpace(route.Method) == "" || strings.TrimSpace(route.Path) == "" {
|
||||
return fmt.Errorf("tide: route %s is missing method or path", route.ID)
|
||||
}
|
||||
if _, ok := groups[route.AuthGroup]; !ok {
|
||||
return fmt.Errorf("tide: route %s has unknown auth group %q", route.ID, route.AuthGroup)
|
||||
}
|
||||
if route.Status != StatusPending && route.Status != StatusPorted {
|
||||
return fmt.Errorf("tide: route %s has unknown status %q", route.ID, route.Status)
|
||||
}
|
||||
if err := validateFixturePath(route.Fixture); err != nil {
|
||||
return fmt.Errorf("tide: route %s: %w", route.ID, err)
|
||||
}
|
||||
for j, c := range route.Cases {
|
||||
if err := validateFixturePath(c.Fixture); err != nil {
|
||||
return fmt.Errorf("tide: route %s case[%d]: %w", route.ID, j, err)
|
||||
}
|
||||
if mode == ModeRequireRecorded {
|
||||
if err := requireCase(route, c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if mode == ModeRequireRecorded {
|
||||
if len(route.Cases) == 0 {
|
||||
return fmt.Errorf("tide: route %s is missing cases", route.ID)
|
||||
}
|
||||
for _, c := range routeCases(route) {
|
||||
path := caseFixturePath(fixtures, route, c)
|
||||
if _, err := LoadFlow(path); err != nil {
|
||||
return fmt.Errorf("tide: route %s missing valid fixture %s: %w", route.ID, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireCase(route Route, c RouteCase) error {
|
||||
if strings.TrimSpace(c.ID) == "" {
|
||||
return fmt.Errorf("tide: route %s has a case without id", route.ID)
|
||||
}
|
||||
if c.Status == 0 {
|
||||
return fmt.Errorf("tide: route %s case %s is missing status", route.ID, c.ID)
|
||||
}
|
||||
req := caseRequest(route, c)
|
||||
if strings.TrimSpace(req.Method) == "" || strings.TrimSpace(req.Path) == "" {
|
||||
return fmt.Errorf("tide: route %s case %s is missing request", route.ID, c.ID)
|
||||
}
|
||||
if caseFixturePath("", route, c) == "" {
|
||||
return fmt.Errorf("tide: route %s case %s is missing fixture", route.ID, c.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateFixturePath(p string) error {
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
if filepath.IsAbs(p) {
|
||||
return fmt.Errorf("fixture %q must be a relative path", p)
|
||||
}
|
||||
clean := filepath.ToSlash(filepath.Clean(p))
|
||||
if clean == ".." || strings.HasPrefix(clean, "../") {
|
||||
return fmt.Errorf("fixture %q escapes the fixture directory", p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func routeCases(route Route) []RouteCase {
|
||||
if len(route.Cases) > 0 {
|
||||
return route.Cases
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func caseRequest(route Route, c RouteCase) Request {
|
||||
if c.Request != nil {
|
||||
return *c.Request
|
||||
}
|
||||
return Request{Method: route.Method, Path: route.Path}
|
||||
}
|
||||
|
||||
func caseFixturePath(root string, route Route, c RouteCase) string {
|
||||
rel := c.Fixture
|
||||
if rel == "" {
|
||||
rel = route.Fixture
|
||||
}
|
||||
if rel == "" && c.ID != "" {
|
||||
rel = filepath.ToSlash(filepath.Join("routes", sanitizeFile(route.ID)+"__"+sanitizeFile(c.ID)+".yaml"))
|
||||
}
|
||||
if rel == "" && route.ID != "" {
|
||||
rel = filepath.ToSlash(filepath.Join("routes", sanitizeFile(route.ID)+".yaml"))
|
||||
}
|
||||
if root == "" {
|
||||
return rel
|
||||
}
|
||||
return filepath.Join(root, rel)
|
||||
}
|
||||
|
||||
func sanitizeFile(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
repl := strings.NewReplacer("/", "_", "\\", "_", " ", "_")
|
||||
return repl.Replace(s)
|
||||
}
|
||||
|
||||
func fixtureHash(path string) (string, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return "", fmt.Errorf("empty fixture")
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func recordedCase(fixtures string, route Route, c RouteCase) (bool, error) {
|
||||
path := caseFixturePath(fixtures, route, c)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if _, err := LoadFlow(path); err != nil {
|
||||
return false, fmt.Errorf("tide: existing fixture %s is invalid: %w", path, err)
|
||||
}
|
||||
if _, err := fixtureHash(path); err != nil {
|
||||
return false, fmt.Errorf("tide: existing fixture %s hash: %w", path, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// RecordManifest records seed then route cases. NextBatch>15 is refused.
|
||||
func RecordManifest(ctx context.Context, m Manifest, cfg ManifestConfig) (Coverage, error) {
|
||||
if cfg.NextBatch > MaxBatch {
|
||||
return Coverage{}, fmt.Errorf("tide: --next-batch %d exceeds %d", cfg.NextBatch, MaxBatch)
|
||||
}
|
||||
if err := ValidateManifest(m, cfg.Fixtures, ModeAllowIncomplete); err != nil {
|
||||
return Coverage{}, err
|
||||
}
|
||||
cov := newCoverage(m)
|
||||
if m.Seed != nil && strings.TrimSpace(m.Seed.Spec) != "" {
|
||||
if err := recordSeed(ctx, m.Seed, cfg); err != nil {
|
||||
return cov, err
|
||||
}
|
||||
}
|
||||
limit := cfg.NextBatch
|
||||
if limit <= 0 {
|
||||
limit = len(m.Routes)
|
||||
}
|
||||
taken := 0
|
||||
for _, route := range m.Routes {
|
||||
cases := route.Cases
|
||||
if len(cases) == 0 {
|
||||
cov.markUnrecorded(route)
|
||||
continue
|
||||
}
|
||||
allRecorded := true
|
||||
for _, c := range cases {
|
||||
ok, err := recordedCase(cfg.Fixtures, route, c)
|
||||
if err != nil {
|
||||
return cov, err
|
||||
}
|
||||
if !ok {
|
||||
allRecorded = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allRecorded {
|
||||
cov.markRecorded(route)
|
||||
continue
|
||||
}
|
||||
if taken >= limit {
|
||||
cov.markUnrecorded(route)
|
||||
continue
|
||||
}
|
||||
if err := recordRoute(ctx, route, cfg); err != nil {
|
||||
return cov, err
|
||||
}
|
||||
cov.markRecorded(route)
|
||||
taken++
|
||||
}
|
||||
cov.ResumeRemaining = countUnrecorded(m, cfg.Fixtures)
|
||||
return cov, nil
|
||||
}
|
||||
|
||||
func recordSeed(ctx context.Context, seed *Seed, cfg ManifestConfig) error {
|
||||
spec, err := LoadFlow(seed.Spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flow, err := RecordFlow(ctx, spec, RecordConfig{Target: cfg.Target, Store: cfg.Store, Rules: cfg.Rules})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dest := seed.Fixture
|
||||
if dest == "" {
|
||||
dest = seed.Spec
|
||||
}
|
||||
if cfg.Fixtures != "" && !filepath.IsAbs(dest) {
|
||||
dest = filepath.Join(cfg.Fixtures, dest)
|
||||
}
|
||||
if _, err := os.Stat(dest); err == nil && !cfg.Update {
|
||||
return nil
|
||||
}
|
||||
return SaveFlow(dest, flow)
|
||||
}
|
||||
|
||||
func recordRoute(ctx context.Context, route Route, cfg ManifestConfig) error {
|
||||
for _, c := range route.Cases {
|
||||
dest := caseFixturePath(cfg.Fixtures, route, c)
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
if cfg.Resume && !cfg.Update {
|
||||
if _, err := LoadFlow(dest); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !cfg.Update {
|
||||
return fmt.Errorf("tide: fixture %s exists; pass --update to overwrite", dest)
|
||||
}
|
||||
}
|
||||
spec := Flow{
|
||||
Version: CurrentVersion,
|
||||
Name: route.ID + "/" + c.ID,
|
||||
SeedHook: route.SeedHook,
|
||||
Steps: []Step{{
|
||||
ID: c.ID,
|
||||
RouteID: route.ID,
|
||||
Request: caseRequest(route, c),
|
||||
Normalize: route.Normalize,
|
||||
Headers: mergeHeaders(route.Headers, c.Headers),
|
||||
}},
|
||||
}
|
||||
flow, err := RecordFlow(ctx, spec, RecordConfig{Target: cfg.Target, Store: cfg.Store, Rules: cfg.Rules})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := SaveFlow(dest, flow); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeHeaders(a, b map[string]string) map[string]string {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string)
|
||||
for k, v := range a {
|
||||
out[k] = v
|
||||
}
|
||||
for k, v := range b {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func countUnrecorded(m Manifest, fixtures string) int {
|
||||
n := 0
|
||||
for _, route := range m.Routes {
|
||||
if len(route.Cases) == 0 {
|
||||
n++
|
||||
continue
|
||||
}
|
||||
for _, c := range route.Cases {
|
||||
ok, err := recordedCase(fixtures, route, c)
|
||||
if err != nil || !ok {
|
||||
n++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ReplayManifest replays recorded route fixtures and builds a coverage table.
|
||||
func ReplayManifest(ctx context.Context, m Manifest, cfg ManifestConfig) (Coverage, error) {
|
||||
if err := ValidateManifest(m, cfg.Fixtures, ModeAllowIncomplete); err != nil {
|
||||
return Coverage{}, err
|
||||
}
|
||||
cov := newCoverage(m)
|
||||
var fail error
|
||||
for _, route := range m.Routes {
|
||||
cases := route.Cases
|
||||
if len(cases) == 0 {
|
||||
cov.markUnrecorded(route)
|
||||
if cfg.Mode == ModeRequireRecorded {
|
||||
fail = firstErr(fail, fmt.Errorf("tide: unrecorded required route %s", route.ID))
|
||||
}
|
||||
continue
|
||||
}
|
||||
routeFail := false
|
||||
routeRecorded := true
|
||||
for _, c := range cases {
|
||||
path := caseFixturePath(cfg.Fixtures, route, c)
|
||||
flow, err := LoadFlow(path)
|
||||
if err != nil {
|
||||
routeRecorded = false
|
||||
if cfg.Mode == ModeRequireRecorded {
|
||||
fail = firstErr(fail, fmt.Errorf("tide: unrecorded required route %s", route.ID))
|
||||
}
|
||||
continue
|
||||
}
|
||||
res, err := ReplayFlow(ctx, flow, ReplayConfig{Target: cfg.Target, Store: cfg.Store, BaseDir: filepath.Dir(path)})
|
||||
if err != nil {
|
||||
var mis *MismatchError
|
||||
if !errors.As(err, &mis) {
|
||||
return cov, err
|
||||
}
|
||||
res = mis.Result
|
||||
}
|
||||
if !res.OK {
|
||||
routeFail = true
|
||||
cov.addDiffs(route.ID, res)
|
||||
if cfg.SelfCheck || route.Status == StatusPorted {
|
||||
fail = firstErr(fail, fmt.Errorf("tide: route %s failed", route.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !routeRecorded {
|
||||
cov.markUnrecorded(route)
|
||||
continue
|
||||
}
|
||||
if routeFail {
|
||||
cov.markFailing(route)
|
||||
continue
|
||||
}
|
||||
cov.markPassing(route)
|
||||
}
|
||||
return cov, fail
|
||||
}
|
||||
|
||||
func firstErr(cur, next error) error {
|
||||
if cur == nil {
|
||||
return next
|
||||
}
|
||||
return cur
|
||||
}
|
||||
|
||||
func newCoverage(m Manifest) Coverage {
|
||||
c := Coverage{Total: len(m.Routes), Rows: make([]CoverageRow, 0, len(m.Routes))}
|
||||
for _, route := range m.Routes {
|
||||
if route.Status == StatusPending {
|
||||
c.Pending++
|
||||
} else if route.Status == StatusPorted {
|
||||
c.Ported++
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Coverage) markRecorded(route Route) {
|
||||
c.Recorded++
|
||||
c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "recorded"})
|
||||
}
|
||||
|
||||
func (c *Coverage) markPassing(route Route) {
|
||||
c.Recorded++
|
||||
c.Passing++
|
||||
c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "passing"})
|
||||
}
|
||||
|
||||
func (c *Coverage) markFailing(route Route) {
|
||||
c.Recorded++
|
||||
c.Failing++
|
||||
c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "failing"})
|
||||
}
|
||||
|
||||
func (c *Coverage) markUnrecorded(route Route) {
|
||||
c.Unrecorded++
|
||||
c.Rows = append(c.Rows, CoverageRow{ID: route.ID, Status: route.Status, Outcome: "unrecorded"})
|
||||
}
|
||||
|
||||
func (c *Coverage) addDiffs(id string, res Result) {
|
||||
for _, sr := range res.Steps {
|
||||
for _, d := range sr.Diffs {
|
||||
c.Diffs = append(c.Diffs, fmt.Sprintf("%s %s: %s expected %s actual %s", id, sr.ID, d.Path, d.Expected, d.Actual))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ParseNextBatch(s string) (int, error) {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("tide: --next-batch: %w", err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
367
tide/manifest_test.go
Normal file
367
tide/manifest_test.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManifestValidationAndCoverage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fixtures")
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manPath := filepath.Join(dir, "manifest.yaml")
|
||||
manYAML := "" +
|
||||
"version: 1\n" +
|
||||
"auth_groups:\n" +
|
||||
" - public\n" +
|
||||
" - session\n" +
|
||||
"routes:\n" +
|
||||
" - id: GET /ok public\n" +
|
||||
" method: GET\n" +
|
||||
" path: /ok\n" +
|
||||
" auth_group: public\n" +
|
||||
" status: ported\n" +
|
||||
" cases:\n" +
|
||||
" - id: ok\n" +
|
||||
" status: 200\n" +
|
||||
" fixture: routes/ok.yaml\n" +
|
||||
" request:\n" +
|
||||
" method: GET\n" +
|
||||
" path: /ok\n" +
|
||||
" - id: GET /missing public\n" +
|
||||
" method: GET\n" +
|
||||
" path: /missing\n" +
|
||||
" auth_group: public\n" +
|
||||
" status: pending\n" +
|
||||
" cases:\n" +
|
||||
" - id: missing\n" +
|
||||
" status: 200\n" +
|
||||
" fixture: routes/missing.yaml\n" +
|
||||
" request:\n" +
|
||||
" method: GET\n" +
|
||||
" path: /missing\n"
|
||||
if err := os.WriteFile(manPath, []byte(manYAML), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := LoadManifest(manPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeAllowIncomplete); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err == nil {
|
||||
t.Fatal("require-recorded must fail with missing fixtures")
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 2 || cov.Unrecorded != 0 {
|
||||
t.Fatalf("record coverage %+v", cov)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageTwoRouteReport(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fx")
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/pass":
|
||||
_, _ = w.Write([]byte(`{"v":1}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"v":2}`))
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
writeRouteFixture(t, srv.URL, fixtures, "pass", "/pass", `{"v":1}`)
|
||||
writeRouteFixture(t, srv.URL, fixtures, "fail", "/fail", `{"v":1}`)
|
||||
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{
|
||||
routeEntry("GET /pass public", "/pass", "ported", "routes/pass.yaml"),
|
||||
routeEntry("GET /fail public", "/fail", "ported", "routes/fail.yaml"),
|
||||
{ID: "GET /none public", Method: "GET", Path: "/none", AuthGroup: "public", Status: StatusPending},
|
||||
},
|
||||
}
|
||||
cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Mode: ModeRequireRecorded})
|
||||
if err == nil {
|
||||
t.Fatal("ported fail + unrecorded required must be nonzero")
|
||||
}
|
||||
if cov.Passing != 1 || cov.Failing != 1 || cov.Unrecorded != 1 || cov.Recorded != 2 {
|
||||
t.Fatalf("coverage %+v", cov)
|
||||
}
|
||||
if !strings.Contains(cov.SummaryLine(), "passing 1") {
|
||||
t.Fatalf("summary %s", cov.SummaryLine())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestResumeBatches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fx")
|
||||
hits := map[string]int{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits[r.URL.Path]++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
m := Manifest{Version: 1, AuthGroups: []string{"public"}}
|
||||
for i := 1; i <= 16; i++ {
|
||||
id := fmt.Sprintf("GET /r/%d public", i)
|
||||
path := fmt.Sprintf("/r/%d", i)
|
||||
m.Routes = append(m.Routes, Route{
|
||||
ID: id,
|
||||
Method: http.MethodGet,
|
||||
Path: path,
|
||||
AuthGroup: "public",
|
||||
Status: StatusPending,
|
||||
Cases: []RouteCase{{
|
||||
ID: "ok",
|
||||
Status: 200,
|
||||
Fixture: fmt.Sprintf("routes/r%d.yaml", i),
|
||||
Request: &Request{Method: http.MethodGet, Path: path},
|
||||
}},
|
||||
})
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 16}); err == nil {
|
||||
t.Fatal("batch >15 must fail")
|
||||
}
|
||||
cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 15 || cov.ResumeRemaining != 1 {
|
||||
t.Fatalf("first batch %+v remaining %d", cov, cov.ResumeRemaining)
|
||||
}
|
||||
cov, err = RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 16 || cov.ResumeRemaining != 0 {
|
||||
t.Fatalf("second batch %+v remaining %d", cov, cov.ResumeRemaining)
|
||||
}
|
||||
for i := 1; i <= 16; i++ {
|
||||
if hits[fmt.Sprintf("/r/%d", i)] != 1 {
|
||||
t.Fatalf("route %d recaptured: %d", i, hits[fmt.Sprintf("/r/%d", i)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAllowIncomplete154(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\nauth_groups:\n - public\nroutes:\n")
|
||||
for i := 1; i <= 154; i++ {
|
||||
fmt.Fprintf(&b, " - id: GET /r/%d public\n method: GET\n path: /r/%d\n auth_group: public\n status: pending\n", i, i)
|
||||
}
|
||||
m, err := ParseManifest([]byte(b.String()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.Routes) != 154 {
|
||||
t.Fatalf("routes %d", len(m.Routes))
|
||||
}
|
||||
if err := ValidateManifest(m, t.TempDir(), ModeAllowIncomplete); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, t.TempDir(), ModeRequireRecorded); err == nil {
|
||||
t.Fatal("154 empty cases must fail require-recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestSidecarRejectAndDigest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n body_file: ../secret.bin\n"
|
||||
p := filepath.Join(dir, "bad.yaml")
|
||||
if err := os.WriteFile(p, []byte(base), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(p); err == nil {
|
||||
t.Fatal("parent sidecar must fail")
|
||||
}
|
||||
good := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n status: 200\n headers:\n Content-Type: application/octet-stream\n body_file: data.bin\n sha256: deadbeef\n"
|
||||
gp := filepath.Join(dir, "good.yaml")
|
||||
if err := os.WriteFile(gp, []byte(good), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "data.bin"), []byte("hello"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
flow, err := LoadFlow(gp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
_, err = ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir})
|
||||
if err == nil || !strings.Contains(err.Error(), "digest") {
|
||||
t.Fatalf("digest mismatch: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte("hello"))
|
||||
good2 := strings.Replace(good, "deadbeef", hex.EncodeToString(sum[:]), 1)
|
||||
if err := os.WriteFile(gp, []byte(good2), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
flow, err = LoadFlow(gp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestUnknownGroupAndDuplicate(t *testing.T) {
|
||||
m, err := ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: oauth\n status: pending\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "auth group") {
|
||||
t.Fatalf("unknown group: %v", err)
|
||||
}
|
||||
m, err = ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: public\n status: pending\n - id: a\n method: GET\n path: /b\n auth_group: public\n status: pending\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("duplicate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func routeEntry(id, path, status, fixture string) Route {
|
||||
return Route{
|
||||
ID: id,
|
||||
Method: http.MethodGet,
|
||||
Path: path,
|
||||
AuthGroup: "public",
|
||||
Status: status,
|
||||
Fixture: fixture,
|
||||
Cases: []RouteCase{{
|
||||
ID: "ok",
|
||||
Status: 200,
|
||||
Fixture: fixture,
|
||||
Request: &Request{Method: http.MethodGet, Path: path},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func writeRouteFixture(t *testing.T, target, fixtures, name, path, body string) {
|
||||
t.Helper()
|
||||
spec := Flow{Version: 1, Name: name, Steps: []Step{{
|
||||
ID: "ok",
|
||||
Request: Request{Method: http.MethodGet, Path: path},
|
||||
}}}
|
||||
orig := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(orig.Close)
|
||||
rec, err := RecordFlow(context.Background(), spec, RecordConfig{Target: orig.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveFlow(filepath.Join(fixtures, "routes", name+".yaml"), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestNoOverwriteWithoutUpdate(t *testing.T) {
|
||||
fixtures := t.TempDir()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{{
|
||||
ID: "GET /a public",
|
||||
Method: http.MethodGet,
|
||||
Path: "/a",
|
||||
AuthGroup: "public",
|
||||
Status: StatusPending,
|
||||
Cases: []RouteCase{
|
||||
{ID: "one", Status: 200, Fixture: "routes/one.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}},
|
||||
{ID: "two", Status: 200, Fixture: "routes/two.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}},
|
||||
},
|
||||
}},
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := Flow{Version: 1, Name: "one", Steps: []Step{{ID: "one", Request: Request{Method: http.MethodGet, Path: "/a"}}}}
|
||||
rec, err := RecordFlow(context.Background(), first, RecordConfig{Target: srv.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveFlow(filepath.Join(fixtures, "routes", "one.yaml"), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures}); err == nil || !strings.Contains(err.Error(), "--update") {
|
||||
t.Fatalf("must refuse overwrite: %v", err)
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Update: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveragePendingMismatchDoesNotFail(t *testing.T) {
|
||||
fixtures := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeRouteFixture(t, "", fixtures, "pend", "/pend", `{"v":1}`)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"v":9}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{routeEntry("GET /pend public", "/pend", StatusPending, "routes/pend.yaml")},
|
||||
}
|
||||
cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures})
|
||||
if err != nil {
|
||||
t.Fatalf("pending mismatch must not fail: %v", err)
|
||||
}
|
||||
if cov.Failing != 1 {
|
||||
t.Fatalf("failing %d", cov.Failing)
|
||||
}
|
||||
if _, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, SelfCheck: true}); err == nil {
|
||||
t.Fatal("self-check must fail pending mismatches")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,10 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
|
||||
continue
|
||||
}
|
||||
sr := StepResult{ID: step.ID, OK: true}
|
||||
want := step
|
||||
if err := materializeSidecar(cfg.BaseDir, &want.Response); err != nil {
|
||||
return result, fmt.Errorf("tide: replay step %s: %w", step.ID, err)
|
||||
}
|
||||
req, err := expandRequest(step.Request, store)
|
||||
if err != nil {
|
||||
sr.OK = false
|
||||
@@ -56,7 +60,7 @@ func ReplayFlow(ctx context.Context, flow Flow, cfg ReplayConfig) (Result, error
|
||||
continue
|
||||
}
|
||||
_ = ScrubStep(store, &live)
|
||||
sr.Diffs = append(sr.Diffs, compareStep(step, live.Response)...)
|
||||
sr.Diffs = append(sr.Diffs, compareStep(want, live.Response)...)
|
||||
if len(sr.Diffs) > 0 {
|
||||
sr.OK = false
|
||||
result.OK = false
|
||||
|
||||
53
tide/report.go
Normal file
53
tide/report.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Coverage is the recorded/passing/failing/unrecorded table over manifest routes.
|
||||
type Coverage struct {
|
||||
Total int
|
||||
Recorded int
|
||||
Passing int
|
||||
Failing int
|
||||
Unrecorded int
|
||||
Pending int
|
||||
Ported int
|
||||
ResumeRemaining int
|
||||
Rows []CoverageRow
|
||||
Diffs []string
|
||||
}
|
||||
|
||||
// CoverageRow is one manifest route's outcome.
|
||||
type CoverageRow struct {
|
||||
ID string
|
||||
Status string
|
||||
Outcome string
|
||||
}
|
||||
|
||||
// CoverageHeaders is the table header for CLI output.
|
||||
func CoverageHeaders() []string {
|
||||
return []string{"route", "status", "outcome"}
|
||||
}
|
||||
|
||||
// CoverageRows renders the coverage table.
|
||||
func (c Coverage) CoverageRows() [][]string {
|
||||
rows := make([][]string, 0, len(c.Rows))
|
||||
for _, r := range c.Rows {
|
||||
rows = append(rows, []string{r.ID, r.Status, r.Outcome})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// SummaryLine is a one-line recorded/passing/failing/unrecorded report.
|
||||
func (c Coverage) SummaryLine() string {
|
||||
return fmt.Sprintf("recorded %d/%d passing %d failing %d unrecorded %d", c.Recorded, c.Total, c.Passing, c.Failing, c.Unrecorded)
|
||||
}
|
||||
|
||||
func (c Coverage) ResumeLine() string {
|
||||
if c.ResumeRemaining <= 0 {
|
||||
return "manifest recording complete"
|
||||
}
|
||||
return "resume remaining " + strconv.Itoa(c.ResumeRemaining)
|
||||
}
|
||||
Reference in New Issue
Block a user