feat(02-02): record manifest route cases and report coverage
Drive ordered route cases through RecordFlow, resume in batches of 15, and print recorded/passing/failing/unrecorded coverage. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jakub Zych
367
tide/manifest_test.go
Normal file
367
tide/manifest_test.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package tide
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManifestValidationAndCoverage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fixtures")
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manPath := filepath.Join(dir, "manifest.yaml")
|
||||
manYAML := "" +
|
||||
"version: 1\n" +
|
||||
"auth_groups:\n" +
|
||||
" - public\n" +
|
||||
" - session\n" +
|
||||
"routes:\n" +
|
||||
" - id: GET /ok public\n" +
|
||||
" method: GET\n" +
|
||||
" path: /ok\n" +
|
||||
" auth_group: public\n" +
|
||||
" status: ported\n" +
|
||||
" cases:\n" +
|
||||
" - id: ok\n" +
|
||||
" status: 200\n" +
|
||||
" fixture: routes/ok.yaml\n" +
|
||||
" request:\n" +
|
||||
" method: GET\n" +
|
||||
" path: /ok\n" +
|
||||
" - id: GET /missing public\n" +
|
||||
" method: GET\n" +
|
||||
" path: /missing\n" +
|
||||
" auth_group: public\n" +
|
||||
" status: pending\n" +
|
||||
" cases:\n" +
|
||||
" - id: missing\n" +
|
||||
" status: 200\n" +
|
||||
" fixture: routes/missing.yaml\n" +
|
||||
" request:\n" +
|
||||
" method: GET\n" +
|
||||
" path: /missing\n"
|
||||
if err := os.WriteFile(manPath, []byte(manYAML), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := LoadManifest(manPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeAllowIncomplete); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err == nil {
|
||||
t.Fatal("require-recorded must fail with missing fixtures")
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 2 || cov.Unrecorded != 0 {
|
||||
t.Fatalf("record coverage %+v", cov)
|
||||
}
|
||||
if err := ValidateManifest(m, fixtures, ModeRequireRecorded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageTwoRouteReport(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fx")
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/pass":
|
||||
_, _ = w.Write([]byte(`{"v":1}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"v":2}`))
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
writeRouteFixture(t, srv.URL, fixtures, "pass", "/pass", `{"v":1}`)
|
||||
writeRouteFixture(t, srv.URL, fixtures, "fail", "/fail", `{"v":1}`)
|
||||
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{
|
||||
routeEntry("GET /pass public", "/pass", "ported", "routes/pass.yaml"),
|
||||
routeEntry("GET /fail public", "/fail", "ported", "routes/fail.yaml"),
|
||||
{ID: "GET /none public", Method: "GET", Path: "/none", AuthGroup: "public", Status: StatusPending},
|
||||
},
|
||||
}
|
||||
cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Mode: ModeRequireRecorded})
|
||||
if err == nil {
|
||||
t.Fatal("ported fail + unrecorded required must be nonzero")
|
||||
}
|
||||
if cov.Passing != 1 || cov.Failing != 1 || cov.Unrecorded != 1 || cov.Recorded != 2 {
|
||||
t.Fatalf("coverage %+v", cov)
|
||||
}
|
||||
if !strings.Contains(cov.SummaryLine(), "passing 1") {
|
||||
t.Fatalf("summary %s", cov.SummaryLine())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestResumeBatches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fixtures := filepath.Join(dir, "fx")
|
||||
hits := map[string]int{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits[r.URL.Path]++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
m := Manifest{Version: 1, AuthGroups: []string{"public"}}
|
||||
for i := 1; i <= 16; i++ {
|
||||
id := fmt.Sprintf("GET /r/%d public", i)
|
||||
path := fmt.Sprintf("/r/%d", i)
|
||||
m.Routes = append(m.Routes, Route{
|
||||
ID: id,
|
||||
Method: http.MethodGet,
|
||||
Path: path,
|
||||
AuthGroup: "public",
|
||||
Status: StatusPending,
|
||||
Cases: []RouteCase{{
|
||||
ID: "ok",
|
||||
Status: 200,
|
||||
Fixture: fmt.Sprintf("routes/r%d.yaml", i),
|
||||
Request: &Request{Method: http.MethodGet, Path: path},
|
||||
}},
|
||||
})
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 16}); err == nil {
|
||||
t.Fatal("batch >15 must fail")
|
||||
}
|
||||
cov, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 15 || cov.ResumeRemaining != 1 {
|
||||
t.Fatalf("first batch %+v remaining %d", cov, cov.ResumeRemaining)
|
||||
}
|
||||
cov, err = RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, NextBatch: 15, Resume: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cov.Recorded != 16 || cov.ResumeRemaining != 0 {
|
||||
t.Fatalf("second batch %+v remaining %d", cov, cov.ResumeRemaining)
|
||||
}
|
||||
for i := 1; i <= 16; i++ {
|
||||
if hits[fmt.Sprintf("/r/%d", i)] != 1 {
|
||||
t.Fatalf("route %d recaptured: %d", i, hits[fmt.Sprintf("/r/%d", i)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestAllowIncomplete154(t *testing.T) {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\nauth_groups:\n - public\nroutes:\n")
|
||||
for i := 1; i <= 154; i++ {
|
||||
fmt.Fprintf(&b, " - id: GET /r/%d public\n method: GET\n path: /r/%d\n auth_group: public\n status: pending\n", i, i)
|
||||
}
|
||||
m, err := ParseManifest([]byte(b.String()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.Routes) != 154 {
|
||||
t.Fatalf("routes %d", len(m.Routes))
|
||||
}
|
||||
if err := ValidateManifest(m, t.TempDir(), ModeAllowIncomplete); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, t.TempDir(), ModeRequireRecorded); err == nil {
|
||||
t.Fatal("154 empty cases must fail require-recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestSidecarRejectAndDigest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n body_file: ../secret.bin\n"
|
||||
p := filepath.Join(dir, "bad.yaml")
|
||||
if err := os.WriteFile(p, []byte(base), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadFlow(p); err == nil {
|
||||
t.Fatal("parent sidecar must fail")
|
||||
}
|
||||
good := "version: 1\nname: bin\nsteps:\n - id: a\n request:\n method: GET\n path: /bin\n response:\n status: 200\n headers:\n Content-Type: application/octet-stream\n body_file: data.bin\n sha256: deadbeef\n"
|
||||
gp := filepath.Join(dir, "good.yaml")
|
||||
if err := os.WriteFile(gp, []byte(good), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "data.bin"), []byte("hello"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
flow, err := LoadFlow(gp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
_, err = ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir})
|
||||
if err == nil || !strings.Contains(err.Error(), "digest") {
|
||||
t.Fatalf("digest mismatch: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte("hello"))
|
||||
good2 := strings.Replace(good, "deadbeef", hex.EncodeToString(sum[:]), 1)
|
||||
if err := os.WriteFile(gp, []byte(good2), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
flow, err = LoadFlow(gp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ReplayFlow(context.Background(), flow, ReplayConfig{Target: srv.URL, BaseDir: dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestUnknownGroupAndDuplicate(t *testing.T) {
|
||||
m, err := ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: oauth\n status: pending\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "auth group") {
|
||||
t.Fatalf("unknown group: %v", err)
|
||||
}
|
||||
m, err = ParseManifest([]byte("version: 1\nauth_groups:\n - public\nroutes:\n - id: a\n method: GET\n path: /a\n auth_group: public\n status: pending\n - id: a\n method: GET\n path: /b\n auth_group: public\n status: pending\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateManifest(m, "", ModeAllowIncomplete); err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("duplicate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func routeEntry(id, path, status, fixture string) Route {
|
||||
return Route{
|
||||
ID: id,
|
||||
Method: http.MethodGet,
|
||||
Path: path,
|
||||
AuthGroup: "public",
|
||||
Status: status,
|
||||
Fixture: fixture,
|
||||
Cases: []RouteCase{{
|
||||
ID: "ok",
|
||||
Status: 200,
|
||||
Fixture: fixture,
|
||||
Request: &Request{Method: http.MethodGet, Path: path},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func writeRouteFixture(t *testing.T, target, fixtures, name, path, body string) {
|
||||
t.Helper()
|
||||
spec := Flow{Version: 1, Name: name, Steps: []Step{{
|
||||
ID: "ok",
|
||||
Request: Request{Method: http.MethodGet, Path: path},
|
||||
}}}
|
||||
orig := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(orig.Close)
|
||||
rec, err := RecordFlow(context.Background(), spec, RecordConfig{Target: orig.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveFlow(filepath.Join(fixtures, "routes", name+".yaml"), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestNoOverwriteWithoutUpdate(t *testing.T) {
|
||||
fixtures := t.TempDir()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{{
|
||||
ID: "GET /a public",
|
||||
Method: http.MethodGet,
|
||||
Path: "/a",
|
||||
AuthGroup: "public",
|
||||
Status: StatusPending,
|
||||
Cases: []RouteCase{
|
||||
{ID: "one", Status: 200, Fixture: "routes/one.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}},
|
||||
{ID: "two", Status: 200, Fixture: "routes/two.yaml", Request: &Request{Method: http.MethodGet, Path: "/a"}},
|
||||
},
|
||||
}},
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := Flow{Version: 1, Name: "one", Steps: []Step{{ID: "one", Request: Request{Method: http.MethodGet, Path: "/a"}}}}
|
||||
rec, err := RecordFlow(context.Background(), first, RecordConfig{Target: srv.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveFlow(filepath.Join(fixtures, "routes", "one.yaml"), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures}); err == nil || !strings.Contains(err.Error(), "--update") {
|
||||
t.Fatalf("must refuse overwrite: %v", err)
|
||||
}
|
||||
if _, err := RecordManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, Update: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveragePendingMismatchDoesNotFail(t *testing.T) {
|
||||
fixtures := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(fixtures, "routes"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeRouteFixture(t, "", fixtures, "pend", "/pend", `{"v":1}`)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"v":9}`))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m := Manifest{
|
||||
Version: 1,
|
||||
AuthGroups: []string{"public"},
|
||||
Routes: []Route{routeEntry("GET /pend public", "/pend", StatusPending, "routes/pend.yaml")},
|
||||
}
|
||||
cov, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures})
|
||||
if err != nil {
|
||||
t.Fatalf("pending mismatch must not fail: %v", err)
|
||||
}
|
||||
if cov.Failing != 1 {
|
||||
t.Fatalf("failing %d", cov.Failing)
|
||||
}
|
||||
if _, err := ReplayManifest(context.Background(), m, ManifestConfig{Target: srv.URL, Fixtures: fixtures, SelfCheck: true}); err == nil {
|
||||
t.Fatal("self-check must fail pending mismatches")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user