feat(14-01): record vendor calls with summer parity:upstream and replay them offline
- WriteUpstream masks vars, hashes long base64 JSON strings and refuses unmasked Authorization/X-Api-Key - multipart requests recorded as ordered parts; the fake compares parts and hashed payloads - loopback CONNECT recording proxy with a local ECDSA parity CA, script and forward modes - parity:upstream command, README and parity docs
This commit is contained in:
Jakub Zych
@@ -2,12 +2,20 @@ package tide
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -33,13 +41,36 @@ type UpstreamExchange struct {
|
||||
// UpstreamRequest is the request the backend sent upstream. URL is absolute
|
||||
// (scheme, host, path and query). Headers holds only the compared headers
|
||||
// (UpstreamCompareHeaders); credential values are {{name}} placeholders.
|
||||
// A multipart/form-data request keeps Parts instead of Body.
|
||||
type UpstreamRequest struct {
|
||||
Method string `yaml:"method"`
|
||||
URL string `yaml:"url"`
|
||||
Headers map[string]string `yaml:"headers,omitempty"`
|
||||
Body string `yaml:"body,omitempty"`
|
||||
Parts []UpstreamPart `yaml:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// UpstreamPart is one multipart/form-data part of a recorded request, in
|
||||
// order. A plain field keeps its Value (credentials masked as {{name}}); a
|
||||
// file part keeps Filename, ContentType and the SHA256 of its bytes, never
|
||||
// the bytes.
|
||||
type UpstreamPart struct {
|
||||
Name string `yaml:"name"`
|
||||
Value string `yaml:"value,omitempty"`
|
||||
Filename string `yaml:"filename,omitempty"`
|
||||
ContentType string `yaml:"content_type,omitempty"`
|
||||
SHA256 string `yaml:"sha256,omitempty"`
|
||||
}
|
||||
|
||||
// MaxUpstreamBody caps one upstream request or response body the fake and
|
||||
// the recording proxy read. Vision requests carry a base64 photo, so it is
|
||||
// larger than DefaultMaxBody.
|
||||
const MaxUpstreamBody = 32 << 20
|
||||
|
||||
// upstreamHashMin is the length above which a base64 JSON string value is
|
||||
// stored as a {{sha256:<hex>}} placeholder.
|
||||
const upstreamHashMin = 1024
|
||||
|
||||
// UpstreamResponse is the recorded vendor answer the fake replays.
|
||||
type UpstreamResponse struct {
|
||||
Status int `yaml:"status"`
|
||||
@@ -133,7 +164,7 @@ func (f *UpstreamFake) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
var body []byte
|
||||
if req.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(io.LimitReader(req.Body, DefaultMaxBody+1))
|
||||
body, err = io.ReadAll(io.LimitReader(req.Body, MaxUpstreamBody+1))
|
||||
_ = req.Body.Close()
|
||||
if err != nil {
|
||||
return nil, f.fail(fmt.Errorf("tide: upstream %s %s: read body: %w", req.Method, req.URL, err))
|
||||
@@ -213,7 +244,7 @@ func (f *UpstreamFake) compare(want UpstreamRequest, got *http.Request, body []b
|
||||
continue
|
||||
}
|
||||
gv := got.Header.Get(name)
|
||||
if wv == gv {
|
||||
if wv == gv || (strings.EqualFold(name, "Content-Type") && sameMultipart(wv, gv)) {
|
||||
continue
|
||||
}
|
||||
if isCredentialHeader(name) {
|
||||
@@ -223,16 +254,18 @@ func (f *UpstreamFake) compare(want UpstreamRequest, got *http.Request, body []b
|
||||
problems = append(problems, fmt.Sprintf("header %s: want %q, got %q", name, wv, gv))
|
||||
}
|
||||
|
||||
wantBody, err := f.store.Expand(want.Body)
|
||||
if len(want.Parts) > 0 {
|
||||
return append(problems, f.compareParts(want.Parts, got.Header.Get("Content-Type"), body)...)
|
||||
}
|
||||
wantBody, err := f.store.expandKeeping(want.Body, isHashPlaceholder)
|
||||
if err != nil {
|
||||
return append(problems, "body: "+err.Error())
|
||||
}
|
||||
jsonBody := isJSONContentType(want.Headers) || isJSONContentType(map[string]string{"Content-Type": got.Header.Get("Content-Type")})
|
||||
switch {
|
||||
case jsonBody && (len(wantBody) > 0 || len(body) > 0):
|
||||
if diffs := diffJSON([]byte(wantBody), body); len(diffs) > 0 {
|
||||
d := diffs[0]
|
||||
problems = append(problems, fmt.Sprintf("body %s: want %s, got %s", d.Path, d.Expected, d.Actual))
|
||||
if d, ok := diffUpstreamJSON([]byte(wantBody), body); !ok {
|
||||
problems = append(problems, fmt.Sprintf("body %s: want %s, got %s", d.Path, clip(d.Expected), clip(d.Actual)))
|
||||
}
|
||||
case wantBody != string(body):
|
||||
problems = append(problems, fmt.Sprintf("body: want %d bytes, got %d bytes", len(wantBody), len(body)))
|
||||
@@ -303,3 +336,347 @@ func redactURL(u *url.URL) string {
|
||||
c.User = nil
|
||||
return c.String()
|
||||
}
|
||||
|
||||
// sameMultipart reports whether both values are multipart/form-data, whose
|
||||
// boundary differs on every request.
|
||||
func sameMultipart(a, b string) bool {
|
||||
return isMultipart(a) && isMultipart(b)
|
||||
}
|
||||
|
||||
func isMultipart(contentType string) bool {
|
||||
media, _, err := mime.ParseMediaType(contentType)
|
||||
return err == nil && media == "multipart/form-data"
|
||||
}
|
||||
|
||||
func (f *UpstreamFake) compareParts(want []UpstreamPart, contentType string, body []byte) []string {
|
||||
got, err := upstreamParts(contentType, body)
|
||||
if err != nil {
|
||||
return []string{"parts: " + err.Error()}
|
||||
}
|
||||
var problems []string
|
||||
if len(want) != len(got) {
|
||||
problems = append(problems, fmt.Sprintf("parts: want %d, got %d", len(want), len(got)))
|
||||
}
|
||||
for i := range min(len(want), len(got)) {
|
||||
w, g := want[i], got[i]
|
||||
value, err := f.store.Expand(w.Value)
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Sprintf("part %d %s: %v", i, w.Name, err))
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case w.Name != g.Name:
|
||||
problems = append(problems, fmt.Sprintf("part %d name: want %q, got %q", i, w.Name, g.Name))
|
||||
case w.Filename != g.Filename:
|
||||
problems = append(problems, fmt.Sprintf("part %d %s filename: want %q, got %q", i, w.Name, w.Filename, g.Filename))
|
||||
case w.ContentType != g.ContentType:
|
||||
problems = append(problems, fmt.Sprintf("part %d %s content type: want %q, got %q", i, w.Name, w.ContentType, g.ContentType))
|
||||
case !strings.EqualFold(w.SHA256, g.SHA256):
|
||||
problems = append(problems, fmt.Sprintf("part %d %s sha256: want %s, got %s", i, w.Name, w.SHA256, g.SHA256))
|
||||
case value != g.Value:
|
||||
problems = append(problems, fmt.Sprintf("part %d %s value differs", i, w.Name))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// upstreamParts splits a multipart/form-data body into recorded parts: plain
|
||||
// fields keep their value, file parts their sha256.
|
||||
func upstreamParts(contentType string, body []byte) ([]UpstreamPart, error) {
|
||||
media, params, err := mime.ParseMediaType(contentType)
|
||||
if err != nil || media != "multipart/form-data" || params["boundary"] == "" {
|
||||
return nil, fmt.Errorf("content type %q is not multipart/form-data", contentType)
|
||||
}
|
||||
mr := multipart.NewReader(bytes.NewReader(body), params["boundary"])
|
||||
var parts []UpstreamPart
|
||||
for {
|
||||
p, err := mr.NextRawPart()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return parts, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := io.ReadAll(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
up := UpstreamPart{Name: p.FormName(), Filename: p.FileName()}
|
||||
if up.Filename != "" || p.Header.Get("Content-Type") != "" {
|
||||
up.ContentType = p.Header.Get("Content-Type")
|
||||
sum := sha256.Sum256(raw)
|
||||
up.SHA256 = hex.EncodeToString(sum[:])
|
||||
} else {
|
||||
up.Value = string(raw)
|
||||
}
|
||||
parts = append(parts, up)
|
||||
}
|
||||
}
|
||||
|
||||
var hashPlaceholderRe = regexp.MustCompile(`^(data:[^,]*;base64,)?\{\{sha256:([0-9a-f]{64})\}\}$`)
|
||||
|
||||
func isHashPlaceholder(name string) bool { return strings.HasPrefix(name, "sha256:") }
|
||||
|
||||
// expandKeeping is Expand that leaves placeholders whose name keep accepts.
|
||||
func (s *Store) expandKeeping(text string, keep func(string) bool) (string, error) {
|
||||
if !strings.Contains(text, "{{") {
|
||||
return text, nil
|
||||
}
|
||||
var missing []string
|
||||
out := placeholderRe.ReplaceAllStringFunc(text, func(m string) string {
|
||||
name := m[2 : len(m)-2]
|
||||
if keep(name) {
|
||||
return m
|
||||
}
|
||||
v, ok := s.Get(name)
|
||||
if !ok {
|
||||
missing = append(missing, name)
|
||||
return m
|
||||
}
|
||||
return v
|
||||
})
|
||||
if len(missing) > 0 {
|
||||
return "", fmt.Errorf("tide: unresolved placeholder %q", missing[0])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// diffUpstreamJSON compares JSON semantically. A recorded string of the form
|
||||
// {{sha256:<hex>}} (optionally behind a data: URL prefix) matches a sent
|
||||
// base64 string whose decoded bytes hash to <hex>.
|
||||
func diffUpstreamJSON(want, got []byte) (Diff, bool) {
|
||||
wv, err := decodeJSON(want)
|
||||
if err != nil {
|
||||
return Diff{Path: "$", Expected: "valid JSON", Actual: err.Error()}, false
|
||||
}
|
||||
gv, err := decodeJSON(got)
|
||||
if err != nil {
|
||||
return Diff{Path: "$", Expected: formatValue(wv), Actual: err.Error()}, false
|
||||
}
|
||||
gv = resolveHashes(wv, gv)
|
||||
var diffs []Diff
|
||||
compareValue("$", wv, gv, &diffs)
|
||||
if len(diffs) > 0 {
|
||||
return diffs[0], false
|
||||
}
|
||||
return Diff{}, true
|
||||
}
|
||||
|
||||
func resolveHashes(want, got any) any {
|
||||
switch w := want.(type) {
|
||||
case map[string]any:
|
||||
g, ok := got.(map[string]any)
|
||||
if !ok {
|
||||
return got
|
||||
}
|
||||
out := make(map[string]any, len(g))
|
||||
for k, v := range g {
|
||||
if wv, ok := w[k]; ok {
|
||||
v = resolveHashes(wv, v)
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
g, ok := got.([]any)
|
||||
if !ok {
|
||||
return got
|
||||
}
|
||||
out := slices.Clone(g)
|
||||
for i := range min(len(w), len(g)) {
|
||||
out[i] = resolveHashes(w[i], g[i])
|
||||
}
|
||||
return out
|
||||
case string:
|
||||
m := hashPlaceholderRe.FindStringSubmatch(w)
|
||||
g, ok := got.(string)
|
||||
if m == nil || !ok || !strings.HasPrefix(g, m[1]) {
|
||||
return got
|
||||
}
|
||||
raw, ok := decodeBase64(strings.TrimPrefix(g, m[1]))
|
||||
if !ok {
|
||||
return got
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if hex.EncodeToString(sum[:]) == m[2] {
|
||||
return w
|
||||
}
|
||||
return got
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func decodeBase64(s string) ([]byte, bool) {
|
||||
for _, enc := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding, base64.URLEncoding, base64.RawURLEncoding} {
|
||||
if raw, err := enc.DecodeString(s); err == nil {
|
||||
return raw, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func clip(s string) string {
|
||||
const max = 120
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + fmt.Sprintf("... (%d bytes)", len(s))
|
||||
}
|
||||
|
||||
// WriteUpstream masks s and writes it to path (mode 0644). Every header value,
|
||||
// URL and body substring equal to a store variable value becomes {{name}};
|
||||
// a JSON string longer than 1024 characters that decodes as base64 (also
|
||||
// behind a data: URL prefix) becomes {{sha256:<hex of the decoded bytes>}}.
|
||||
// An Authorization or X-Api-Key request header that is not fully masked
|
||||
// refuses the write, naming the header but never its value.
|
||||
func WriteUpstream(path string, s UpstreamSidecar, store *Store) error {
|
||||
if s.Version == 0 {
|
||||
s.Version = CurrentVersion
|
||||
}
|
||||
masked, err := maskUpstream(s, store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateUpstream(masked); err != nil {
|
||||
return fmt.Errorf("tide: upstream %s: %w", path, err)
|
||||
}
|
||||
raw, err := yaml.MarshalWithOptions(masked, yaml.UseLiteralStyleIfMultiline(true))
|
||||
if err != nil {
|
||||
return fmt.Errorf("tide: marshal upstream: %w", err)
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("tide: create upstream dir: %w", err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".upstream-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("tide: upstream temp: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
if _, err := tmp.Write(raw); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmpName)
|
||||
return fmt.Errorf("tide: write upstream: %w", err)
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
_ = os.Remove(tmpName)
|
||||
return fmt.Errorf("tide: write upstream: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func maskUpstream(s UpstreamSidecar, store *Store) (UpstreamSidecar, error) {
|
||||
pairs := store.replacements()
|
||||
out := UpstreamSidecar{Version: s.Version, Exchanges: make([]UpstreamExchange, len(s.Exchanges))}
|
||||
for i, ex := range s.Exchanges {
|
||||
req := ex.Request
|
||||
req.URL = replaceAll(req.URL, pairs, true)
|
||||
req.Headers = scrubMap(req.Headers, pairs, true)
|
||||
if isJSONContentType(req.Headers) {
|
||||
req.Body = hashBase64Strings(req.Body)
|
||||
}
|
||||
req.Body = replaceAll(req.Body, pairs, false)
|
||||
if len(req.Parts) > 0 {
|
||||
req.Parts = slices.Clone(req.Parts)
|
||||
for j := range req.Parts {
|
||||
req.Parts[j].Value = replaceAll(req.Parts[j].Value, pairs, false)
|
||||
}
|
||||
}
|
||||
for name, v := range req.Headers {
|
||||
if isCredentialHeader(name) && !fullyMasked(v) {
|
||||
return UpstreamSidecar{}, fmt.Errorf("tide: upstream exchange %d: %s header is not masked by a vars entry; refusing to write a live credential", i, name)
|
||||
}
|
||||
}
|
||||
resp := ex.Response
|
||||
resp.Headers = scrubMap(resp.Headers, pairs, true)
|
||||
resp.Body = replaceAll(resp.Body, pairs, false)
|
||||
out.Exchanges[i] = UpstreamExchange{Request: req, Response: resp}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// credentialResidueRe is what may remain of a credential header once its
|
||||
// placeholders are removed: nothing, or a scheme word optionally followed by
|
||||
// one key= label (for example "Bearer" or "Discogs token=").
|
||||
var credentialResidueRe = regexp.MustCompile(`^(?:[A-Za-z]+(?:\s+[A-Za-z_]+=)?)?$`)
|
||||
|
||||
func fullyMasked(v string) bool {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return true
|
||||
}
|
||||
if !strings.HasSuffix(v, "}}") || !placeholderRe.MatchString(v) {
|
||||
return false
|
||||
}
|
||||
return credentialResidueRe.MatchString(strings.TrimSpace(placeholderRe.ReplaceAllString(v, "")))
|
||||
}
|
||||
|
||||
// hashBase64Strings replaces long base64 JSON string values in body with
|
||||
// {{sha256:<hex>}}, editing the text in place so the rest of the body keeps
|
||||
// its recorded bytes.
|
||||
func hashBase64Strings(body string) string {
|
||||
if len(body) <= upstreamHashMin {
|
||||
return body
|
||||
}
|
||||
v, err := decodeJSON([]byte(body))
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
var long []string
|
||||
collectStrings(v, &long)
|
||||
for _, str := range long {
|
||||
prefix, payload := "", str
|
||||
if strings.HasPrefix(str, "data:") {
|
||||
if i := strings.Index(str, ";base64,"); i > 0 {
|
||||
prefix, payload = str[:i+len(";base64,")], str[i+len(";base64,"):]
|
||||
}
|
||||
}
|
||||
raw, ok := decodeBase64(payload)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
placeholder := prefix + "{{sha256:" + hex.EncodeToString(sum[:]) + "}}"
|
||||
for _, enc := range jsonStringForms(str) {
|
||||
body = strings.ReplaceAll(body, enc, `"`+placeholder+`"`)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func collectStrings(v any, out *[]string) {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
for _, x := range t {
|
||||
collectStrings(x, out)
|
||||
}
|
||||
case []any:
|
||||
for _, x := range t {
|
||||
collectStrings(x, out)
|
||||
}
|
||||
case string:
|
||||
if len(t) > upstreamHashMin {
|
||||
*out = append(*out, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jsonStringForms returns the quoted JSON spellings a backend may have used
|
||||
// for s: Go's encoding and PHP's, which also escapes "/".
|
||||
func jsonStringForms(s string) []string {
|
||||
raw, _ := json.Marshal(s)
|
||||
forms := []string{string(raw), `"` + s + `"`}
|
||||
if esc := `"` + phpJSONEscape(s) + `"`; !slices.Contains(forms, esc) {
|
||||
forms = append(forms, esc)
|
||||
}
|
||||
return forms
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user