diff --git a/.planning/phases/06-http-routing-auth-groups-and-rate-limiting/06-04-SUMMARY.md b/.planning/phases/06-http-routing-auth-groups-and-rate-limiting/06-04-SUMMARY.md new file mode 100644 index 0000000..4c1f082 --- /dev/null +++ b/.planning/phases/06-http-routing-auth-groups-and-rate-limiting/06-04-SUMMARY.md @@ -0,0 +1,163 @@ +--- +phase: 06-http-routing-auth-groups-and-rate-limiting +plan: 04 +subsystem: api +tags: [ssrf, fetchguard, https, dial-control, netip, httptest] + +requires: + - phase: 01-framework-kernel-foundation + provides: compass.Config Lookup/Open layered YAML +provides: + - fetchguard.Fetch SSRF-guarded HTTPS GET with typed Reason errors + - fetchguard.Policy AllowHostsMode and PublicOnlyMode + - Dial-time private/reserved/CGNAT IP block via net.Dialer.Control + - Streaming io.LimitReader byte cap and no-follow redirects +affects: + - phase-12-manual-cover-url + - phase-14-discogs-cover-import + +tech-stack: + added: [] + patterns: + - Dial-time SSRF check on net.Dialer.Control after Addr.Unmap + - Host allow-list is exact or dotted-suffix ("." + allowed), never a raw suffix + - http.ErrUseLastResponse returns 3xx as Result; caller re-invokes Fetch to follow + - io.LimitReader(body, maxBytes+1) for streaming too_large, never post-hoc len() + +key-files: + created: + - fetchguard/fetch.go + - fetchguard/policy.go + - fetchguard/ip.go + - fetchguard/fetch_test.go + - fetchguard/ip_test.go + modified: [] + +key-decisions: + - "Redirects are not errors: Fetch returns the 3xx Result via http.ErrUseLastResponse so the caller must re-invoke Fetch to follow Location (T-06-17)" + - "isReservedOrPrivate does not Unmap; fetch.go's dial hook Unmaps before classifying, closing the v4-mapped-v6 metadata bypass" + - "Guarded Transport disables HTTP_PROXY so Control sees the target address, not a proxy" + - "Unexported Policy.tlsConfig and skipReservedCheck exist only so httptest TLS listeners on 127.0.0.1 can exercise redirect/byte-cap paths; production callers leave both unset" + +patterns-established: + - "SSRF outbound fetch lives in framework package fetchguard; app call sites (ManualCoverUrlFetcher, CoverImporter) arrive in later phases (D-13)" + - "Failure reasons are a closed *Error.Reason set matching PHP: invalid_url, scheme, unresolvable, private_ip, network_error, too_large" + +requirements-completed: [HTTP-07] + +duration: 8min +completed: 2026-09-19 +--- + +# Phase 6 Plan 04: SSRF-guarded outbound fetch helper Summary + +**Dial-time SSRF-guarded `fetchguard.Fetch` with AllowHosts/PublicOnly modes, PHP CIDR table, streaming byte cap, and typed failure reasons — helper and tests only** + +## Performance + +- **Duration:** 8 min +- **Started:** 2026-09-19T16:54:42Z +- **Completed:** 2026-09-19T17:02:28Z +- **Tasks:** 2 +- **Files modified:** 5 + +## Accomplishments + +- Ported `ManualCoverUrlFetcher.php` `PRIVATE_V4_CIDRS` / `PRIVATE_V6_PREFIXES` onto `netip.Prefix`, including CGNAT `100.64.0.0/10` and cloud-metadata `169.254.0.0/16`, with Unmap documented as the dial-hook caller's job. +- Shipped `fetchguard.Fetch`: https-only, host allow-list (exact + dotted-suffix so `evil-discogs.com` cannot match `discogs.com`), always-on private-IP block at `net.Dialer.Control` (closes PHP's admitted DNS-rebinding TOCTOU), no automatic redirects, streaming `io.LimitReader` cap, config defaults 10 MiB / 10s with explicit zero/negative as an error (D-14). +- Zero production call sites — no `ManualCoverUrlFetcher` or `CoverImporter` wiring (D-13). + +## Task Commits + +Each task was committed atomically (TDD RED then GREEN): + +1. **Task 1: Private/reserved IP classification table** + - `f7e7ca7` test(06-04): add failing test for private/reserved IP table + - `d3630e7` feat(06-04): implement private/reserved IP classification table +2. **Task 2: Policy, defaults, and the dial-time-guarded Fetch entry point** + - `bae9de8` test(06-04): add failing tests for SSRF-guarded Fetch + - `7923fe8` feat(06-04): implement dial-time SSRF-guarded Fetch + +**Plan metadata:** (this commit) docs(06-04): complete SSRF-guarded fetch helper plan + +## Files Created/Modified + +- `fetchguard/ip.go` — `isReservedOrPrivate` CIDR table (PHP constants, fail-closed on invalid addr) +- `fetchguard/ip_test.go` — table-driven cases including Unmap, CGNAT, metadata, 172.16/12 boundaries +- `fetchguard/policy.go` — Mode, Reason, Policy, Error, Defaults, DefaultsFromConfig via Lookup +- `fetchguard/fetch.go` — `Fetch` with dial Control, ErrUseLastResponse, LimitReader, typed errors +- `fetchguard/fetch_test.go` — httptest TLS cases for scheme/host/private_ip/redirect/too_large/config + +## Decisions Made + +- Pin redirect behavior to `http.ErrUseLastResponse`: a 3xx is a non-error `Result` so following Location requires a new `Fetch` through the same guard (T-06-17). +- `isReservedOrPrivate` classifies an already-Unmap()'d `netip.Addr`; the dial hook Unmaps, matching Pitfall 6. +- Disable `HTTP_PROXY` on the guarded transport so Control observes the target IP, not a proxy hop. +- Unexported `Policy.tlsConfig` / `skipReservedCheck` let tests talk to loopback httptest servers without weakening production checks. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 3 - Blocking] Unexported test-only Policy fields for httptest loopback** +- **Found during:** Task 2 (Fetch entry point) +- **Issue:** httptest TLS servers bind 127.0.0.1, which the production dial-time private-IP block correctly rejects. Redirect and streaming-cap tests need a real connection to that listener; mocking `http.RoundTripper` would skip the thing under test for private_ip but would also skip LimitReader/CheckRedirect. +- **Fix:** Unexported `Policy.tlsConfig` and `skipReservedCheck`, set only from `fetch_test.go` via `withTestLoopback`. Private-IP tests leave both unset and assert `ReasonPrivateIP` against the real Control hook. +- **Files modified:** `fetchguard/policy.go`, `fetchguard/fetch.go`, `fetchguard/fetch_test.go` +- **Verification:** `go test ./fetchguard/... -race` — private_ip subtests fail closed; redirect/too_large pass against real TLS listeners +- **Committed in:** `7923fe8` (Task 2 GREEN) + +**2. [Rule 2 - Missing Critical] Disable HTTP_PROXY on the guarded transport** +- **Found during:** Task 2 (Fetch entry point) +- **Issue:** Honoring `HTTP_PROXY` would make Control see the proxy address, not the user-supplied target — an SSRF bypass. +- **Fix:** Set `Transport.Proxy` to nil (no proxy) on the per-call client. +- **Files modified:** `fetchguard/fetch.go` +- **Verification:** private_ip tests still hit Control on 127.0.0.1; `go vet ./...` clean +- **Committed in:** `7923fe8` (Task 2 GREEN) + +--- + +**Total deviations:** 2 auto-fixed (1 blocking, 1 missing-critical) +**Impact on plan:** Both required for correctness/security of the helper and its httptest proof. No call sites added. No scope creep. + +## Issues Encountered + +Grok worktree clone had `.git` as a directory and HEAD on `master`. Created `worktree-agent-01a0ba96` from `ecab09c` without rewinding `master`, then committed only on that branch. + +## User Setup Required + +None - no external service configuration required. + +## Next Phase Readiness + +- `fetchguard.Fetch` is ready for Phase 12 `ManualCoverUrlFetcher` (PublicOnlyMode) and Phase 14 Discogs `CoverImporter` (AllowHostsMode) call sites. +- Callers must pass an already-resolved `Policy` (or nil cfg to use 10 MiB / 10s). Following a 3xx means a new `Fetch` of Location. +- No blockers. + +## TDD Gate Compliance + +Per-task RED then GREEN commits are present: + +1. `f7e7ca7` test(06-04) (RED Task 1) +2. `d3630e7` feat(06-04) (GREEN Task 1) +3. `bae9de8` test(06-04) (RED Task 2) +4. `7923fe8` feat(06-04) (GREEN Task 2) + +## Self-Check: PASSED + +- FOUND: `fetchguard/ip.go` +- FOUND: `fetchguard/ip_test.go` +- FOUND: `fetchguard/policy.go` +- FOUND: `fetchguard/fetch.go` +- FOUND: `fetchguard/fetch_test.go` +- FOUND: `f7e7ca7` +- FOUND: `d3630e7` +- FOUND: `bae9de8` +- FOUND: `7923fe8` +- `go vet ./...` clean +- `go test ./fetchguard/... -race` pass +- `go test ./... -short` pass + +--- +*Phase: 06-http-routing-auth-groups-and-rate-limiting* +*Completed: 2026-09-19* diff --git a/fetchguard/fetch.go b/fetchguard/fetch.go new file mode 100644 index 0000000..fd1d81d --- /dev/null +++ b/fetchguard/fetch.go @@ -0,0 +1,175 @@ +package fetchguard + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "syscall" + "time" + + "git.golem15.com/golem15/summercms/compass" +) + +// Result is a successful (including non-2xx, including 3xx) Fetch response. +type Result struct { + Body []byte + ContentType string + StatusCode int +} + +var errPrivateIP = errors.New("private_ip") + +// Fetch validates url against policy, resolves defaults for any zero +// MaxBytes/Timeout via DefaultsFromConfig (or Defaults() when cfg is nil), +// then performs the guarded HTTPS GET. +// +// Redirects are never followed: CheckRedirect returns http.ErrUseLastResponse, +// so a 3xx response is returned as a non-error Result. Callers that want to +// follow Location must re-invoke Fetch, which re-runs the same guard. +// +// A non-nil error is always *Error with a Reason from the closed set. +func Fetch(ctx context.Context, rawURL string, policy Policy, cfg *compass.Config) (*Result, error) { + if ctx == nil { + ctx = context.Background() + } + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return nil, &Error{Reason: ReasonInvalidURL, Err: err} + } + if strings.ToLower(parsed.Scheme) != "https" { + return nil, &Error{Reason: ReasonScheme} + } + host := parsed.Hostname() + if policy.Mode == AllowHostsMode && !hostAllowed(host, policy.AllowHosts) { + return nil, &Error{Reason: ReasonInvalidURL} + } + + maxBytes, timeout, err := resolveLimits(policy, cfg) + if err != nil { + return nil, err + } + + client := &http.Client{ + Timeout: timeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + // User-supplied URLs must not be forwarded through HTTP_PROXY: + // the dial-time IP check would then see the proxy, not the target. + Proxy: nil, + DialContext: (&net.Dialer{ + Timeout: timeout, + Control: dialControl(policy), + }).DialContext, + TLSClientConfig: policy.tlsConfig, + DisableKeepAlives: true, + ForceAttemptHTTP2: true, + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + return nil, &Error{Reason: ReasonInvalidURL, Err: err} + } + resp, err := client.Do(req) + if err != nil { + return nil, mapTransportError(err) + } + defer resp.Body.Close() + + data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, &Error{Reason: ReasonNetworkError, Err: err} + } + if int64(len(data)) == maxBytes+1 { + return nil, &Error{Reason: ReasonTooLarge} + } + return &Result{ + Body: data, + ContentType: resp.Header.Get("Content-Type"), + StatusCode: resp.StatusCode, + }, nil +} + +func resolveLimits(policy Policy, cfg *compass.Config) (int64, time.Duration, error) { + maxBytes := policy.MaxBytes + timeout := policy.Timeout + if maxBytes <= 0 || timeout <= 0 { + var ( + defMax int64 + defTO time.Duration + err error + ) + if cfg != nil { + defMax, defTO, err = DefaultsFromConfig(cfg) + if err != nil { + return 0, 0, &Error{Reason: ReasonInvalidURL, Err: err} + } + } else { + defMax, defTO = Defaults() + } + if maxBytes <= 0 { + maxBytes = defMax + } + if timeout <= 0 { + timeout = defTO + } + } + if maxBytes <= 0 || timeout <= 0 { + return 0, 0, &Error{Reason: ReasonInvalidURL, Err: fmt.Errorf("max bytes and timeout must be positive")} + } + return maxBytes, timeout, nil +} + +func hostAllowed(host string, allowed []string) bool { + host = strings.ToLower(host) + for _, a := range allowed { + a = strings.ToLower(a) + if a == "" { + continue + } + if host == a || strings.HasSuffix(host, "."+a) { + return true + } + } + return false +} + +func dialControl(policy Policy) func(network, address string, c syscall.RawConn) error { + return func(network, address string, c syscall.RawConn) error { + if policy.skipReservedCheck { + return nil + } + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + addr, err := netip.ParseAddr(host) + if err != nil { + return fmt.Errorf("fetchguard: unparseable dial address %q: %w", host, err) + } + addr = addr.Unmap() + if isReservedOrPrivate(addr) { + return errPrivateIP + } + return nil + } +} + +func mapTransportError(err error) *Error { + if errors.Is(err, errPrivateIP) { + return &Error{Reason: ReasonPrivateIP, Err: err} + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return &Error{Reason: ReasonUnresolvable, Err: err} + } + return &Error{Reason: ReasonNetworkError, Err: err} +} diff --git a/fetchguard/fetch_test.go b/fetchguard/fetch_test.go new file mode 100644 index 0000000..d6b75dd --- /dev/null +++ b/fetchguard/fetch_test.go @@ -0,0 +1,276 @@ +package fetchguard + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "git.golem15.com/golem15/summercms/compass" +) + +func TestFetchMalformedURL(t *testing.T) { + _, err := Fetch(t.Context(), "not a url", Policy{Mode: PublicOnlyMode}, nil) + if reasonFrom(t, err) != ReasonInvalidURL { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonInvalidURL) + } +} + +func TestFetchHTTPSchemeRejectedWithoutIO(t *testing.T) { + var hits atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + hits.Add(1) + })) + t.Cleanup(srv.Close) + + _, err := Fetch(t.Context(), srv.URL, Policy{Mode: PublicOnlyMode}, nil) + if reasonFrom(t, err) != ReasonScheme { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonScheme) + } + if hits.Load() != 0 { + t.Fatal("http URL must not cause network I/O") + } +} + +func TestFetchAllowHostsRejectsUnknownHostBeforeDial(t *testing.T) { + var hits atomic.Int64 + srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + hits.Add(1) + })) + t.Cleanup(srv.Close) + + _, err := Fetch(t.Context(), srv.URL, Policy{ + Mode: AllowHostsMode, + AllowHosts: []string{"discogs.com"}, + }, nil) + if reasonFrom(t, err) != ReasonInvalidURL { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonInvalidURL) + } + if hits.Load() != 0 { + t.Fatal("host allow-list miss must not dial") + } +} + +func TestFetchAllowHostsRejectsDottedSuffixBypass(t *testing.T) { + _, err := Fetch(t.Context(), "https://evil-discogs.com/cover.jpg", Policy{ + Mode: AllowHostsMode, + AllowHosts: []string{"discogs.com"}, + }, nil) + if reasonFrom(t, err) != ReasonInvalidURL { + t.Fatalf("evil-discogs.com must not match discogs.com, reason = %q", reasonFrom(t, err)) + } +} + +func TestFetchPrivateIPBlockedInBothModes(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("handler must not run for a private dial") + })) + t.Cleanup(srv.Close) + + t.Run("AllowHostsMode", func(t *testing.T) { + _, err := Fetch(t.Context(), srv.URL, Policy{ + Mode: AllowHostsMode, + AllowHosts: []string{"127.0.0.1"}, + Timeout: 2 * time.Second, + MaxBytes: 1024, + }, nil) + if reasonFrom(t, err) != ReasonPrivateIP { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonPrivateIP) + } + }) + t.Run("PublicOnlyMode", func(t *testing.T) { + _, err := Fetch(t.Context(), srv.URL, Policy{ + Mode: PublicOnlyMode, + Timeout: 2 * time.Second, + MaxBytes: 1024, + }, nil) + if reasonFrom(t, err) != ReasonPrivateIP { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonPrivateIP) + } + }) +} + +func TestFetchDoesNotFollowRedirect(t *testing.T) { + var followed atomic.Bool + mux := http.NewServeMux() + mux.HandleFunc("/target", func(w http.ResponseWriter, r *http.Request) { + followed.Store(true) + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "/target") + w.WriteHeader(http.StatusFound) + }) + srv := httptest.NewTLSServer(mux) + t.Cleanup(srv.Close) + + res, err := Fetch(t.Context(), srv.URL, withTestLoopback(srv, Policy{ + Mode: PublicOnlyMode, + MaxBytes: 1024, + Timeout: 5 * time.Second, + }), nil) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.StatusCode != http.StatusFound { + t.Fatalf("status = %d, want %d (3xx returned, not followed)", res.StatusCode, http.StatusFound) + } + if followed.Load() { + t.Fatal("redirect must not be followed") + } +} + +func TestFetchTooLargeIsStreaming(t *testing.T) { + const maxBytes int64 = 256 + var written atomic.Int64 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + buf := make([]byte, 64) + for written.Load() < 8<<20 { + n, err := w.Write(buf) + written.Add(int64(n)) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + if err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + + _, err := Fetch(t.Context(), srv.URL, withTestLoopback(srv, Policy{ + Mode: PublicOnlyMode, + MaxBytes: maxBytes, + Timeout: 5 * time.Second, + }), nil) + if reasonFrom(t, err) != ReasonTooLarge { + t.Fatalf("reason = %q, want %s", reasonFrom(t, err), ReasonTooLarge) + } + // TCP/HTTP buffering can write a little past maxBytes+1; the client must + // not have pulled an unbounded body first (the handler would hit 8MiB). + if got := written.Load(); got > 64<<10 { + t.Fatalf("server wrote %d bytes, client appears to have buffered unbounded body", got) + } + if written.Load() < maxBytes+1 { + t.Fatalf("server wrote %d bytes, want at least maxBytes+1 so the cap was hit", written.Load()) + } +} + +func TestFetchZeroPolicyUsesDefaults(t *testing.T) { + max, timeout := Defaults() + if max != 10*1024*1024 { + t.Fatalf("Defaults maxBytes = %d, want 10MiB", max) + } + if timeout != 10*time.Second { + t.Fatalf("Defaults timeout = %s, want 10s", timeout) + } + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + io.WriteString(w, "ok") + })) + t.Cleanup(srv.Close) + + res, err := Fetch(t.Context(), srv.URL, withTestLoopback(srv, Policy{Mode: PublicOnlyMode}), nil) + if err != nil { + t.Fatalf("Fetch with zero MaxBytes/Timeout and nil cfg: %v", err) + } + if string(res.Body) != "ok" { + t.Fatalf("body = %q, want ok", res.Body) + } +} + +func TestDefaultsFromConfigExplicitZeroIsError(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "http.yaml"), []byte("fetch:\n max_bytes: 0\n timeout_seconds: 10\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatalf("Open: %v", err) + } + if _, ok := cfg.Lookup("http.fetch.max_bytes"); !ok { + t.Fatal("expected http.fetch.max_bytes to be present in loaded YAML") + } + _, _, err = DefaultsFromConfig(cfg) + if err == nil { + t.Fatal("explicit http.fetch.max_bytes: 0 must be an error, not a silent fallback") + } +} + +func TestDefaultsFromConfigExplicitNegativeIsError(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "http.yaml"), []byte("fetch:\n max_bytes: 10485760\n timeout_seconds: -1\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatalf("Open: %v", err) + } + _, _, err = DefaultsFromConfig(cfg) + if err == nil { + t.Fatal("explicit http.fetch.timeout_seconds: -1 must be an error") + } +} + +func TestDefaultsFromConfigAbsentFallsBack(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("name: t\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{Dir: dir, Environ: []string{}}) + if err != nil { + t.Fatalf("Open: %v", err) + } + max, timeout, err := DefaultsFromConfig(cfg) + if err != nil { + t.Fatalf("DefaultsFromConfig: %v", err) + } + wantMax, wantTO := Defaults() + if max != wantMax || timeout != wantTO { + t.Fatalf("got %d %s, want Defaults() %d %s", max, timeout, wantMax, wantTO) + } +} + +func TestErrorUnwrap(t *testing.T) { + inner := errors.New("dial tcp") + err := &Error{Reason: ReasonNetworkError, Err: inner} + if err.Error() == "" { + t.Fatal("Error() must be non-empty") + } + if !errors.Is(err, inner) { + t.Fatal("Unwrap must expose the inner error") + } +} + +func reasonFrom(t *testing.T, err error) Reason { + t.Helper() + if err == nil { + t.Fatal("expected error") + } + var fe *Error + if !errors.As(err, &fe) { + t.Fatalf("err = %v (%T), want *Error", err, err) + } + return fe.Reason +} + +// withTestLoopback trusts the httptest TLS cert and skips the reserved-IP +// dial check so redirect/byte-cap tests can exercise a real listener on +// 127.0.0.1. Production Policy values leave both fields unset. +func withTestLoopback(srv *httptest.Server, p Policy) Policy { + tr, ok := srv.Client().Transport.(*http.Transport) + if !ok { + panic("httptest client transport is not *http.Transport") + } + p.tlsConfig = tr.TLSClientConfig + p.skipReservedCheck = true + return p +} diff --git a/fetchguard/ip.go b/fetchguard/ip.go new file mode 100644 index 0000000..06b77ab --- /dev/null +++ b/fetchguard/ip.go @@ -0,0 +1,45 @@ +package fetchguard + +import "net/netip" + +// privateV4 is a literal port of ManualCoverUrlFetcher.php PRIVATE_V4_CIDRS. +var privateV4 = []netip.Prefix{ + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.168.0.0/16"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("0.0.0.0/8"), +} + +// privateV6 is a literal port of PRIVATE_V6_PREFIXES. PHP lists bare "::1" +// as a prefix-less loopback literal; it is expressed here as ::1/128 so +// Prefix.Contains works uniformly with the CIDR entries. +var privateV6 = []netip.Prefix{ + netip.MustParsePrefix("::1/128"), + netip.MustParsePrefix("fe80::/10"), + netip.MustParsePrefix("fc00::/7"), +} + +// isReservedOrPrivate classifies addr against the PHP private/loopback/ +// reserved/CGNAT table. The caller must pass an already-Unmap()-ed address +// (fetch.go's dial hook); this function does not Unmap. +func isReservedOrPrivate(addr netip.Addr) bool { + if !addr.IsValid() { + return true + } + if addr.IsMulticast() || addr.IsUnspecified() { + return true + } + table := privateV4 + if !addr.Is4() { + table = privateV6 + } + for _, prefix := range table { + if prefix.Contains(addr) { + return true + } + } + return false +} diff --git a/fetchguard/ip_test.go b/fetchguard/ip_test.go new file mode 100644 index 0000000..4c5d270 --- /dev/null +++ b/fetchguard/ip_test.go @@ -0,0 +1,54 @@ +package fetchguard + +import ( + "net/netip" + "testing" +) + +func TestIsReservedOrPrivate(t *testing.T) { + tests := []struct { + name string + ip string + want bool + // unmap documents that IPv4-mapped IPv6 literals are the CALLER's + // responsibility to Unmap() before classification (fetch.go dial hook). + unmap bool + }{ + {name: "loopback v4", ip: "127.0.0.1", want: true}, + {name: "rfc1918 10/8", ip: "10.1.2.3", want: true}, + {name: "rfc1918 172.16/12 start", ip: "172.16.0.1", want: true}, + {name: "rfc1918 172.16/12 end", ip: "172.31.255.255", want: true}, + {name: "rfc1918 192.168/16", ip: "192.168.1.1", want: true}, + {name: "cloud metadata link-local", ip: "169.254.169.254", want: true}, + {name: "cgnat 100.64/10", ip: "100.64.0.1", want: true}, + {name: "this-network 0.0.0.0/8", ip: "0.0.0.1", want: true}, + {name: "public 8.8.8.8", ip: "8.8.8.8", want: false}, + {name: "public 1.1.1.1", ip: "1.1.1.1", want: false}, + {name: "public 93.184.216.34", ip: "93.184.216.34", want: false}, + {name: "loopback v6", ip: "::1", want: true}, + {name: "link-local v6", ip: "fe80::1", want: true}, + {name: "unique-local v6", ip: "fc00::1", want: true}, + {name: "public v6", ip: "2606:4700:4700::1111", want: false}, + {name: "v4-mapped metadata after Unmap", ip: "::ffff:169.254.169.254", want: true, unmap: true}, + {name: "multicast v4", ip: "224.0.0.1", want: true}, + {name: "unspecified v4", ip: "0.0.0.0", want: true}, + {name: "just below 172.16.0.0/12", ip: "172.15.255.255", want: false}, + {name: "just above 172.16.0.0/12", ip: "172.32.0.0", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + addr, err := netip.ParseAddr(tt.ip) + if err != nil { + t.Fatalf("ParseAddr(%q): %v", tt.ip, err) + } + if tt.unmap { + addr = addr.Unmap() + } + got := isReservedOrPrivate(addr) + if got != tt.want { + t.Fatalf("isReservedOrPrivate(%s) = %v, want %v", addr, got, tt.want) + } + }) + } +} diff --git a/fetchguard/policy.go b/fetchguard/policy.go new file mode 100644 index 0000000..b861010 --- /dev/null +++ b/fetchguard/policy.go @@ -0,0 +1,147 @@ +package fetchguard + +import ( + "crypto/tls" + "fmt" + "math" + "time" + + "git.golem15.com/golem15/summercms/compass" +) + +// Mode selects host-allow-list vs any-public-host. The private/loopback/ +// reserved IP block is always on regardless of Mode (D-11). +type Mode int + +const ( + AllowHostsMode Mode = iota + PublicOnlyMode +) + +// Reason is the closed set of Fetch failure reasons, matching PHP's +// invalid_url/scheme/unresolvable/private_ip/network_error/too_large family. +type Reason string + +const ( + ReasonInvalidURL Reason = "invalid_url" + ReasonScheme Reason = "scheme" + ReasonUnresolvable Reason = "unresolvable" + ReasonPrivateIP Reason = "private_ip" + ReasonNetworkError Reason = "network_error" + ReasonTooLarge Reason = "too_large" +) + +// Policy is supplied per call. +type Policy struct { + Mode Mode + AllowHosts []string // exact or dotted-suffix match; used in AllowHostsMode + MaxBytes int64 // 0 means use the config/framework default, never unlimited + Timeout time.Duration + + // tlsConfig, if set, is Transport.TLSClientConfig so tests can trust an + // httptest certificate. Production callers leave it nil. + tlsConfig *tls.Config + // skipReservedCheck disables the dial-time private-IP block so tests can + // exercise a real httptest listener on 127.0.0.1. Production callers + // leave it false. + skipReservedCheck bool +} + +// Error carries the typed Reason plus the underlying error for logging. +type Error struct { + Reason Reason + Err error +} + +func (e *Error) Error() string { + if e == nil { + return "fetchguard: error" + } + if e.Err != nil { + return fmt.Sprintf("fetchguard: %s: %v", e.Reason, e.Err) + } + return fmt.Sprintf("fetchguard: %s", e.Reason) +} + +func (e *Error) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +// Defaults are the framework fallback: 10 MiB, 10s, matching PHP. +func Defaults() (maxBytes int64, timeout time.Duration) { + return 10 * 1024 * 1024, 10 * time.Second +} + +// DefaultsFromConfig reads http.fetch.max_bytes / http.fetch.timeout_seconds +// from cfg, falling back to Defaults() for absent keys. An explicitly +// configured zero or negative value is an error (D-14). +func DefaultsFromConfig(cfg *compass.Config) (maxBytes int64, timeout time.Duration, err error) { + maxBytes, timeout = Defaults() + if cfg == nil { + return maxBytes, timeout, nil + } + if v, ok := cfg.Lookup("http.fetch.max_bytes"); ok { + n, err := configInt64("http.fetch.max_bytes", v) + if err != nil { + return 0, 0, err + } + if n <= 0 { + return 0, 0, fmt.Errorf("fetchguard: http.fetch.max_bytes must be positive, got %d", n) + } + maxBytes = n + } + if v, ok := cfg.Lookup("http.fetch.timeout_seconds"); ok { + n, err := configInt64("http.fetch.timeout_seconds", v) + if err != nil { + return 0, 0, err + } + if n <= 0 { + return 0, 0, fmt.Errorf("fetchguard: http.fetch.timeout_seconds must be positive, got %d", n) + } + timeout = time.Duration(n) * time.Second + } + return maxBytes, timeout, nil +} + +func configInt64(path string, v any) (int64, error) { + switch n := v.(type) { + case int: + return int64(n), nil + case int8: + return int64(n), nil + case int16: + return int64(n), nil + case int32: + return int64(n), nil + case int64: + return n, nil + case uint: + return int64(n), nil + case uint8: + return int64(n), nil + case uint16: + return int64(n), nil + case uint32: + return int64(n), nil + case uint64: + if n > math.MaxInt64 { + return 0, fmt.Errorf("fetchguard: %s overflows int64", path) + } + return int64(n), nil + case float64: + if math.Trunc(n) != n { + return 0, fmt.Errorf("fetchguard: %s is not an integer", path) + } + if n > math.MaxInt64 || n < math.MinInt64 { + return 0, fmt.Errorf("fetchguard: %s overflows int64", path) + } + return int64(n), nil + case float32: + return configInt64(path, float64(n)) + default: + return 0, fmt.Errorf("fetchguard: %s has unexpected type %T", path, v) + } +}