package fetchguard_test import ( "context" "errors" "fmt" "io" "net/http" "net/http/httptest" "net/netip" "strings" "testing" "time" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/fetchguard" ) func ExampleFetch() { ctx := context.Background() // Only the application's image host, at most 5 MiB within 5 seconds. images := fetchguard.Policy{ Mode: fetchguard.AllowHostsMode, AllowHosts: []string{"images.example.com"}, MaxBytes: 5 << 20, Timeout: 5 * time.Second, } // Any public host, for a URL a user pasted. public := fetchguard.Policy{Mode: fetchguard.PublicOnlyMode} for _, c := range []struct { url string policy fetchguard.Policy }{ {"http://images.example.com/cover.jpg", images}, {"https://cdn.attacker.example/cover.jpg", images}, {"https://127.0.0.1/admin", public}, {"https://169.254.169.254/latest/meta-data/", public}, {"https://[::ffff:10.0.0.1]/", public}, {"https://%zz", public}, } { // The last argument is the application's config (app.Config), for // limits the policy leaves at zero; nil uses the framework defaults. _, err := fetchguard.Fetch(ctx, c.url, c.policy, nil) var fe *fetchguard.Error if errors.As(err, &fe) { fmt.Println(fe.Reason, c.url) } } fmt.Println(fetchguard.Defaults()) // Output: // scheme http://images.example.com/cover.jpg // invalid_url https://cdn.attacker.example/cover.jpg // private_ip https://127.0.0.1/admin // private_ip https://169.254.169.254/latest/meta-data/ // private_ip https://[::ffff:10.0.0.1]/ // invalid_url https://%zz // 10485760 10s } // stubVendor stands in for a vendor API in the example; tests use the tide // upstream fake the same way. type stubVendor struct{} func (stubVendor) RoundTrip(r *http.Request) (*http.Response, error) { body := `{"id":42}` if r.Header.Get("Authorization") != "Bearer example-token" { body = `{"error":"unauthorized"}` } return &http.Response{ StatusCode: http.StatusCreated, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(strings.NewReader(body)), Request: r, }, nil } func ExampleClient_PostJSON() { client, err := fetchguard.NewClient(fetchguard.Policy{ Mode: fetchguard.AllowHostsMode, AllowHosts: []string{"api.example.com"}, Timeout: 10 * time.Second, }, nil) if err != nil { panic(err) } header := http.Header{} header.Set("Authorization", fetchguard.Bearer("example-token")) // Production code passes its own context; the example routes the call to // a stub instead of the network. ctx := fetchguard.WithTransport(context.Background(), stubVendor{}) res, err := client.PostJSON(ctx, "https://api.example.com/v1/items", header, map[string]string{"name": "widget"}) if err != nil { panic(err) } fmt.Println(res.StatusCode, string(res.Body)) // Output: // 201 {"id":42} } func ExampleIsPrivateAddr() { for _, ip := range []string{"8.8.8.8", "10.0.0.7", "::ffff:127.0.0.1"} { fmt.Println(ip, fetchguard.IsPrivateAddr(netip.MustParseAddr(ip))) } // Output: // 8.8.8.8 false // 10.0.0.7 true // ::ffff:127.0.0.1 true } // errRejectedURL answers an image URL that points inside the network. var errRejectedURL = errors.New("image URL rejected") // fetchImage downloads an image from imageHost or one of its subdomains // (images.example.com in production). cfg is the application's config // (app.Config). func fetchImage(ctx context.Context, cfg *compass.Config, imageHost, imageURL string) ([]byte, error) { // docs:start fetch policy := fetchguard.Policy{ Mode: fetchguard.AllowHostsMode, AllowHosts: []string{imageHost}, MaxBytes: 5 << 20, Timeout: 5 * time.Second, } res, err := fetchguard.Fetch(ctx, imageURL, policy, cfg) if err != nil { var fe *fetchguard.Error if errors.As(err, &fe) && fe.Reason == fetchguard.ReasonPrivateIP { return nil, errRejectedURL } return nil, err } if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("image fetch: status %d", res.StatusCode) } image := res.Body // docs:end fetch return image, nil } // TestDocsFetchImage runs fetchImage offline: an allowed host that resolves // into the network is rejected at dial time, a host outside the list and a // plain http URL never reach the network. func TestDocsFetchImage(t *testing.T) { site := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Error("the guard let a request through to a loopback server") })) defer site.Close() if _, err := fetchImage(t.Context(), nil, "127.0.0.1", site.URL+"/cover.jpg"); !errors.Is(err, errRejectedURL) { t.Fatalf("loopback image = %v, want errRejectedURL", err) } for url, want := range map[string]fetchguard.Reason{ "https://cdn.attacker.example/cover.jpg": fetchguard.ReasonInvalidURL, "http://images.example.com/cover.jpg": fetchguard.ReasonScheme, } { _, err := fetchImage(t.Context(), nil, "images.example.com", url) var fe *fetchguard.Error if !errors.As(err, &fe) || fe.Reason != want { t.Fatalf("%s: %v, want %s", url, err, want) } } } // vendorFunc is a vendor API stand-in for WithTransport. type vendorFunc func(*http.Request) (*http.Response, error) func (f vendorFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } // syncItems sends payload to the vendor API, then uploads a report file. func syncItems(ctx context.Context, cfg *compass.Config, token string, payload any, f io.Reader) (*fetchguard.Result, error) { // docs:start client vendor, err := fetchguard.NewClient(fetchguard.Policy{ Mode: fetchguard.AllowHostsMode, AllowHosts: []string{"api.example.com"}, Timeout: 10 * time.Second, }, cfg) if err != nil { return nil, err } header := http.Header{} header.Set("Authorization", fetchguard.Bearer(token)) res, err := vendor.PostJSON(ctx, "https://api.example.com/v1/items", header, payload) if err != nil { return nil, err } if res.StatusCode == http.StatusTooManyRequests { wait := res.Header.Get("Retry-After") return nil, fmt.Errorf("vendor busy, retry after %s seconds", wait) } // Multipart upload: fields first, then files, each in slice order. res, err = vendor.PostMultipart(ctx, "https://api.example.com/v1/files", header, []fetchguard.FormField{{Name: "title", Value: "Report"}}, []fetchguard.FormFile{{Field: "file", Filename: "report.pdf", ContentType: "application/pdf", Body: f}}, ) // docs:end client return res, err } // TestDocsSyncItems runs syncItems against a stub vendor: the JSON call and // the upload carry the bearer token, and a 429 stops before the upload. func TestDocsSyncItems(t *testing.T) { var calls []string busy := false vendor := vendorFunc(func(r *http.Request) (*http.Response, error) { body, _ := io.ReadAll(r.Body) calls = append(calls, r.URL.Path+" "+r.Header.Get("Authorization")) status, header := http.StatusCreated, http.Header{} if busy { status = http.StatusTooManyRequests header.Set("Retry-After", "30") } if r.URL.Path == "/v1/files" && (!strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") || !strings.Contains(string(body), `filename="report.pdf"`) || !strings.Contains(string(body), "Report")) { status = http.StatusBadRequest } return &http.Response{StatusCode: status, Header: header, Body: io.NopCloser(strings.NewReader(`{}`)), Request: r}, nil }) ctx := fetchguard.WithTransport(t.Context(), vendor) res, err := syncItems(ctx, nil, "example-token", map[string]string{"name": "widget"}, strings.NewReader("%PDF-1.7")) if err != nil || res.StatusCode != http.StatusCreated { t.Fatalf("syncItems = %+v, %v", res, err) } want := []string{"/v1/items Bearer example-token", "/v1/files Bearer example-token"} if strings.Join(calls, "|") != strings.Join(want, "|") { t.Fatalf("calls = %q", calls) } calls, busy = nil, true if _, err := syncItems(ctx, nil, "example-token", nil, strings.NewReader("")); err == nil || !strings.Contains(err.Error(), "retry after 30 seconds") || len(calls) != 1 { t.Fatalf("busy vendor: %v after %q", err, calls) } }