- cooler: ExampleFromApp and the articlesController purge declarations; AllowExport run by TestDocsAllowExport on miniredis - diary: blogLog and ExampleFor; the generated main's run function, run by TestDocsRun in a temp directory with diary.Reset on cleanup - festival: PostPublished/ExampleBus_Fire and PostFormExtended/Collected/ ExampleBus_Collect, with a TestDocsDeclarations for Collected - fetchguard: fetch region run by TestDocsFetchImage (loopback refused as private_ip), client region run by TestDocsSyncItems on a stub transport
239 lines
8.0 KiB
Go
239 lines
8.0 KiB
Go
package fetchguard_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/netip"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/compass"
|
|
"git.golem15.com/golem15/summercms/modules/fetchguard"
|
|
)
|
|
|
|
func ExampleFetch() {
|
|
ctx := context.Background()
|
|
// Only the application's image host, at most 5 MiB within 5 seconds.
|
|
images := fetchguard.Policy{
|
|
Mode: fetchguard.AllowHostsMode,
|
|
AllowHosts: []string{"images.example.com"},
|
|
MaxBytes: 5 << 20,
|
|
Timeout: 5 * time.Second,
|
|
}
|
|
// Any public host, for a URL a user pasted.
|
|
public := fetchguard.Policy{Mode: fetchguard.PublicOnlyMode}
|
|
|
|
for _, c := range []struct {
|
|
url string
|
|
policy fetchguard.Policy
|
|
}{
|
|
{"http://images.example.com/cover.jpg", images},
|
|
{"https://cdn.attacker.example/cover.jpg", images},
|
|
{"https://127.0.0.1/admin", public},
|
|
{"https://169.254.169.254/latest/meta-data/", public},
|
|
{"https://[::ffff:10.0.0.1]/", public},
|
|
{"https://%zz", public},
|
|
} {
|
|
// The last argument is the application's config (app.Config), for
|
|
// limits the policy leaves at zero; nil uses the framework defaults.
|
|
_, err := fetchguard.Fetch(ctx, c.url, c.policy, nil)
|
|
var fe *fetchguard.Error
|
|
if errors.As(err, &fe) {
|
|
fmt.Println(fe.Reason, c.url)
|
|
}
|
|
}
|
|
fmt.Println(fetchguard.Defaults())
|
|
// Output:
|
|
// scheme http://images.example.com/cover.jpg
|
|
// invalid_url https://cdn.attacker.example/cover.jpg
|
|
// private_ip https://127.0.0.1/admin
|
|
// private_ip https://169.254.169.254/latest/meta-data/
|
|
// private_ip https://[::ffff:10.0.0.1]/
|
|
// invalid_url https://%zz
|
|
// 10485760 10s
|
|
}
|
|
|
|
// stubVendor stands in for a vendor API in the example; tests use the tide
|
|
// upstream fake the same way.
|
|
type stubVendor struct{}
|
|
|
|
func (stubVendor) RoundTrip(r *http.Request) (*http.Response, error) {
|
|
body := `{"id":42}`
|
|
if r.Header.Get("Authorization") != "Bearer example-token" {
|
|
body = `{"error":"unauthorized"}`
|
|
}
|
|
return &http.Response{
|
|
StatusCode: http.StatusCreated,
|
|
Header: http.Header{"Content-Type": {"application/json"}},
|
|
Body: io.NopCloser(strings.NewReader(body)),
|
|
Request: r,
|
|
}, nil
|
|
}
|
|
|
|
func ExampleClient_PostJSON() {
|
|
client, err := fetchguard.NewClient(fetchguard.Policy{
|
|
Mode: fetchguard.AllowHostsMode,
|
|
AllowHosts: []string{"api.example.com"},
|
|
Timeout: 10 * time.Second,
|
|
}, nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
header := http.Header{}
|
|
header.Set("Authorization", fetchguard.Bearer("example-token"))
|
|
|
|
// Production code passes its own context; the example routes the call to
|
|
// a stub instead of the network.
|
|
ctx := fetchguard.WithTransport(context.Background(), stubVendor{})
|
|
res, err := client.PostJSON(ctx, "https://api.example.com/v1/items", header, map[string]string{"name": "widget"})
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
fmt.Println(res.StatusCode, string(res.Body))
|
|
// Output:
|
|
// 201 {"id":42}
|
|
}
|
|
|
|
func ExampleIsPrivateAddr() {
|
|
for _, ip := range []string{"8.8.8.8", "10.0.0.7", "::ffff:127.0.0.1"} {
|
|
fmt.Println(ip, fetchguard.IsPrivateAddr(netip.MustParseAddr(ip)))
|
|
}
|
|
// Output:
|
|
// 8.8.8.8 false
|
|
// 10.0.0.7 true
|
|
// ::ffff:127.0.0.1 true
|
|
}
|
|
|
|
// errRejectedURL answers an image URL that points inside the network.
|
|
var errRejectedURL = errors.New("image URL rejected")
|
|
|
|
// fetchImage downloads an image from imageHost or one of its subdomains
|
|
// (images.example.com in production). cfg is the application's config
|
|
// (app.Config).
|
|
func fetchImage(ctx context.Context, cfg *compass.Config, imageHost, imageURL string) ([]byte, error) {
|
|
// docs:start fetch
|
|
policy := fetchguard.Policy{
|
|
Mode: fetchguard.AllowHostsMode,
|
|
AllowHosts: []string{imageHost},
|
|
MaxBytes: 5 << 20,
|
|
Timeout: 5 * time.Second,
|
|
}
|
|
|
|
res, err := fetchguard.Fetch(ctx, imageURL, policy, cfg)
|
|
if err != nil {
|
|
var fe *fetchguard.Error
|
|
if errors.As(err, &fe) && fe.Reason == fetchguard.ReasonPrivateIP {
|
|
return nil, errRejectedURL
|
|
}
|
|
return nil, err
|
|
}
|
|
if res.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("image fetch: status %d", res.StatusCode)
|
|
}
|
|
image := res.Body
|
|
// docs:end fetch
|
|
return image, nil
|
|
}
|
|
|
|
// TestDocsFetchImage runs fetchImage offline: an allowed host that resolves
|
|
// into the network is rejected at dial time, a host outside the list and a
|
|
// plain http URL never reach the network.
|
|
func TestDocsFetchImage(t *testing.T) {
|
|
site := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
t.Error("the guard let a request through to a loopback server")
|
|
}))
|
|
defer site.Close()
|
|
if _, err := fetchImage(t.Context(), nil, "127.0.0.1", site.URL+"/cover.jpg"); !errors.Is(err, errRejectedURL) {
|
|
t.Fatalf("loopback image = %v, want errRejectedURL", err)
|
|
}
|
|
for url, want := range map[string]fetchguard.Reason{
|
|
"https://cdn.attacker.example/cover.jpg": fetchguard.ReasonInvalidURL,
|
|
"http://images.example.com/cover.jpg": fetchguard.ReasonScheme,
|
|
} {
|
|
_, err := fetchImage(t.Context(), nil, "images.example.com", url)
|
|
var fe *fetchguard.Error
|
|
if !errors.As(err, &fe) || fe.Reason != want {
|
|
t.Fatalf("%s: %v, want %s", url, err, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// vendorFunc is a vendor API stand-in for WithTransport.
|
|
type vendorFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f vendorFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
|
|
|
// syncItems sends payload to the vendor API, then uploads a report file.
|
|
func syncItems(ctx context.Context, cfg *compass.Config, token string, payload any, f io.Reader) (*fetchguard.Result, error) {
|
|
// docs:start client
|
|
vendor, err := fetchguard.NewClient(fetchguard.Policy{
|
|
Mode: fetchguard.AllowHostsMode,
|
|
AllowHosts: []string{"api.example.com"},
|
|
Timeout: 10 * time.Second,
|
|
}, cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
header := http.Header{}
|
|
header.Set("Authorization", fetchguard.Bearer(token))
|
|
res, err := vendor.PostJSON(ctx, "https://api.example.com/v1/items", header, payload)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if res.StatusCode == http.StatusTooManyRequests {
|
|
wait := res.Header.Get("Retry-After")
|
|
return nil, fmt.Errorf("vendor busy, retry after %s seconds", wait)
|
|
}
|
|
|
|
// Multipart upload: fields first, then files, each in slice order.
|
|
res, err = vendor.PostMultipart(ctx, "https://api.example.com/v1/files", header,
|
|
[]fetchguard.FormField{{Name: "title", Value: "Report"}},
|
|
[]fetchguard.FormFile{{Field: "file", Filename: "report.pdf", ContentType: "application/pdf", Body: f}},
|
|
)
|
|
// docs:end client
|
|
return res, err
|
|
}
|
|
|
|
// TestDocsSyncItems runs syncItems against a stub vendor: the JSON call and
|
|
// the upload carry the bearer token, and a 429 stops before the upload.
|
|
func TestDocsSyncItems(t *testing.T) {
|
|
var calls []string
|
|
busy := false
|
|
vendor := vendorFunc(func(r *http.Request) (*http.Response, error) {
|
|
body, _ := io.ReadAll(r.Body)
|
|
calls = append(calls, r.URL.Path+" "+r.Header.Get("Authorization"))
|
|
status, header := http.StatusCreated, http.Header{}
|
|
if busy {
|
|
status = http.StatusTooManyRequests
|
|
header.Set("Retry-After", "30")
|
|
}
|
|
if r.URL.Path == "/v1/files" && (!strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") ||
|
|
!strings.Contains(string(body), `filename="report.pdf"`) || !strings.Contains(string(body), "Report")) {
|
|
status = http.StatusBadRequest
|
|
}
|
|
return &http.Response{StatusCode: status, Header: header, Body: io.NopCloser(strings.NewReader(`{}`)), Request: r}, nil
|
|
})
|
|
ctx := fetchguard.WithTransport(t.Context(), vendor)
|
|
|
|
res, err := syncItems(ctx, nil, "example-token", map[string]string{"name": "widget"}, strings.NewReader("%PDF-1.7"))
|
|
if err != nil || res.StatusCode != http.StatusCreated {
|
|
t.Fatalf("syncItems = %+v, %v", res, err)
|
|
}
|
|
want := []string{"/v1/items Bearer example-token", "/v1/files Bearer example-token"}
|
|
if strings.Join(calls, "|") != strings.Join(want, "|") {
|
|
t.Fatalf("calls = %q", calls)
|
|
}
|
|
|
|
calls, busy = nil, true
|
|
if _, err := syncItems(ctx, nil, "example-token", nil, strings.NewReader("")); err == nil ||
|
|
!strings.Contains(err.Error(), "retry after 30 seconds") || len(calls) != 1 {
|
|
t.Fatalf("busy vendor: %v after %q", err, calls)
|
|
}
|
|
}
|