package surf import ( "context" "database/sql" "net/http" "net/http/httptest" "strings" "testing" "time" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/compass" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/lagoon/attach" "git.golem15.com/golem15/summercms/modules/pact" "git.golem15.com/golem15/summercms/modules/party" _ "github.com/jackc/pgx/v5/stdlib" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/postgres" "gocloud.dev/blob" "gocloud.dev/blob/memblob" "gorm.io/gorm" ) // uploadsConfig is an http config with the body limits and the given // storage.uploads keys set. func uploadsConfig(t *testing.T, keys map[string]any) *compass.Config { t.Helper() cfg := writeHTTPConfig(t, "body_limits:\n default_bytes: 1024\n upload_bytes: 1024\n") for k, v := range keys { if err := cfg.Set(k, v); err != nil { t.Fatal(err) } } return cfg } // uploadsPathPlugin registers GET under the uploads prefix. type uploadsPathPlugin struct{ path string } func (uploadsPathPlugin) ID() string { return "acme.media" } func (uploadsPathPlugin) Requires() []string { return nil } func (uploadsPathPlugin) Register(*backpack.App) error { return nil } func (uploadsPathPlugin) Boot(*backpack.App) error { return nil } func (p uploadsPathPlugin) Routes(r pact.Router) error { r.Get(p.path, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("plugin")) }) return nil } // uploadsGorm starts a Postgres container with the framework migrations // (system_files). An unavailable Docker daemon fails the test. func uploadsGorm(t *testing.T) *gorm.DB { t.Helper() if testing.Short() { t.Skip("requires testcontainers postgres") } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) t.Cleanup(cancel) ctr, err := postgres.Run(ctx, "postgres:16-alpine", postgres.WithDatabase("uploads"), postgres.WithUsername("uploads"), postgres.WithPassword("uploads"), postgres.BasicWaitStrategies(), ) if err != nil { t.Fatalf("postgres: %v", err) } t.Cleanup(func() { _ = testcontainers.TerminateContainer(ctr) }) dsn, err := ctr.ConnectionString(ctx, "sslmode=disable") if err != nil { t.Fatal(err) } db, err := sql.Open("pgx", dsn) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = db.Close() }) if err := db.PingContext(ctx); err != nil { t.Fatal(err) } gdb, err := lagoon.Use(ctx, db) if err != nil { t.Fatal(err) } if err := lagoon.Migrate(gdb, nil); err != nil { t.Fatal(err) } return gdb } type uploadsResponse struct { code int header http.Header body string } func uploadsDo(h http.Handler, method, target string) uploadsResponse { rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(method, target, nil)) return uploadsResponse{code: rec.Code, header: rec.Header(), body: rec.Body.String()} } // uploadsRoutes returns the routes summercms.surf owns. func uploadsRoutes(t *testing.T, app *backpack.App, plugins []party.Plugin) []RouteInfo { t.Helper() rt, err := BuildRouter(app, plugins) if err != nil { t.Fatal(err) } var out []RouteInfo for _, info := range rt.Routes() { if info.PluginID == uploadsPluginID { out = append(out, info) } } return out } const ( uploadsPublicDisk = "abc123xyz.png" uploadsPrivateDisk = "def456uvw.png" uploadsOrphanDisk = "ghi789rst.png" uploadsHTMLDisk = "jkl012mno.txt" uploadsPublicBody = "public-png-bytes" ) // TestUploadsMount: with storage.uploads.serve off (or absent) no route is // registered; on, GET /{path...} serves a public blob with its // content type and nosniff, answers HEAD without a body, and 404s a // private row, a missing row, traversal paths and an unpublished bucket or // database. A custom prefix is honoured, and a plugin route at or under // the prefix fails BuildRouter. func TestUploadsMount(t *testing.T) { publicPath := "/storage/uploads/abc/123/xyz/" + uploadsPublicDisk t.Run("off registers nothing", func(t *testing.T) { for _, keys := range []map[string]any{nil, {"storage.uploads.serve": false}} { app := backpack.New(uploadsConfig(t, keys)) if routes := uploadsRoutes(t, app, nil); len(routes) != 0 { t.Fatalf("keys %v: uploads routes %v", keys, routes) } h, err := Assemble(app, nil) if err != nil { t.Fatal(err) } if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound { t.Fatalf("keys %v: GET %s = %d, want 404", keys, publicPath, r.code) } } }) t.Run("on registers one GET route", func(t *testing.T) { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true})) routes := uploadsRoutes(t, app, nil) if len(routes) != 1 || routes[0].Method != http.MethodGet || routes[0].Pattern != "/storage/uploads/{path...}" || !routes[0].Raw { t.Fatalf("uploads routes = %+v", routes) } }) t.Run("a prefix that is not an absolute URL path fails BuildRouter", func(t *testing.T) { for _, prefix := range []string{ "storage/uploads", // no leading slash: ServeMux reads "storage" as a host "storage/app/uploads/public", // the WinterCMS disk path, not its URL "https://cdn.example.com/uploads", // a CDN origin "//cdn.example.com/uploads", // a scheme-relative URL "/storage//uploads", "/storage/{id}/uploads", "/storage/uploads?x=1", "/storage/uploads#top", "/storage/../uploads", "/storage/./uploads", "/storage/up loads", "/", } { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true, "storage.uploads.public_path_prefix": prefix})) _, err := BuildRouter(app, nil) if err == nil || !strings.Contains(err.Error(), "storage.uploads.public_path_prefix") || !strings.Contains(err.Error(), "absolute URL path") { t.Fatalf("prefix %q: err = %v, want a refusal naming storage.uploads.public_path_prefix", prefix, err) } } // Valid prefixes, with or without a trailing slash, and an invalid // one while serve is off (the key then only feeds attach.PublicURL). for _, prefix := range []string{"/storage/uploads", "/storage/app/uploads/public/", "/media", " /files "} { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true, "storage.uploads.public_path_prefix": prefix})) if _, err := BuildRouter(app, nil); err != nil { t.Fatalf("prefix %q: %v", prefix, err) } } app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.public_path_prefix": "https://cdn.example.com/uploads"})) if _, err := BuildRouter(app, nil); err != nil { t.Fatalf("serve off with a CDN prefix: %v", err) } }) t.Run("a plugin route at or under the prefix fails BuildRouter", func(t *testing.T) { for _, path := range []string{"/storage/uploads/{path...}", "/storage/uploads/avatars/{id}", "/storage/uploads"} { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true})) _, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: path}}) if err == nil || !strings.Contains(err.Error(), "storage.uploads") { t.Fatalf("plugin route %s: err = %v, want a refusal naming storage.uploads", path, err) } } // A sibling path and the same path with the key off are fine. app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true})) if _, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: "/storage/uploadsx/{id}"}}); err != nil { t.Fatalf("sibling path: %v", err) } app = backpack.New(uploadsConfig(t, nil)) if _, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: "/storage/uploads/avatars/{id}"}}); err != nil { t.Fatalf("key off: %v", err) } }) t.Run("nothing published answers 404", func(t *testing.T) { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true})) h, err := Assemble(app, nil) if err != nil { t.Fatal(err) } r := uploadsDo(h, http.MethodGet, publicPath) if r.code != http.StatusNotFound || r.header.Get("X-Content-Type-Options") != "nosniff" { t.Fatalf("no bucket, no database: %d nosniff=%q", r.code, r.header.Get("X-Content-Type-Options")) } bucket := memblob.OpenBucket(nil) t.Cleanup(func() { _ = bucket.Close() }) if err := bucket.WriteAll(t.Context(), attach.BlobKey(uploadsPublicDisk), []byte(uploadsPublicBody), nil); err != nil { t.Fatal(err) } if err := app.Publish(bucket); err != nil { t.Fatal(err) } if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound { t.Fatalf("bucket without a database: %d, want 404", r.code) } }) gdb := uploadsGorm(t) ctx := t.Context() bucket := memblob.OpenBucket(nil) t.Cleanup(func() { _ = bucket.Close() }) pub, priv := true, false for _, f := range []attach.File{ {DiskName: uploadsPublicDisk, FileName: "cover.png", FileSize: 1, ContentType: "image/png", IsPublic: &pub}, {DiskName: uploadsPrivateDisk, FileName: "secret.png", FileSize: 1, ContentType: "image/png", IsPublic: &priv}, {DiskName: uploadsHTMLDisk, FileName: "notes.txt", FileSize: 1, ContentType: "text/html; charset=utf-8", IsPublic: &pub}, } { if err := gdb.Create(&f).Error; err != nil { t.Fatal(err) } } for disk, body := range map[string]string{ uploadsPublicDisk: uploadsPublicBody, uploadsPrivateDisk: "private-png-bytes", uploadsOrphanDisk: "orphan-png-bytes", } { if err := bucket.WriteAll(ctx, attach.BlobKey(disk), []byte(body), &blob.WriterOptions{ContentType: "image/png"}); err != nil { t.Fatal(err) } } if err := bucket.WriteAll(ctx, attach.BlobKey(uploadsHTMLDisk), []byte(""), &blob.WriterOptions{ContentType: "text/html; charset=utf-8"}); err != nil { t.Fatal(err) } htmlPath := "/storage/uploads/" + attach.BlobKey(uploadsHTMLDisk) t.Run("database without a bucket answers 404", func(t *testing.T) { app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true})) if err := app.Publish(gdb); err != nil { t.Fatal(err) } h, err := Assemble(app, nil) if err != nil { t.Fatal(err) } if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound { t.Fatalf("database without a bucket: %d, want 404", r.code) } }) serving := func(t *testing.T, prefix string) http.Handler { t.Helper() keys := map[string]any{"storage.uploads.serve": true} if prefix != "" { keys["storage.uploads.public_path_prefix"] = prefix } app := backpack.New(uploadsConfig(t, keys)) if err := app.Publish(gdb); err != nil { t.Fatal(err) } if err := app.Publish(bucket); err != nil { t.Fatal(err) } h, err := Assemble(app, nil) if err != nil { t.Fatal(err) } return h } t.Run("serves public blobs and refuses the rest", func(t *testing.T) { h := serving(t, "") r := uploadsDo(h, http.MethodGet, publicPath) if r.code != http.StatusOK || r.body != uploadsPublicBody { t.Fatalf("GET %s = %d %q", publicPath, r.code, r.body) } if ct := r.header.Get("Content-Type"); ct != "image/png" { t.Fatalf("Content-Type = %q, want image/png", ct) } if got := r.header.Get("X-Content-Type-Options"); got != "nosniff" { t.Fatalf("X-Content-Type-Options = %q, want nosniff", got) } if r.header.Get("Content-Disposition") != "" || r.header.Get("Content-Security-Policy") != uploadsContentSecurityPolicy { t.Fatalf("image/png: Content-Disposition=%q CSP=%q, want inline with the sandbox CSP", r.header.Get("Content-Disposition"), r.header.Get("Content-Security-Policy")) } // A public file whose stored type is text/html (a .txt upload whose // bytes start with ) is served as an attachment, sandboxed. html := uploadsDo(h, http.MethodGet, htmlPath) if html.code != http.StatusOK || html.header.Get("Content-Disposition") != "attachment" || html.header.Get("Content-Security-Policy") != uploadsContentSecurityPolicy { t.Fatalf("GET %s = %d Content-Disposition=%q CSP=%q, want 200 attachment with the sandbox CSP", htmlPath, html.code, html.header.Get("Content-Disposition"), html.header.Get("Content-Security-Policy")) } head := uploadsDo(h, http.MethodHead, publicPath) if head.code != http.StatusOK || head.body != "" || head.header.Get("Content-Type") != "image/png" || head.header.Get("X-Content-Type-Options") != "nosniff" { t.Fatalf("HEAD %s = %d %q %v", publicPath, head.code, head.body, head.header) } for _, target := range []string{ "/storage/uploads/def/456/uvw/" + uploadsPrivateDisk, // is_public false "/storage/uploads/ghi/789/rst/" + uploadsOrphanDisk, // blob without a row "/storage/uploads/zzz/999/yyy/zzz999yyy.png", // neither "/storage/uploads/abc/123/" + uploadsPublicDisk, // partition too short "/storage/uploads/def/456/xyz/" + uploadsPublicDisk, // wrong partition "/storage/uploads/abc/123/xyz/..%2f..%2f..%2fsecret", // encoded traversal "/storage/uploads/abc/123/xyz/%2e%2e%2f" + uploadsPrivateDisk, "/storage/uploads/abc/123/xyz/..%5c" + uploadsPrivateDisk, // encoded backslash } { r := uploadsDo(h, http.MethodGet, target) if r.code != http.StatusNotFound || r.header.Get("X-Content-Type-Options") != "nosniff" { t.Fatalf("GET %s = %d nosniff=%q, want 404 with nosniff", target, r.code, r.header.Get("X-Content-Type-Options")) } if strings.Contains(r.body, "png-bytes") { t.Fatalf("GET %s leaked a blob: %q", target, r.body) } } // A raw dot-dot path never reaches the handler: ServeMux answers it // with a redirect to the cleaned path, which is then served or // refused by the same rules. for target, want := range map[string]int{ "/storage/uploads/abc/123/../../def/456/uvw/" + uploadsPrivateDisk: http.StatusNotFound, "/storage/uploads/../../../etc/passwd": http.StatusNotFound, } { r := uploadsDo(h, http.MethodGet, target) if strings.Contains(r.body, "png-bytes") { t.Fatalf("GET %s leaked a blob: %d %q", target, r.code, r.body) } if r.code == http.StatusMovedPermanently || r.code == http.StatusTemporaryRedirect || r.code == http.StatusPermanentRedirect { loc := r.header.Get("Location") if strings.Contains(loc, "..") { t.Fatalf("GET %s redirects to %q", target, loc) } if got := uploadsDo(h, http.MethodGet, loc); got.code != want { t.Fatalf("GET %s -> %s = %d, want %d", target, loc, got.code, want) } continue } if r.code != want { t.Fatalf("GET %s = %d, want %d or a redirect", target, r.code, want) } } }) t.Run("a custom prefix is honoured", func(t *testing.T) { h := serving(t, "/media/files/") custom := "/media/files/abc/123/xyz/" + uploadsPublicDisk if r := uploadsDo(h, http.MethodGet, custom); r.code != http.StatusOK || r.body != uploadsPublicBody || r.header.Get("X-Content-Type-Options") != "nosniff" { t.Fatalf("GET %s = %d %q", custom, r.code, r.body) } if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound { t.Fatalf("GET %s under the default prefix = %d, want 404", publicPath, r.code) } }) } // TestUploadsResponseHeaders: every uploads response carries nosniff and the // sandboxing Content-Security-Policy; a successful response whose type is // not an inline raster image (HTML, SVG, PDF, text, a missing type) gets // Content-Disposition: attachment, on GET and HEAD; images stay inline and // error responses get no disposition. func TestUploadsResponseHeaders(t *testing.T) { serve := func(ct string, status int) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if status == http.StatusNotFound { http.NotFound(w, req) return } if ct != "" { w.Header().Set("Content-Type", ct) } if req.Method == http.MethodHead { return } _, _ = w.Write([]byte("")) }) } for _, tc := range []struct { ct string status int attachment bool }{ {"image/png", 200, false}, {"image/jpeg", 200, false}, {"IMAGE/WEBP; charset=binary", 200, false}, {"image/gif", 200, false}, {"image/avif", 200, false}, {"text/html; charset=utf-8", 200, true}, {"image/svg+xml", 200, true}, {"application/pdf", 200, true}, {"text/plain; charset=utf-8", 200, true}, {"application/octet-stream", 200, true}, {"", 200, true}, {"not a media type;;", 200, true}, {"text/plain", http.StatusNotFound, false}, } { for _, method := range []string{http.MethodGet, http.MethodHead} { rec := httptest.NewRecorder() serveUploads(rec, httptest.NewRequest(method, "/storage/uploads/abc/123/xyz/abc123xyz.png", nil), serve(tc.ct, tc.status)) h := rec.Header() if h.Get("X-Content-Type-Options") != "nosniff" || h.Get("Content-Security-Policy") != uploadsContentSecurityPolicy { t.Fatalf("%s %q: nosniff=%q csp=%q", method, tc.ct, h.Get("X-Content-Type-Options"), h.Get("Content-Security-Policy")) } got := h.Get("Content-Disposition") if tc.attachment && got != "attachment" { t.Fatalf("%s %q (%d): Content-Disposition = %q, want attachment", method, tc.ct, tc.status, got) } if !tc.attachment && got != "" { t.Fatalf("%s %q (%d): Content-Disposition = %q, want none", method, tc.ct, tc.status, got) } } } if !strings.Contains(uploadsContentSecurityPolicy, "sandbox") || !strings.Contains(uploadsContentSecurityPolicy, "default-src 'none'") { t.Fatalf("uploads CSP %q does not sandbox", uploadsContentSecurityPolicy) } }