Jakub Zych
41a03b9987
The route answered with the blob's stored Content-Type, which attach.Store sniffs from the bytes, so a public text upload starting with <html> was served as text/html on the application origin; nosniff does not help when the declared type is already HTML. Every response now carries a sandboxing Content-Security-Policy, and a successful response whose type is not a raster image (JPEG, PNG, GIF, WebP, AVIF) gets Content-Disposition: attachment, on GET and HEAD. Covers stay inline.
408 lines
16 KiB
Go
408 lines
16 KiB
Go
package surf
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/compass"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/party"
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
"github.com/testcontainers/testcontainers-go"
|
|
"github.com/testcontainers/testcontainers-go/modules/postgres"
|
|
"gocloud.dev/blob"
|
|
"gocloud.dev/blob/memblob"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// uploadsConfig is an http config with the body limits and the given
|
|
// storage.uploads keys set.
|
|
func uploadsConfig(t *testing.T, keys map[string]any) *compass.Config {
|
|
t.Helper()
|
|
cfg := writeHTTPConfig(t, "body_limits:\n default_bytes: 1024\n upload_bytes: 1024\n")
|
|
for k, v := range keys {
|
|
if err := cfg.Set(k, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// uploadsPathPlugin registers GET <path> under the uploads prefix.
|
|
type uploadsPathPlugin struct{ path string }
|
|
|
|
func (uploadsPathPlugin) ID() string { return "acme.media" }
|
|
func (uploadsPathPlugin) Requires() []string { return nil }
|
|
func (uploadsPathPlugin) Register(*backpack.App) error { return nil }
|
|
func (uploadsPathPlugin) Boot(*backpack.App) error { return nil }
|
|
func (p uploadsPathPlugin) Routes(r pact.Router) error {
|
|
r.Get(p.path, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("plugin")) })
|
|
return nil
|
|
}
|
|
|
|
// uploadsGorm starts a Postgres container with the framework migrations
|
|
// (system_files). An unavailable Docker daemon fails the test.
|
|
func uploadsGorm(t *testing.T) *gorm.DB {
|
|
t.Helper()
|
|
if testing.Short() {
|
|
t.Skip("requires testcontainers postgres")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
t.Cleanup(cancel)
|
|
ctr, err := postgres.Run(ctx,
|
|
"postgres:16-alpine",
|
|
postgres.WithDatabase("uploads"),
|
|
postgres.WithUsername("uploads"),
|
|
postgres.WithPassword("uploads"),
|
|
postgres.BasicWaitStrategies(),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("postgres: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = testcontainers.TerminateContainer(ctr) })
|
|
dsn, err := ctr.ConnectionString(ctx, "sslmode=disable")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
if err := db.PingContext(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gdb, err := lagoon.Use(ctx, db)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := lagoon.Migrate(gdb, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return gdb
|
|
}
|
|
|
|
type uploadsResponse struct {
|
|
code int
|
|
header http.Header
|
|
body string
|
|
}
|
|
|
|
func uploadsDo(h http.Handler, method, target string) uploadsResponse {
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, httptest.NewRequest(method, target, nil))
|
|
return uploadsResponse{code: rec.Code, header: rec.Header(), body: rec.Body.String()}
|
|
}
|
|
|
|
// uploadsRoutes returns the routes summercms.surf owns.
|
|
func uploadsRoutes(t *testing.T, app *backpack.App, plugins []party.Plugin) []RouteInfo {
|
|
t.Helper()
|
|
rt, err := BuildRouter(app, plugins)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out []RouteInfo
|
|
for _, info := range rt.Routes() {
|
|
if info.PluginID == uploadsPluginID {
|
|
out = append(out, info)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
const (
|
|
uploadsPublicDisk = "abc123xyz.png"
|
|
uploadsPrivateDisk = "def456uvw.png"
|
|
uploadsOrphanDisk = "ghi789rst.png"
|
|
uploadsHTMLDisk = "jkl012mno.txt"
|
|
uploadsPublicBody = "public-png-bytes"
|
|
)
|
|
|
|
// TestUploadsMount: with storage.uploads.serve off (or absent) no route is
|
|
// registered; on, GET <prefix>/{path...} serves a public blob with its
|
|
// content type and nosniff, answers HEAD without a body, and 404s a
|
|
// private row, a missing row, traversal paths and an unpublished bucket or
|
|
// database. A custom prefix is honoured, and a plugin route at or under
|
|
// the prefix fails BuildRouter.
|
|
func TestUploadsMount(t *testing.T) {
|
|
publicPath := "/storage/uploads/abc/123/xyz/" + uploadsPublicDisk
|
|
|
|
t.Run("off registers nothing", func(t *testing.T) {
|
|
for _, keys := range []map[string]any{nil, {"storage.uploads.serve": false}} {
|
|
app := backpack.New(uploadsConfig(t, keys))
|
|
if routes := uploadsRoutes(t, app, nil); len(routes) != 0 {
|
|
t.Fatalf("keys %v: uploads routes %v", keys, routes)
|
|
}
|
|
h, err := Assemble(app, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound {
|
|
t.Fatalf("keys %v: GET %s = %d, want 404", keys, publicPath, r.code)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("on registers one GET route", func(t *testing.T) {
|
|
app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true}))
|
|
routes := uploadsRoutes(t, app, nil)
|
|
if len(routes) != 1 || routes[0].Method != http.MethodGet || routes[0].Pattern != "/storage/uploads/{path...}" || !routes[0].Raw {
|
|
t.Fatalf("uploads routes = %+v", routes)
|
|
}
|
|
})
|
|
|
|
t.Run("a plugin route at or under the prefix fails BuildRouter", func(t *testing.T) {
|
|
for _, path := range []string{"/storage/uploads/{path...}", "/storage/uploads/avatars/{id}", "/storage/uploads"} {
|
|
app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true}))
|
|
_, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: path}})
|
|
if err == nil || !strings.Contains(err.Error(), "storage.uploads") {
|
|
t.Fatalf("plugin route %s: err = %v, want a refusal naming storage.uploads", path, err)
|
|
}
|
|
}
|
|
// A sibling path and the same path with the key off are fine.
|
|
app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true}))
|
|
if _, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: "/storage/uploadsx/{id}"}}); err != nil {
|
|
t.Fatalf("sibling path: %v", err)
|
|
}
|
|
app = backpack.New(uploadsConfig(t, nil))
|
|
if _, err := BuildRouter(app, []party.Plugin{uploadsPathPlugin{path: "/storage/uploads/avatars/{id}"}}); err != nil {
|
|
t.Fatalf("key off: %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("nothing published answers 404", func(t *testing.T) {
|
|
app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true}))
|
|
h, err := Assemble(app, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := uploadsDo(h, http.MethodGet, publicPath)
|
|
if r.code != http.StatusNotFound || r.header.Get("X-Content-Type-Options") != "nosniff" {
|
|
t.Fatalf("no bucket, no database: %d nosniff=%q", r.code, r.header.Get("X-Content-Type-Options"))
|
|
}
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
if err := bucket.WriteAll(t.Context(), attach.BlobKey(uploadsPublicDisk), []byte(uploadsPublicBody), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := app.Publish(bucket); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound {
|
|
t.Fatalf("bucket without a database: %d, want 404", r.code)
|
|
}
|
|
})
|
|
|
|
gdb := uploadsGorm(t)
|
|
ctx := t.Context()
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
pub, priv := true, false
|
|
for _, f := range []attach.File{
|
|
{DiskName: uploadsPublicDisk, FileName: "cover.png", FileSize: 1, ContentType: "image/png", IsPublic: &pub},
|
|
{DiskName: uploadsPrivateDisk, FileName: "secret.png", FileSize: 1, ContentType: "image/png", IsPublic: &priv},
|
|
{DiskName: uploadsHTMLDisk, FileName: "notes.txt", FileSize: 1, ContentType: "text/html; charset=utf-8", IsPublic: &pub},
|
|
} {
|
|
if err := gdb.Create(&f).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for disk, body := range map[string]string{
|
|
uploadsPublicDisk: uploadsPublicBody,
|
|
uploadsPrivateDisk: "private-png-bytes",
|
|
uploadsOrphanDisk: "orphan-png-bytes",
|
|
} {
|
|
if err := bucket.WriteAll(ctx, attach.BlobKey(disk), []byte(body), &blob.WriterOptions{ContentType: "image/png"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := bucket.WriteAll(ctx, attach.BlobKey(uploadsHTMLDisk), []byte("<html><script>alert(1)</script></html>"), &blob.WriterOptions{ContentType: "text/html; charset=utf-8"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
htmlPath := "/storage/uploads/" + attach.BlobKey(uploadsHTMLDisk)
|
|
|
|
t.Run("database without a bucket answers 404", func(t *testing.T) {
|
|
app := backpack.New(uploadsConfig(t, map[string]any{"storage.uploads.serve": true}))
|
|
if err := app.Publish(gdb); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h, err := Assemble(app, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound {
|
|
t.Fatalf("database without a bucket: %d, want 404", r.code)
|
|
}
|
|
})
|
|
|
|
serving := func(t *testing.T, prefix string) http.Handler {
|
|
t.Helper()
|
|
keys := map[string]any{"storage.uploads.serve": true}
|
|
if prefix != "" {
|
|
keys["storage.uploads.public_path_prefix"] = prefix
|
|
}
|
|
app := backpack.New(uploadsConfig(t, keys))
|
|
if err := app.Publish(gdb); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := app.Publish(bucket); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h, err := Assemble(app, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return h
|
|
}
|
|
|
|
t.Run("serves public blobs and refuses the rest", func(t *testing.T) {
|
|
h := serving(t, "")
|
|
r := uploadsDo(h, http.MethodGet, publicPath)
|
|
if r.code != http.StatusOK || r.body != uploadsPublicBody {
|
|
t.Fatalf("GET %s = %d %q", publicPath, r.code, r.body)
|
|
}
|
|
if ct := r.header.Get("Content-Type"); ct != "image/png" {
|
|
t.Fatalf("Content-Type = %q, want image/png", ct)
|
|
}
|
|
if got := r.header.Get("X-Content-Type-Options"); got != "nosniff" {
|
|
t.Fatalf("X-Content-Type-Options = %q, want nosniff", got)
|
|
}
|
|
if r.header.Get("Content-Disposition") != "" || r.header.Get("Content-Security-Policy") != uploadsContentSecurityPolicy {
|
|
t.Fatalf("image/png: Content-Disposition=%q CSP=%q, want inline with the sandbox CSP", r.header.Get("Content-Disposition"), r.header.Get("Content-Security-Policy"))
|
|
}
|
|
// A public file whose stored type is text/html (a .txt upload whose
|
|
// bytes start with <html>) is served as an attachment, sandboxed.
|
|
html := uploadsDo(h, http.MethodGet, htmlPath)
|
|
if html.code != http.StatusOK || html.header.Get("Content-Disposition") != "attachment" || html.header.Get("Content-Security-Policy") != uploadsContentSecurityPolicy {
|
|
t.Fatalf("GET %s = %d Content-Disposition=%q CSP=%q, want 200 attachment with the sandbox CSP", htmlPath, html.code, html.header.Get("Content-Disposition"), html.header.Get("Content-Security-Policy"))
|
|
}
|
|
head := uploadsDo(h, http.MethodHead, publicPath)
|
|
if head.code != http.StatusOK || head.body != "" || head.header.Get("Content-Type") != "image/png" || head.header.Get("X-Content-Type-Options") != "nosniff" {
|
|
t.Fatalf("HEAD %s = %d %q %v", publicPath, head.code, head.body, head.header)
|
|
}
|
|
for _, target := range []string{
|
|
"/storage/uploads/def/456/uvw/" + uploadsPrivateDisk, // is_public false
|
|
"/storage/uploads/ghi/789/rst/" + uploadsOrphanDisk, // blob without a row
|
|
"/storage/uploads/zzz/999/yyy/zzz999yyy.png", // neither
|
|
"/storage/uploads/abc/123/" + uploadsPublicDisk, // partition too short
|
|
"/storage/uploads/def/456/xyz/" + uploadsPublicDisk, // wrong partition
|
|
"/storage/uploads/abc/123/xyz/..%2f..%2f..%2fsecret", // encoded traversal
|
|
"/storage/uploads/abc/123/xyz/%2e%2e%2f" + uploadsPrivateDisk,
|
|
"/storage/uploads/abc/123/xyz/..%5c" + uploadsPrivateDisk, // encoded backslash
|
|
} {
|
|
r := uploadsDo(h, http.MethodGet, target)
|
|
if r.code != http.StatusNotFound || r.header.Get("X-Content-Type-Options") != "nosniff" {
|
|
t.Fatalf("GET %s = %d nosniff=%q, want 404 with nosniff", target, r.code, r.header.Get("X-Content-Type-Options"))
|
|
}
|
|
if strings.Contains(r.body, "png-bytes") {
|
|
t.Fatalf("GET %s leaked a blob: %q", target, r.body)
|
|
}
|
|
}
|
|
// A raw dot-dot path never reaches the handler: ServeMux answers it
|
|
// with a redirect to the cleaned path, which is then served or
|
|
// refused by the same rules.
|
|
for target, want := range map[string]int{
|
|
"/storage/uploads/abc/123/../../def/456/uvw/" + uploadsPrivateDisk: http.StatusNotFound,
|
|
"/storage/uploads/../../../etc/passwd": http.StatusNotFound,
|
|
} {
|
|
r := uploadsDo(h, http.MethodGet, target)
|
|
if strings.Contains(r.body, "png-bytes") {
|
|
t.Fatalf("GET %s leaked a blob: %d %q", target, r.code, r.body)
|
|
}
|
|
if r.code == http.StatusMovedPermanently || r.code == http.StatusTemporaryRedirect || r.code == http.StatusPermanentRedirect {
|
|
loc := r.header.Get("Location")
|
|
if strings.Contains(loc, "..") {
|
|
t.Fatalf("GET %s redirects to %q", target, loc)
|
|
}
|
|
if got := uploadsDo(h, http.MethodGet, loc); got.code != want {
|
|
t.Fatalf("GET %s -> %s = %d, want %d", target, loc, got.code, want)
|
|
}
|
|
continue
|
|
}
|
|
if r.code != want {
|
|
t.Fatalf("GET %s = %d, want %d or a redirect", target, r.code, want)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("a custom prefix is honoured", func(t *testing.T) {
|
|
h := serving(t, "/media/files/")
|
|
custom := "/media/files/abc/123/xyz/" + uploadsPublicDisk
|
|
if r := uploadsDo(h, http.MethodGet, custom); r.code != http.StatusOK || r.body != uploadsPublicBody || r.header.Get("X-Content-Type-Options") != "nosniff" {
|
|
t.Fatalf("GET %s = %d %q", custom, r.code, r.body)
|
|
}
|
|
if r := uploadsDo(h, http.MethodGet, publicPath); r.code != http.StatusNotFound {
|
|
t.Fatalf("GET %s under the default prefix = %d, want 404", publicPath, r.code)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestUploadsResponseHeaders: every uploads response carries nosniff and the
|
|
// sandboxing Content-Security-Policy; a successful response whose type is
|
|
// not an inline raster image (HTML, SVG, PDF, text, a missing type) gets
|
|
// Content-Disposition: attachment, on GET and HEAD; images stay inline and
|
|
// error responses get no disposition.
|
|
func TestUploadsResponseHeaders(t *testing.T) {
|
|
serve := func(ct string, status int) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
if status == http.StatusNotFound {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
if ct != "" {
|
|
w.Header().Set("Content-Type", ct)
|
|
}
|
|
if req.Method == http.MethodHead {
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte("<html><script>alert(1)</script></html>"))
|
|
})
|
|
}
|
|
for _, tc := range []struct {
|
|
ct string
|
|
status int
|
|
attachment bool
|
|
}{
|
|
{"image/png", 200, false},
|
|
{"image/jpeg", 200, false},
|
|
{"IMAGE/WEBP; charset=binary", 200, false},
|
|
{"image/gif", 200, false},
|
|
{"image/avif", 200, false},
|
|
{"text/html; charset=utf-8", 200, true},
|
|
{"image/svg+xml", 200, true},
|
|
{"application/pdf", 200, true},
|
|
{"text/plain; charset=utf-8", 200, true},
|
|
{"application/octet-stream", 200, true},
|
|
{"", 200, true},
|
|
{"not a media type;;", 200, true},
|
|
{"text/plain", http.StatusNotFound, false},
|
|
} {
|
|
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
|
rec := httptest.NewRecorder()
|
|
serveUploads(rec, httptest.NewRequest(method, "/storage/uploads/abc/123/xyz/abc123xyz.png", nil), serve(tc.ct, tc.status))
|
|
h := rec.Header()
|
|
if h.Get("X-Content-Type-Options") != "nosniff" || h.Get("Content-Security-Policy") != uploadsContentSecurityPolicy {
|
|
t.Fatalf("%s %q: nosniff=%q csp=%q", method, tc.ct, h.Get("X-Content-Type-Options"), h.Get("Content-Security-Policy"))
|
|
}
|
|
got := h.Get("Content-Disposition")
|
|
if tc.attachment && got != "attachment" {
|
|
t.Fatalf("%s %q (%d): Content-Disposition = %q, want attachment", method, tc.ct, tc.status, got)
|
|
}
|
|
if !tc.attachment && got != "" {
|
|
t.Fatalf("%s %q (%d): Content-Disposition = %q, want none", method, tc.ct, tc.status, got)
|
|
}
|
|
}
|
|
}
|
|
if !strings.Contains(uploadsContentSecurityPolicy, "sandbox") || !strings.Contains(uploadsContentSecurityPolicy, "default-src 'none'") {
|
|
t.Fatalf("uploads CSP %q does not sandbox", uploadsContentSecurityPolicy)
|
|
}
|
|
}
|