test(19.3-05): build tags, S3 tag guard, DiskExtension and public_url
- manifest_tags_test.go: build.tags load, trim, dedupe and validation errors naming build.tags; SaveManifest keeps tags and plugin order; goBuildArgs puts one -tags before -o - s3_test.go: s3:// refused without summercms_s3 naming the tag, opens with it (both build modes, any scheme case) - diskext_test.go: RESEARCH probe rows and table consistency - url_test.go: public_url base joining, File.URL and thumbnails, malformed values refused naming the key - store_test.go: Postgres regression table through Store - gate: attach names in both tag modes, Store regression in docker
This commit is contained in:
Jakub Zych
111
modules/lagoon/attach/diskext_test.go
Normal file
111
modules/lagoon/attach/diskext_test.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDiskExtensionTable (plan 05 of phase 19.3, D-15) pins DiskExtension on
|
||||
// the RESEARCH probe rows and on the consistency of its tables: a client
|
||||
// extension that belongs to the sniffed type is kept, any other becomes the
|
||||
// type's canonical one, and a script-capable extension under a generic type
|
||||
// becomes txt (text/*) or bin.
|
||||
func TestDiskExtensionTable(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
contentType, clientExt, want string
|
||||
}{
|
||||
// RESEARCH probe rows.
|
||||
{"image/png", "html", "png"},
|
||||
{"image/jpeg", "jpeg", "jpeg"},
|
||||
{"image/jpeg", "jpg", "jpg"},
|
||||
{"image/jpeg", "png", "jpg"},
|
||||
{"video/webm", "mkv", "mkv"},
|
||||
{"video/webm", "webm", "webm"},
|
||||
{"application/zip", "docx", "docx"},
|
||||
{"application/zip", "xlsx", "xlsx"},
|
||||
{"text/plain; charset=utf-8", "ics", "ics"},
|
||||
{"text/plain; charset=utf-8", "html", "txt"},
|
||||
{"text/xml; charset=utf-8", "svg", "txt"},
|
||||
{"application/octet-stream", "js", "bin"},
|
||||
{"application/pdf", "txt", "pdf"},
|
||||
// HTML bytes under their own name never stay html.
|
||||
{"text/html; charset=utf-8", "html", "txt"},
|
||||
{"text/html; charset=utf-8", "txt", "txt"},
|
||||
// Normalisation: case, a leading dot, parameters and spaces.
|
||||
{"IMAGE/PNG", ".PNG", "png"},
|
||||
{" image/jpeg ; q=1", "JPE", "jpe"},
|
||||
{"application/octet-stream", ".Docx", "docx"},
|
||||
// Extensions outside [a-z0-9]{1,10} fall back like a script-capable one.
|
||||
{"application/octet-stream", "", "bin"},
|
||||
{"text/plain", "", "txt"},
|
||||
{"application/octet-stream", "tar.gz", "bin"},
|
||||
{"text/plain", "averyverylongext", "txt"},
|
||||
{"application/octet-stream", "ex e", "bin"},
|
||||
// Other specific types.
|
||||
{"image/gif", "png", "gif"},
|
||||
{"image/webp", "webp", "webp"},
|
||||
{"application/ogg", "opus", "opus"},
|
||||
{"audio/mpeg", "wav", "mp3"},
|
||||
{"font/woff2", "woff2", "woff2"},
|
||||
{"application/x-gzip", "tgz", "tgz"},
|
||||
} {
|
||||
if got := DiskExtension(tc.contentType, tc.clientExt); got != tc.want {
|
||||
t.Errorf("DiskExtension(%q, %q) = %q, want %q", tc.contentType, tc.clientExt, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every specific type keeps each of its own extensions, maps a
|
||||
// script-capable one to its canonical extension, and lists no
|
||||
// script-capable extension itself.
|
||||
for ct, exts := range sniffedExtensions {
|
||||
if len(exts) == 0 {
|
||||
t.Fatalf("%s has no extensions", ct)
|
||||
}
|
||||
for _, ext := range exts {
|
||||
if !extPattern.MatchString(ext) {
|
||||
t.Errorf("%s: extension %q does not match the disk name pattern", ct, ext)
|
||||
}
|
||||
if slices.Contains(scriptCapableExtensions, ext) {
|
||||
t.Errorf("%s: script-capable extension %q is kept", ct, ext)
|
||||
}
|
||||
if got := DiskExtension(ct, ext); got != ext {
|
||||
t.Errorf("DiskExtension(%q, %q) = %q", ct, ext, got)
|
||||
}
|
||||
}
|
||||
for _, bad := range scriptCapableExtensions {
|
||||
if got := DiskExtension(ct, bad); got != exts[0] {
|
||||
t.Errorf("DiskExtension(%q, %q) = %q, want %q", ct, bad, got, exts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Under every generic type a script-capable extension is rewritten.
|
||||
for _, ct := range []string{"text/plain; charset=utf-8", "text/html; charset=utf-8", "text/xml; charset=utf-8", "text/css"} {
|
||||
for _, bad := range scriptCapableExtensions {
|
||||
if got := DiskExtension(ct, bad); got != "txt" {
|
||||
t.Errorf("DiskExtension(%q, %q) = %q, want txt", ct, bad, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, ct := range []string{"application/octet-stream", "application/zip", "application/x-msdownload", ""} {
|
||||
for _, bad := range scriptCapableExtensions {
|
||||
if got := DiskExtension(ct, bad); got != "bin" {
|
||||
t.Errorf("DiskExtension(%q, %q) = %q, want bin", ct, bad, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The table agrees with what the standard library actually sniffs.
|
||||
for body, want := range map[string]string{
|
||||
"\x89PNG\r\n\x1a\n": "png",
|
||||
"%PDF-1.7\n": "pdf",
|
||||
"<html><body>x</body></html>": "txt",
|
||||
"<?xml version=\"1.0\"?><svg xmlns=\"http://www.w3.org/2000/svg\"/>": "txt",
|
||||
} {
|
||||
ct := http.DetectContentType([]byte(body))
|
||||
if got := DiskExtension(ct, "html"); got != want {
|
||||
t.Errorf("body %q sniffed %q: DiskExtension = %q, want %q", body, ct, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
68
modules/lagoon/attach/s3_test.go
Normal file
68
modules/lagoon/attach/s3_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOpenBucketS3 (plan 05 of phase 19.3, D-13) has no build constraint: it
|
||||
// runs in both modes and branches on s3Enabled. Without the summercms_s3 tag
|
||||
// an s3:// bucket_url (any case of the scheme) fails boot naming the tag and
|
||||
// where to set it; with the tag the bucket opens on dummy AWS credentials
|
||||
// (s3blob connects lazily, so nothing leaves the host) and closes.
|
||||
func TestOpenBucketS3(t *testing.T) {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "unit-access-key")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "unit-secret-key")
|
||||
t.Setenv("AWS_REGION", "eu-central-1")
|
||||
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
|
||||
t.Cleanup(func() {
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
setPublicURLBase("")
|
||||
})
|
||||
if S3BuildTag != "summercms_s3" {
|
||||
t.Fatalf("S3BuildTag = %q", S3BuildTag)
|
||||
}
|
||||
|
||||
for _, raw := range []string{
|
||||
"s3://unit-bucket?region=eu-central-1",
|
||||
"S3://unit-bucket?region=eu-central-1",
|
||||
"s3://unit-bucket?region=auto&endpoint=https://storage.example.com&use_path_style=true",
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
cfg := openTestConfig(t, "uploads:\n bucket_url: \""+raw+"\"\n")
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if !s3Enabled {
|
||||
if err == nil {
|
||||
_ = bucket.Close()
|
||||
t.Fatal("s3:// opened without the summercms_s3 tag")
|
||||
}
|
||||
for _, want := range []string{"storage.uploads.bucket_url", S3BuildTag, "build.tags", "summer.yaml", "-tags " + S3BuildTag, "GOFLAGS=-tags=" + S3BuildTag} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error %q does not name %q", err, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "unit-bucket") || strings.Contains(err.Error(), "unit-secret") {
|
||||
t.Fatalf("error %q repeats the bucket URL", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("OpenBucket with the tag: %v", err)
|
||||
}
|
||||
if err := bucket.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Other schemes never reach the tag check.
|
||||
for _, raw := range []string{"mem://", "file:///tmp/x", "fileblob:///tmp/x", "s3x://b", "gs://bucket", "s3:no-slashes"} {
|
||||
if err := checkS3Driver(raw); err != nil {
|
||||
t.Fatalf("checkS3Driver(%q) = %v", raw, err)
|
||||
}
|
||||
}
|
||||
err := checkS3Driver("s3://b")
|
||||
if s3Enabled != (err == nil) {
|
||||
t.Fatalf("checkS3Driver(s3://b) = %v with s3Enabled %t", err, s3Enabled)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"regexp"
|
||||
@@ -343,3 +344,83 @@ func TestStoreDiskExtensionSmoke(t *testing.T) {
|
||||
t.Fatalf("HTML body named note.txt: disk name %q, file_name %q", note.DiskName, note.FileName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStoreDiskExtension (plan 05 of phase 19.3, D-15) runs the stored-XSS
|
||||
// regression table through Store on Postgres: the disk name and the URL end
|
||||
// in the extension DiskExtension picks for the sniffed bytes, file_name keeps
|
||||
// the client's name and content_type the sniffed type.
|
||||
func TestStoreDiskExtension(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("requires testcontainers postgres")
|
||||
}
|
||||
ctx := t.Context()
|
||||
gdb := attachGorm(t)
|
||||
if err := lagoon.Migrate(gdb, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := memblob.OpenBucket(nil)
|
||||
t.Cleanup(func() { _ = bucket.Close() })
|
||||
lim := attach.Limits{Extensions: []string{"html", "png", "jpg", "jpeg", "docx", "txt", "ics", "svg", "js", "pdf", "mkv"}}
|
||||
|
||||
var jpegBody bytes.Buffer
|
||||
if err := jpeg.Encode(&jpegBody, image.NewRGBA(image.Rect(0, 0, 4, 4)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var docxBody bytes.Buffer
|
||||
zw := zip.NewWriter(&docxBody)
|
||||
w, err := zw.Create("word/document.xml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte("<w:document/>")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
webm := []byte("\x1a\x45\xdf\xa3\x9f\x42\x86\x81\x01\x42\xf7\x81\x01\x42\xf2\x81\x04\x42\xf3\x81\x08\x42\x82\x84webm\x42\x87\x81\x04\x42\x85\x81\x02")
|
||||
|
||||
for _, tc := range []struct {
|
||||
name, fileName string
|
||||
body []byte
|
||||
ext, ct string
|
||||
}{
|
||||
{"png named html", "x.html", smokePNG(t), "png", "image/png"},
|
||||
{"jpeg named png", "photo.png", jpegBody.Bytes(), "jpg", "image/jpeg"},
|
||||
{"jpeg named jpeg", "photo.jpeg", jpegBody.Bytes(), "jpeg", "image/jpeg"},
|
||||
{"docx under zip", "report.docx", docxBody.Bytes(), "docx", "application/zip"},
|
||||
{"ics under text", "event.ics", []byte("BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n"), "ics", "text/plain"},
|
||||
{"html named txt", "note.txt", []byte("<html><body><script>alert(1)</script></body></html>"), "txt", "text/html"},
|
||||
{"html named html", "page.html", []byte("<!DOCTYPE html><html><script>alert(1)</script></html>"), "txt", "text/html"},
|
||||
{"svg under xml", "logo.svg", []byte(`<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>`), "txt", "text/xml"},
|
||||
{"js under text", "app.js", []byte("console.log(document.cookie)\n"), "txt", "text/plain"},
|
||||
// A generic sniff keeps the content_type rule (from the extension).
|
||||
{"js under octet-stream", "app.js", []byte{0x00, 0x01, 0x02, 'a', 'l', 'e', 'r', 't'}, "bin", "text/javascript"},
|
||||
{"pdf named txt", "doc.txt", []byte("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n1 0 obj\n<<>>\nendobj\n"), "pdf", "application/pdf"},
|
||||
{"mkv under webm", "clip.mkv", webm, "mkv", "video/webm"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: tc.fileName, Body: bytes.NewReader(tc.body), Public: true}, lim)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(f.DiskName, "."+tc.ext) || !strings.HasSuffix(f.URL(), "/"+f.DiskName) || !strings.HasSuffix(f.URL(), "."+tc.ext) {
|
||||
t.Fatalf("disk name %q URL %q, want .%s", f.DiskName, f.URL(), tc.ext)
|
||||
}
|
||||
if f.FileName != tc.fileName || !strings.HasPrefix(f.ContentType, tc.ct) {
|
||||
t.Fatalf("file_name %q content_type %q, want %q %q", f.FileName, f.ContentType, tc.fileName, tc.ct)
|
||||
}
|
||||
var stored attach.File
|
||||
if err := gdb.First(&stored, f.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.DiskName != f.DiskName || stored.FileName != tc.fileName {
|
||||
t.Fatalf("stored row %+v", stored)
|
||||
}
|
||||
got, err := bucket.ReadAll(ctx, attach.BlobKey(f.DiskName))
|
||||
if err != nil || !bytes.Equal(got, tc.body) {
|
||||
t.Fatalf("blob under %s: %v", attach.BlobKey(f.DiskName), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,3 +362,107 @@ func TestBucketAndURLEdges(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicURLBase (plan 05 of phase 19.3, D-12) builds direct URLs from
|
||||
// storage.uploads.public_url: trailing slashes on the base and leading
|
||||
// slashes on the key collapse to one, File.URL and thumbnails use the base,
|
||||
// and clearing it restores the proxy prefix.
|
||||
func TestPublicURLBase(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
setPublicURLBase("")
|
||||
})
|
||||
key := BlobKey("651a2b3c4d5e61234567.png")
|
||||
for _, tc := range []struct{ base, key, want string }{
|
||||
{"https://cdn.example.com", key, "https://cdn.example.com/" + key},
|
||||
{"https://cdn.example.com/", key, "https://cdn.example.com/" + key},
|
||||
{"https://cdn.example.com/uploads//", "/" + key, "https://cdn.example.com/uploads/" + key},
|
||||
{"http://bucket.s3.example.com/a/b", "//" + key, "http://bucket.s3.example.com/a/b/" + key},
|
||||
} {
|
||||
setPublicURLBase(tc.base)
|
||||
if got := PublicURL(tc.key); got != tc.want {
|
||||
t.Errorf("base %q key %q: PublicURL = %q, want %q", tc.base, tc.key, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
bucket := memblob.OpenBucket(nil)
|
||||
t.Cleanup(func() { _ = bucket.Close() })
|
||||
setPublicURLBase("https://cdn.example.com/uploads/")
|
||||
if PublicURLBase() != "https://cdn.example.com/uploads/" {
|
||||
t.Fatalf("PublicURLBase = %q", PublicURLBase())
|
||||
}
|
||||
f := &File{ID: 31, DiskName: "651a2b3c4d5e61234567.png"}
|
||||
if got, want := f.URL(), "https://cdn.example.com/uploads/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
|
||||
t.Fatalf("File.URL = %q, want %q", got, want)
|
||||
}
|
||||
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
thumb, err := f.Thumb(t.Context(), bucket, 40, 30, "crop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "https://cdn.example.com/uploads/651/a2b/3c4/thumb_31_40_30_0_0_crop.png"; thumb != want {
|
||||
t.Fatalf("thumb = %q, want %q", thumb, want)
|
||||
}
|
||||
|
||||
setPublicURLBase("")
|
||||
if got := f.URL(); got != "/storage/uploads/651/a2b/3c4/651a2b3c4d5e61234567.png" {
|
||||
t.Fatalf("without a base File.URL = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicURLValidation refuses every malformed storage.uploads.public_url
|
||||
// at OpenBucket, naming the key and never echoing credentials; an unset or
|
||||
// blank value keeps the proxy prefix.
|
||||
func TestPublicURLValidation(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
setPublicURLBase("")
|
||||
})
|
||||
for _, bad := range []string{
|
||||
"ftp://cdn.example.com/uploads",
|
||||
"file:///srv/uploads",
|
||||
"https://user:secret@cdn.example.com",
|
||||
"https://token@cdn.example.com",
|
||||
"https://cdn.example.com/uploads?v=1",
|
||||
"https://cdn.example.com/uploads?",
|
||||
"https://cdn.example.com/uploads#top",
|
||||
"/storage/uploads",
|
||||
"cdn.example.com/uploads",
|
||||
"https:///uploads",
|
||||
"https://cdn.example.com:bad/",
|
||||
} {
|
||||
setPublicURLBase("https://stale.example.com")
|
||||
cfg := openTestConfig(t, "uploads:\n bucket_url: \"mem://\"\n public_url: \""+bad+"\"\n")
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if err == nil {
|
||||
_ = bucket.Close()
|
||||
t.Fatalf("public_url %q accepted", bad)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "storage.uploads.public_url") || strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "token@") {
|
||||
t.Fatalf("public_url %q: error %q", bad, err)
|
||||
}
|
||||
if _, perr := parsePublicURL(bad); perr == nil {
|
||||
t.Fatalf("parsePublicURL(%q) accepted", bad)
|
||||
}
|
||||
}
|
||||
for _, unset := range []string{"", " "} {
|
||||
setPublicURLBase("https://stale.example.com")
|
||||
cfg := openTestConfig(t, "uploads:\n bucket_url: \"mem://\"\n public_url: \""+unset+"\"\n")
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("public_url %q: %v", unset, err)
|
||||
}
|
||||
_ = bucket.Close()
|
||||
if PublicURLBase() != "" || PublicURL("a/b.png") != "/storage/uploads/a/b.png" {
|
||||
t.Fatalf("public_url %q: base %q URL %q", unset, PublicURLBase(), PublicURL("a/b.png"))
|
||||
}
|
||||
}
|
||||
for _, ok := range []string{"https://cdn.example.com", "HTTP://cdn.example.com:8080/u/", " https://cdn.example.com/uploads "} {
|
||||
u, err := parsePublicURL(ok)
|
||||
if err != nil || u == nil {
|
||||
t.Fatalf("parsePublicURL(%q) = %v, %v", ok, u, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user