c8d41a68c6
- lagoon: Date and TimeOfDay through JSON, text and real DATE/TIME columns; Fill text fallback without changing earlier conversions; required on zero dates; deferred_bindings shape, store isolation and envelope; PurgeDeferred cut-off, after-commit blobs, SKIP LOCKED, skipped types and the deferred:purge command - attach: Store limits, extensions, MIME patterns, default lists, key shape and blob cleanup; IsAllowedImage formats, polyglots, ceiling - conga and pact: framework purge schedule entry and forged jobs; the six relation child hook interfaces - cabana: fileupload and datepicker compile, upload, remove, caption, reorder and bounds; deferred commit order, rollback, applied-only and concurrent saves; relation contracts, forms, CRUD, deferral, schema
287 lines
11 KiB
Go
287 lines
11 KiB
Go
package attach_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"image"
|
|
"image/png"
|
|
"io"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
|
"gocloud.dev/blob"
|
|
"gocloud.dev/blob/memblob"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
func smokePNG(t *testing.T) []byte {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 4, 3))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func bucketKeys(t *testing.T, bucket *blob.Bucket) []string {
|
|
t.Helper()
|
|
var keys []string
|
|
iter := bucket.List(nil)
|
|
for {
|
|
obj, err := iter.Next(context.Background())
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
keys = append(keys, obj.Key)
|
|
}
|
|
return keys
|
|
}
|
|
|
|
// TestStoreSmoke stores a guarded PNG and a body of exactly MaxBytes.
|
|
func TestStoreSmoke(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("requires testcontainers postgres")
|
|
}
|
|
ctx := t.Context()
|
|
gdb := attachGorm(t)
|
|
if err := lagoon.Migrate(gdb, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
|
|
data := smokePNG(t)
|
|
f, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: `C:\photos\Cover.PNG`, Body: bytes.NewReader(data), Public: false}, attach.Limits{Image: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.ID == 0 || f.SortOrder != int(f.ID) {
|
|
t.Fatalf("sort_order %d, id %d", f.SortOrder, f.ID)
|
|
}
|
|
if !strings.HasSuffix(f.DiskName, ".png") || len(f.DiskName) != 26 {
|
|
t.Fatalf("disk name %q", f.DiskName)
|
|
}
|
|
if f.FileName != "Cover.PNG" || f.ContentType != "image/png" || f.FileSize != int64(len(data)) || f.Public() {
|
|
t.Fatalf("row %+v", f)
|
|
}
|
|
var stored attach.File
|
|
if err := gdb.First(&stored, f.ID).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stored.SortOrder != int(f.ID) || stored.Public() || stored.AttachmentID != "" {
|
|
t.Fatalf("stored row %+v", stored)
|
|
}
|
|
got, err := bucket.ReadAll(ctx, attach.BlobKey(f.DiskName))
|
|
if err != nil || !bytes.Equal(got, data) {
|
|
t.Fatalf("blob %v", err)
|
|
}
|
|
|
|
exact := bytes.Repeat([]byte("a"), 64)
|
|
g, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(exact)}, attach.Limits{MaxBytes: 64})
|
|
if err != nil {
|
|
t.Fatalf("exactly MaxBytes: %v", err)
|
|
}
|
|
if g.FileSize != 64 || g.Public() || g.ContentType != "text/plain" {
|
|
t.Fatalf("row %+v", g)
|
|
}
|
|
}
|
|
|
|
// TestStoreSmokeRefusals covers the refusals that happen before any row is
|
|
// written: an SVG in image mode and bodies of MaxBytes+1 bytes.
|
|
func TestStoreSmokeRefusals(t *testing.T) {
|
|
ctx := t.Context()
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
db := &gorm.DB{}
|
|
|
|
svg := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>`)
|
|
_, err := attach.Store(ctx, db, bucket, attach.Upload{FileName: "x.png", Body: bytes.NewReader(svg)}, attach.Limits{Image: true})
|
|
if !errors.Is(err, attach.ErrNotImage) {
|
|
t.Fatalf("svg bytes: %v", err)
|
|
}
|
|
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "x.svg", Body: bytes.NewReader(svg)}, attach.Limits{Image: true})
|
|
if !errors.Is(err, attach.ErrFileType) {
|
|
t.Fatalf("svg extension: %v", err)
|
|
}
|
|
|
|
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(bytes.Repeat([]byte("a"), 65))}, attach.Limits{MaxBytes: 64})
|
|
if !errors.Is(err, attach.ErrTooLarge) {
|
|
t.Fatalf("small body: %v", err)
|
|
}
|
|
// Past the 1 MiB read-ahead the limit is enforced while streaming.
|
|
const limit = 2 << 20
|
|
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(bytes.Repeat([]byte("a"), limit+1))}, attach.Limits{MaxBytes: limit})
|
|
if !errors.Is(err, attach.ErrTooLarge) {
|
|
t.Fatalf("streamed body: %v", err)
|
|
}
|
|
if keys := bucketKeys(t, bucket); len(keys) != 0 {
|
|
t.Fatalf("blobs left behind: %v", keys)
|
|
}
|
|
}
|
|
|
|
// TestStore covers attach.Store against Postgres (D-07, D-08): a failed
|
|
// row insert deletes its blob, size limits at the boundary and while
|
|
// streaming, extension and MIME rules, the default lists per mode, the
|
|
// content type from the sniff or the extension, the disk name and blob key
|
|
// shape, sort_order and is_public. No refusal leaves a blob behind.
|
|
func TestStore(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("requires testcontainers postgres")
|
|
}
|
|
ctx := t.Context()
|
|
gdb := attachGorm(t)
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
png := smokePNG(t)
|
|
|
|
t.Run("row failure deletes the blob", func(t *testing.T) {
|
|
// system_files does not exist yet: the insert fails after the blob
|
|
// was written, and Store deletes it again.
|
|
_, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: "a.png", Body: bytes.NewReader(png)}, attach.Limits{Image: true})
|
|
if err == nil || !strings.Contains(err.Error(), "store row") {
|
|
t.Fatalf("err = %v", err)
|
|
}
|
|
if keys := bucketKeys(t, bucket); len(keys) != 0 {
|
|
t.Fatalf("blobs left %v", keys)
|
|
}
|
|
})
|
|
if err := lagoon.Migrate(gdb, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
store := func(name string, body []byte, lim attach.Limits, public bool) (*attach.File, error) {
|
|
return attach.Store(ctx, gdb, bucket, attach.Upload{FileName: name, Body: bytes.NewReader(body), Public: public}, lim)
|
|
}
|
|
refuse := func(t *testing.T, what string, want error, name string, body []byte, lim attach.Limits) {
|
|
t.Helper()
|
|
before := len(bucketKeys(t, bucket))
|
|
f, err := store(name, body, lim, false)
|
|
if !errors.Is(err, want) {
|
|
t.Fatalf("%s: file=%v err=%v, want %v", what, f, err, want)
|
|
}
|
|
if after := len(bucketKeys(t, bucket)); after != before {
|
|
t.Fatalf("%s left a blob (%d -> %d)", what, before, after)
|
|
}
|
|
}
|
|
|
|
t.Run("size", func(t *testing.T) {
|
|
for _, n := range []int64{1, 64, 1 << 20, 1<<20 + 1, 3 << 20} {
|
|
body := bytes.Repeat([]byte("a"), int(n))
|
|
f, err := store("notes.txt", body, attach.Limits{MaxBytes: n}, false)
|
|
if err != nil || f.FileSize != n {
|
|
t.Fatalf("exactly %d bytes: %v %v", n, f, err)
|
|
}
|
|
refuse(t, "one byte over", attach.ErrTooLarge, "notes.txt", append(body, 'b'), attach.Limits{MaxBytes: n})
|
|
}
|
|
big := bytes.Repeat([]byte("z"), 3<<20)
|
|
if f, err := store("big.txt", big, attach.Limits{}, false); err != nil || f.FileSize != int64(len(big)) {
|
|
t.Fatalf("MaxBytes 0 means no limit: %v %v", f, err)
|
|
}
|
|
if _, err := store("neg.txt", []byte("x"), attach.Limits{MaxBytes: -1}, false); err == nil {
|
|
t.Fatal("a negative limit was accepted")
|
|
}
|
|
})
|
|
|
|
t.Run("extensions", func(t *testing.T) {
|
|
f, err := store(`..\..\Uploads/PHOTO.PNG`, png, attach.Limits{Image: true}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.FileName != "PHOTO.PNG" || !regexp.MustCompile(`^[0-9a-f]{22}\.png$`).MatchString(f.DiskName) {
|
|
t.Fatalf("name %q disk %q", f.FileName, f.DiskName)
|
|
}
|
|
key := attach.BlobKey(f.DiskName)
|
|
if strings.Contains(key, "Uploads") || strings.Contains(key, "..") || strings.Contains(key, "PHOTO") {
|
|
t.Fatalf("client path reached the key %q", key)
|
|
}
|
|
if ok, err := bucket.Exists(ctx, key); err != nil || !ok {
|
|
t.Fatalf("blob at %s: %v %v", key, ok, err)
|
|
}
|
|
if f, err := store("a.png", png, attach.Limits{Extensions: []string{" .PNG "}}, false); err != nil || !strings.HasSuffix(f.DiskName, ".png") {
|
|
t.Fatalf("normalised extension list: %v %v", f, err)
|
|
}
|
|
for _, name := range []string{"noext", "dot.", "a.toolongextension", "a.p-g", "a.p g", "a.ünï"} {
|
|
refuse(t, "name "+name, attach.ErrFileType, name, png, attach.Limits{Image: true})
|
|
}
|
|
refuse(t, "extension outside the list", attach.ErrFileType, "a.gif", png, attach.Limits{Extensions: []string{"png"}})
|
|
})
|
|
|
|
t.Run("default lists", func(t *testing.T) {
|
|
for _, ext := range []string{"svg", "js", "html", "css", "xml", "swf", "map", "less", "scss", "php"} {
|
|
refuse(t, "file mode ."+ext, attach.ErrFileType, "x."+ext, []byte("x"), attach.Limits{})
|
|
}
|
|
for _, ext := range []string{"pdf", "txt", "zip", "docx", "png"} {
|
|
if _, err := store("x."+ext, []byte("plain bytes"), attach.Limits{}, false); err != nil {
|
|
t.Fatalf("file mode .%s: %v", ext, err)
|
|
}
|
|
}
|
|
for _, ext := range []string{"pdf", "svg", "bmp", "avif"} {
|
|
refuse(t, "image mode ."+ext, attach.ErrFileType, "x."+ext, png, attach.Limits{Image: true})
|
|
}
|
|
refuse(t, "image mode non-image bytes", attach.ErrNotImage, "x.png", []byte("<html></html>"), attach.Limits{Image: true})
|
|
refuse(t, "image mode truncated", attach.ErrNotImage, "x.png", png[:16], attach.Limits{Image: true})
|
|
if f, err := store("x.webp", guardWebP, attach.Limits{Image: true}, false); err != nil || f.ContentType != "image/webp" {
|
|
t.Fatalf("webp: %v %v", f, err)
|
|
}
|
|
})
|
|
|
|
t.Run("content type and MIME patterns", func(t *testing.T) {
|
|
// The sniff wins over the extension.
|
|
if f, err := store("looks.txt", png, attach.Limits{Extensions: []string{"txt"}}, false); err != nil || f.ContentType != "image/png" {
|
|
t.Fatalf("sniffed type: %v %v", f, err)
|
|
}
|
|
// Bytes the sniff cannot place take the extension's type.
|
|
opaque := []byte{0x00, 0x01, 0x02, 0x03, 0xfe, 0xff}
|
|
if f, err := store("doc.pdf", opaque, attach.Limits{}, false); err != nil || f.ContentType != "application/pdf" {
|
|
t.Fatalf("extension fallback: %v %v", f, err)
|
|
}
|
|
if _, err := store("a.png", png, attach.Limits{MIMETypes: []string{"image/*"}}, false); err != nil {
|
|
t.Fatalf("image/*: %v", err)
|
|
}
|
|
if _, err := store("a.png", png, attach.Limits{MIMETypes: []string{"IMAGE/PNG"}}, false); err != nil {
|
|
t.Fatalf("case-insensitive pattern: %v", err)
|
|
}
|
|
if _, err := store("doc.pdf", opaque, attach.Limits{MIMETypes: []string{"pdf"}}, false); err != nil {
|
|
t.Fatalf("bare extension pattern: %v", err)
|
|
}
|
|
refuse(t, "text under image/*", attach.ErrMIMEType, "a.txt", []byte("hello"), attach.Limits{MIMETypes: []string{"image/*"}})
|
|
refuse(t, "png under text/plain", attach.ErrMIMEType, "a.png", png, attach.Limits{MIMETypes: []string{"text/plain", "", "jpg"}})
|
|
refuse(t, "png under */jpeg", attach.ErrMIMEType, "a.png", png, attach.Limits{MIMETypes: []string{"*/jpeg"}})
|
|
})
|
|
|
|
t.Run("row", func(t *testing.T) {
|
|
for _, public := range []bool{true, false} {
|
|
f, err := store("row.png", png, attach.Limits{Image: true}, public)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var stored attach.File
|
|
if err := gdb.First(&stored, f.ID).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stored.SortOrder != int(stored.ID) || stored.Public() != public || stored.AttachmentID != "" || stored.AttachmentType != "" || stored.Field != "" {
|
|
t.Fatalf("row %+v public=%v", stored, public)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("arguments", func(t *testing.T) {
|
|
if _, err := attach.Store(ctx, nil, bucket, attach.Upload{FileName: "a.png", Body: bytes.NewReader(png)}, attach.Limits{}); err == nil {
|
|
t.Fatal("nil db accepted")
|
|
}
|
|
if _, err := attach.Store(ctx, gdb, nil, attach.Upload{FileName: "a.png", Body: bytes.NewReader(png)}, attach.Limits{}); err == nil {
|
|
t.Fatal("nil bucket accepted")
|
|
}
|
|
if _, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: "a.png"}, attach.Limits{}); err == nil {
|
|
t.Fatal("nil body accepted")
|
|
}
|
|
})
|
|
}
|