Jakub Zych
2c961b8408
uniqueMediaKey checked Exists and the write followed without a precondition, so two uploads of the same name (clipboard images are all image.png) could both pick the free key and the second replaced the first. StoreMedia now reserves the chosen key in-process while it writes, and writes with IfNotExist; a key another process stored after the check moves the upload on to the next free name.
257 lines
8.7 KiB
Go
257 lines
8.7 KiB
Go
package classes
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
|
"gocloud.dev/blob"
|
|
"gocloud.dev/blob/memblob"
|
|
)
|
|
|
|
func TestMediaObjectPath(t *testing.T) {
|
|
got := mediaObjectPath("../etc", "cover.png")
|
|
if !strings.HasPrefix(got, "/journal/") {
|
|
t.Fatalf("path = %q, want /journal/ prefix", got)
|
|
}
|
|
if strings.Contains(got, "..") {
|
|
t.Fatalf("path still contains ..: %q", got)
|
|
}
|
|
if got := mediaObjectPath("photos", "a.png"); got != "/journal/photos/a.png" {
|
|
t.Fatalf("nested = %q", got)
|
|
}
|
|
if got := mediaObjectPath("journal/photos", "a.png"); got != "/journal/photos/a.png" {
|
|
t.Fatalf("prefixed folder = %q", got)
|
|
}
|
|
if got := mediaObjectPath("", "a.png"); got != "/journal/a.png" {
|
|
t.Fatalf("no folder = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestMediaFolderPatternRejectsDotDot(t *testing.T) {
|
|
folder := "../etc"
|
|
if MediaFolderPattern.MatchString(folder) && !strings.Contains(folder, "..") {
|
|
t.Fatal("folder ../etc should be rejected")
|
|
}
|
|
if !strings.Contains(folder, "..") {
|
|
t.Fatal("test fixture lost ..")
|
|
}
|
|
if MediaFolderPattern.MatchString("ok_folder-1") {
|
|
return
|
|
}
|
|
t.Fatal("ok_folder-1 should match")
|
|
}
|
|
|
|
// failingReader fails every read.
|
|
type failingReader struct{}
|
|
|
|
func (failingReader) Read([]byte) (int, error) { return 0, errors.New("disk on fire") }
|
|
|
|
func TestStoreMedia(t *testing.T) {
|
|
ctx := context.Background()
|
|
bucket := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = bucket.Close() })
|
|
mediaErr := func(t *testing.T, err error, field, message string) {
|
|
t.Helper()
|
|
var refused *MediaError
|
|
if !errors.As(err, &refused) || refused.Field != field || refused.Message != message || refused.Error() != message {
|
|
t.Fatalf("error = %v, want MediaError on %s %q", err, field, message)
|
|
}
|
|
}
|
|
|
|
t.Run("allowed extensions, key layout and URL", func(t *testing.T) {
|
|
for _, tc := range []struct{ folder, name, path string }{
|
|
{"photos", "a.jpg", "/journal/photos/a.jpg"},
|
|
{"/journal/photos/", "b.JPEG", "/journal/photos/b.JPEG"},
|
|
{"", "c.png", "/journal/c.png"},
|
|
{"x//y", "d.gif", "/journal/x/y/d.gif"},
|
|
{"Journal", "e.webp", "/journal/e.webp"},
|
|
{"posts", `C:\uploads\f.png`, "/journal/posts/f.png"},
|
|
{"posts", "../../g.png", "/journal/posts/g.png"},
|
|
} {
|
|
got, err := StoreMedia(ctx, bucket, tc.folder, tc.name, bytes.NewReader(testPNG))
|
|
if err != nil {
|
|
t.Fatalf("%s/%s: %v", tc.folder, tc.name, err)
|
|
}
|
|
key := strings.TrimPrefix(tc.path, "/")
|
|
if got.Path != tc.path || got.URL != attach.PublicURL(key) {
|
|
t.Fatalf("%s/%s = %+v, want path %s", tc.folder, tc.name, got, tc.path)
|
|
}
|
|
data, err := bucket.ReadAll(ctx, key)
|
|
if err != nil || !bytes.Equal(data, testPNG) {
|
|
t.Fatalf("stored %s: %v", key, err)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("a taken name gets the next numbered suffix", func(t *testing.T) {
|
|
var paths []string
|
|
for i := 0; i < 3; i++ {
|
|
got, err := StoreMedia(ctx, bucket, "dup", "same.png", bytes.NewReader(testPNG))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
paths = append(paths, got.Path)
|
|
}
|
|
if strings.Join(paths, " ") != "/journal/dup/same.png /journal/dup/same-1.png /journal/dup/same-2.png" {
|
|
t.Fatalf("paths = %v", paths)
|
|
}
|
|
})
|
|
|
|
// WR-03: concurrent uploads of the same name each get their own key and
|
|
// keep their own bytes.
|
|
t.Run("concurrent uploads of one name never overwrite each other", func(t *testing.T) {
|
|
race := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = race.Close() })
|
|
const uploads = 16
|
|
var wg sync.WaitGroup
|
|
results := make([]StoredMedia, uploads)
|
|
errs := make([]error, uploads)
|
|
bodies := make([][]byte, uploads)
|
|
for i := range uploads {
|
|
bodies[i] = append(append([]byte{}, testPNG...), []byte(fmt.Sprintf("upload-%02d", i))...)
|
|
}
|
|
start := make(chan struct{})
|
|
for i := range uploads {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
results[i], errs[i] = StoreMedia(ctx, race, "editor", "image.png", bytes.NewReader(bodies[i]))
|
|
}()
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
seen := map[string]int{}
|
|
for i := range uploads {
|
|
if errs[i] != nil {
|
|
t.Fatalf("upload %d: %v", i, errs[i])
|
|
}
|
|
if prev, dup := seen[results[i].Path]; dup {
|
|
t.Fatalf("uploads %d and %d both stored %s", prev, i, results[i].Path)
|
|
}
|
|
seen[results[i].Path] = i
|
|
data, err := race.ReadAll(ctx, strings.TrimPrefix(results[i].Path, "/"))
|
|
if err != nil || !bytes.Equal(data, bodies[i]) {
|
|
t.Fatalf("upload %d at %s holds another upload's bytes (err %v)", i, results[i].Path, err)
|
|
}
|
|
}
|
|
if len(mediaReservations.keys) != 0 {
|
|
t.Fatalf("reservations left behind: %v", mediaReservations.keys)
|
|
}
|
|
})
|
|
|
|
t.Run("a key another process stores after the check moves the upload on", func(t *testing.T) {
|
|
other := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = other.Close() })
|
|
planted := map[string]bool{}
|
|
mediaBeforeWrite = func(ctx context.Context, b *blob.Bucket, key string) {
|
|
if len(planted) < 2 {
|
|
planted[key] = true
|
|
if err := b.WriteAll(ctx, key, []byte("other process"), nil); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
}
|
|
t.Cleanup(func() { mediaBeforeWrite = nil })
|
|
got, err := StoreMedia(ctx, other, "editor", "image.png", bytes.NewReader(testPNG))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Path != "/journal/editor/image-2.png" {
|
|
t.Fatalf("path = %s, want the first name nobody else wrote", got.Path)
|
|
}
|
|
for key := range planted {
|
|
if data, _ := other.ReadAll(ctx, key); string(data) != "other process" {
|
|
t.Fatalf("%s was overwritten", key)
|
|
}
|
|
}
|
|
if data, _ := other.ReadAll(ctx, "journal/editor/image-2.png"); !bytes.Equal(data, testPNG) {
|
|
t.Fatal("the upload's bytes are not at its path")
|
|
}
|
|
})
|
|
|
|
t.Run("a writer that always loses gives up with an opaque error", func(t *testing.T) {
|
|
busy := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = busy.Close() })
|
|
mediaBeforeWrite = func(ctx context.Context, b *blob.Bucket, key string) {
|
|
_ = b.WriteAll(ctx, key, []byte("other process"), nil)
|
|
}
|
|
t.Cleanup(func() { mediaBeforeWrite = nil })
|
|
_, err := StoreMedia(ctx, busy, "editor", "image.png", bytes.NewReader(testPNG))
|
|
if err == nil || errors.As(err, new(*MediaError)) {
|
|
t.Fatalf("error = %v, want an opaque error", err)
|
|
}
|
|
if len(mediaReservations.keys) != 0 {
|
|
t.Fatalf("reservations left behind: %v", mediaReservations.keys)
|
|
}
|
|
})
|
|
|
|
t.Run("a name with every suffix taken fails instead of overwriting", func(t *testing.T) {
|
|
full := memblob.OpenBucket(nil)
|
|
t.Cleanup(func() { _ = full.Close() })
|
|
keys := []string{"journal/full/a.png"}
|
|
for n := 1; n < mediaMaxSuffix; n++ {
|
|
keys = append(keys, "journal/full/a-"+strconv.Itoa(n)+".png")
|
|
}
|
|
for _, key := range keys {
|
|
if err := full.WriteAll(ctx, key, []byte("old"), nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
_, err := StoreMedia(ctx, full, "full", "a.png", bytes.NewReader(testPNG))
|
|
if err == nil || errors.As(err, new(*MediaError)) {
|
|
t.Fatalf("error = %v, want an opaque error", err)
|
|
}
|
|
for _, key := range keys {
|
|
if data, _ := full.ReadAll(ctx, key); string(data) != "old" {
|
|
t.Fatalf("%s was overwritten", key)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("other extensions are refused", func(t *testing.T) {
|
|
for _, name := range []string{"a.svg", "a.html", "a.bmp", "a", "a.png.exe", ".png.txt"} {
|
|
_, err := StoreMedia(ctx, bucket, "", name, bytes.NewReader(testPNG))
|
|
mediaErr(t, err, "file", "The file must be an image.")
|
|
}
|
|
})
|
|
|
|
t.Run("the size limit is 10240 KiB", func(t *testing.T) {
|
|
if MediaMaxBytes != 10240*1024 {
|
|
t.Fatalf("MediaMaxBytes = %d", MediaMaxBytes)
|
|
}
|
|
exact := make([]byte, MediaMaxBytes)
|
|
if _, err := StoreMedia(ctx, bucket, "size", "exact.png", bytes.NewReader(exact)); err != nil {
|
|
t.Fatalf("exactly MediaMaxBytes: %v", err)
|
|
}
|
|
_, err := StoreMedia(ctx, bucket, "size", "over.png", bytes.NewReader(make([]byte, MediaMaxBytes+1)))
|
|
mediaErr(t, err, "file", "The file may not be greater than 10240 kilobytes.")
|
|
if ok, _ := bucket.Exists(ctx, "journal/size/over.png"); ok {
|
|
t.Fatal("an oversized file was stored")
|
|
}
|
|
})
|
|
|
|
t.Run("the folder rule is checked", func(t *testing.T) {
|
|
for _, folder := range []string{"../up", "a..b", "a b", `a\b`, strings.Repeat("a", 101), "a?b"} {
|
|
_, err := StoreMedia(ctx, bucket, folder, "a.png", bytes.NewReader(testPNG))
|
|
mediaErr(t, err, "folder", "The folder format is invalid.")
|
|
}
|
|
})
|
|
|
|
t.Run("a missing bucket and a read failure are opaque errors", func(t *testing.T) {
|
|
if _, err := StoreMedia(ctx, nil, "", "a.png", bytes.NewReader(testPNG)); err == nil || errors.As(err, new(*MediaError)) {
|
|
t.Fatalf("nil bucket error = %v", err)
|
|
}
|
|
if _, err := StoreMedia(ctx, bucket, "", "a.png", failingReader{}); err == nil || errors.As(err, new(*MediaError)) || !strings.Contains(err.Error(), "disk on fire") {
|
|
t.Fatalf("read failure error = %v", err)
|
|
}
|
|
})
|
|
}
|