feat(12.2-01): add deferred bindings, guarded upload store and purge

- deferred_bindings migration set under summercms.deferred with backend_user_id
- lagoon.DeferredBind/Unbind/Bindings/Forget/Slaves scoped by DeferredKey
- lagoon.PurgeDeferred with SKIP LOCKED batches and after-commit blob deletes
- attach.Store with the ported image guard, extension and MIME limits
- attach.Relation, attach.HasRelations, attach.BlobKeys, File.ThumbKey
- lagoon README and attachments docs
This commit is contained in:
Jakub Zych
2026-10-02 17:36:43 +02:00
parent 79e2a43095
commit 19f4cf8232
14 changed files with 1280 additions and 15 deletions

View File

@@ -107,7 +107,10 @@ func init() {
Register(&File{})
}
func blobKeysFor(f File) []string {
// BlobKeys returns the blob keys of f: the original's partitioned key and
// the thumb_<id>_ prefix of its thumbnails. DeleteKeys treats the second as
// a prefix, so passing both removes the original and every thumbnail.
func BlobKeys(f File) []string {
part := PartitionDirectory(f.DiskName)
return []string{
part + f.DiskName,
@@ -141,7 +144,7 @@ func DeleteForOwner(tx *gorm.DB, owner Owner, ownerID string, afterCommit func(b
}
var keys []string
for _, f := range files {
keys = append(keys, blobKeysFor(f)...)
keys = append(keys, BlobKeys(f)...)
}
if err := tx.Where("attachment_type = ? AND attachment_id = ?", morph, ownerID).Delete(&File{}).Error; err != nil {
return err

View File

@@ -86,7 +86,7 @@ func TestDeleteKeysThumbPrefixIsIDDelimited(t *testing.T) {
t.Fatal(err)
}
}
if err := DeleteKeys(ctx, bucket, blobKeysFor(File{ID: 4, DiskName: disk})); err != nil {
if err := DeleteKeys(ctx, bucket, BlobKeys(File{ID: 4, DiskName: disk})); err != nil {
t.Fatal(err)
}
for _, key := range doomed {

View File

@@ -0,0 +1,47 @@
package attach
import (
"bytes"
"image"
"net/http"
"slices"
)
// AllowedImageMIMEs are the content types IsAllowedImage accepts, as
// sniffed from the bytes: JPEG, PNG, GIF and WebP, the formats the
// thumbnailer decodes.
var AllowedImageMIMEs = []string{"image/jpeg", "image/png", "image/gif", "image/webp"}
// MaxImagePixels is the largest image (width times height) IsAllowedImage
// accepts, the same ceiling File.Thumb applies before decoding, so every
// accepted image can be thumbnailed.
const MaxImagePixels = maxThumbSourcePixels
// IsAllowedImage reports whether data is a JPEG, PNG, GIF or WebP image:
// the bytes must sniff as one of AllowedImageMIMEs (http.DetectContentType,
// independent of any file name or client header), the header must decode
// through image.DecodeConfig as that format with a positive width and
// height, and the image must not exceed MaxImagePixels. Decoding the header
// rejects a polyglot whose first bytes alone look right. It fails closed:
// empty or unreadable content is refused. data may be a prefix of the file
// as long as it holds the image header.
func IsAllowedImage(data []byte) bool {
if len(data) == 0 {
return false
}
if !slices.Contains(AllowedImageMIMEs, http.DetectContentType(data)) {
return false
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
return false
}
if int64(cfg.Width)*int64(cfg.Height) > MaxImagePixels {
return false
}
switch format {
case "jpeg", "png", "gif", "webp":
return true
}
return false
}

View File

@@ -0,0 +1,21 @@
package attach
// Relation declares one WinterCMS attachOne or attachMany relation of an
// Owner model. Name is the relation name stored in system_files.field, Many
// is true for attachMany (false for attachOne), and Public decides the
// is_public flag of files stored through the relation: a protected relation
// (Public false) stores is_public=false rows, for which the framework never
// builds a public URL.
type Relation struct {
Name string
Many bool
Public bool
}
// HasRelations is implemented by an Owner model that declares its
// attachment relations, the Go form of WinterCMS's $attachOne and
// $attachMany arrays. A form field that edits attachments must name one of
// the declared relations.
type HasRelations interface {
AttachRelations() []Relation
}

View File

@@ -0,0 +1,281 @@
package attach
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"mime"
"net/http"
"path"
"regexp"
"slices"
"strings"
"gocloud.dev/blob"
"gorm.io/gorm"
)
// sniffBytes is how much of an upload Store reads ahead for the content
// sniff and the image guard.
const sniffBytes = 1 << 20
var (
// ErrTooLarge is returned by Store when the body exceeds Limits.MaxBytes.
ErrTooLarge = errors.New("attach: file is too large")
// ErrFileType is returned by Store when the file name's extension is
// missing, malformed or not in the allowed extension list.
ErrFileType = errors.New("attach: file type is not allowed")
// ErrMIMEType is returned by Store when the content type matches none
// of Limits.MIMETypes.
ErrMIMEType = errors.New("attach: file content type is not allowed")
// ErrNotImage is returned by Store in image mode when the bytes are not
// an image IsAllowedImage accepts.
ErrNotImage = errors.New("attach: file is not an allowed image")
)
// DefaultImageExtensions is the extension list of an image upload when
// Limits.Extensions is empty: jpg, jpeg, png, gif and webp, the formats
// IsAllowedImage and the thumbnailer handle. WinterCMS's image list also
// has avif, bmp and svg; they are left out because nothing here decodes
// them and svg can carry script.
var DefaultImageExtensions = []string{"jpg", "jpeg", "png", "gif", "webp"}
// DefaultFileExtensions is the extension list of a file upload when
// Limits.Extensions is empty: WinterCMS's default list (winter/storm
// Filesystem\Definitions::defaultExtensions) minus the script-capable types
// svg, js, map, css, less, scss, swf and xml. The final list is avi, avif,
// bmp, doc, docx, eot, flv, gif, ico, ics, jpeg, jpg, mkv, mov, mp3, mp4,
// mpeg, ods, odt, ogg, pdf, png, ppt, pptx, rar, ttf, txt, wav, webm, webp,
// wmv, woff, woff2, xls, xlsx and zip.
var DefaultFileExtensions = []string{
"avi", "avif", "bmp", "doc", "docx", "eot", "flv", "gif", "ico", "ics",
"jpeg", "jpg", "mkv", "mov", "mp3", "mp4", "mpeg", "ods", "odt", "ogg",
"pdf", "png", "ppt", "pptx", "rar", "ttf", "txt", "wav", "webm", "webp",
"wmv", "woff", "woff2", "xls", "xlsx", "zip",
}
var extPattern = regexp.MustCompile(`^[a-z0-9]{1,10}$`)
// Upload is one file to store. FileName is the client's file name: only its
// extension and base name are used (for the allowed-type check and the
// file_name column); no part of it reaches a blob key. Body is read once,
// to the end or to the size limit. Public sets the row's is_public flag.
type Upload struct {
FileName string
Body io.Reader
Public bool
}
// Limits restricts what Store accepts.
//
// MaxBytes is the largest body in bytes; 0 means no limit of its own (the
// caller's request body cap still applies). Extensions lists the allowed
// lower-case extensions without the dot; empty means DefaultImageExtensions
// when Image is set, else DefaultFileExtensions. MIMETypes, when not empty,
// must match the stored content type: an entry containing a slash is a MIME
// pattern such as "image/png" or "image/*", an entry without one is an
// extension. Image applies the image guard (IsAllowedImage) to the content.
type Limits struct {
MaxBytes int64
Extensions []string
MIMETypes []string
Image bool
}
// Store saves an upload as an unattached system_files row.
//
// It accepts the client extension, lower-cased, only when it matches
// [a-z0-9]{1,10} and is allowed by Limits (else ErrFileType). It reads up to
// 1 MiB ahead to sniff the content type from the bytes; in image mode those
// bytes must pass IsAllowedImage (else ErrNotImage), and Limits.MIMETypes is
// checked against the sniffed type, or the extension's registered type when
// the sniff only says application/octet-stream (else ErrMIMEType). The body
// is then streamed into bucket at BlobKey of a server-generated disk name (22
// random lowercase hex characters, a dot and the extension); a body longer
// than Limits.MaxBytes aborts the write, deletes the key and returns
// ErrTooLarge. Finally it inserts the row with empty attachment columns,
// is_public from Upload.Public, the byte size and the content type, and sets
// sort_order to the new id as WinterCMS's Sortable trait does. When the row
// cannot be written the blob is deleted again.
//
// db may be a transaction. The blob is written before the row, so a caller
// whose transaction rolls back after Store returned must delete the
// returned file's BlobKeys itself.
func Store(ctx context.Context, db *gorm.DB, bucket *blob.Bucket, in Upload, lim Limits) (*File, error) {
if ctx == nil {
ctx = context.Background()
}
if db == nil {
return nil, fmt.Errorf("attach: store db is nil")
}
if bucket == nil {
return nil, fmt.Errorf("attach: bucket is nil")
}
if in.Body == nil {
return nil, fmt.Errorf("attach: upload body is nil")
}
if lim.MaxBytes < 0 {
return nil, fmt.Errorf("attach: negative size limit %d", lim.MaxBytes)
}
name := clientBaseName(in.FileName)
ext := strings.ToLower(strings.TrimPrefix(path.Ext(name), "."))
if !extPattern.MatchString(ext) || !slices.Contains(allowedExtensions(lim), ext) {
return nil, fmt.Errorf("%w: %q", ErrFileType, ext)
}
br := bufio.NewReaderSize(in.Body, sniffBytes)
head, err := br.Peek(sniffBytes)
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {
return nil, fmt.Errorf("attach: read upload: %w", err)
}
if lim.MaxBytes > 0 && int64(len(head)) > lim.MaxBytes {
return nil, ErrTooLarge
}
if lim.Image && !IsAllowedImage(head) {
return nil, ErrNotImage
}
contentType := baseMediaType(http.DetectContentType(head))
if contentType == "application/octet-stream" {
if byExt := baseMediaType(mime.TypeByExtension("." + ext)); byExt != "" {
contentType = byExt
}
}
if len(lim.MIMETypes) > 0 && !mimeAllowed(lim.MIMETypes, contentType, ext) {
return nil, fmt.Errorf("%w: %s", ErrMIMEType, contentType)
}
diskName, err := newDiskName(ext)
if err != nil {
return nil, err
}
key := BlobKey(diskName)
size, err := writeBlob(ctx, bucket, key, br, contentType, lim.MaxBytes)
if err != nil {
return nil, err
}
public := in.Public
f := &File{
DiskName: diskName,
FileName: name,
FileSize: size,
ContentType: contentType,
IsPublic: &public,
}
q := db.Session(&gorm.Session{NewDB: true, Context: ctx})
err = q.Transaction(func(tx *gorm.DB) error {
if err := tx.Create(f).Error; err != nil {
return err
}
f.SortOrder = int(f.ID)
return tx.Model(&File{}).Where("id = ?", f.ID).Update("sort_order", f.SortOrder).Error
})
if err != nil {
_ = deleteKey(context.WithoutCancel(ctx), bucket, key)
return nil, fmt.Errorf("attach: store row: %w", err)
}
return f, nil
}
// writeBlob streams r into key and returns the byte count. With limit > 0 a
// body of more than limit bytes aborts the write and deletes the key.
func writeBlob(ctx context.Context, bucket *blob.Bucket, key string, r io.Reader, contentType string, limit int64) (int64, error) {
writeCtx, cancel := context.WithCancel(ctx)
defer cancel()
w, err := bucket.NewWriter(writeCtx, key, &blob.WriterOptions{ContentType: contentType})
if err != nil {
return 0, fmt.Errorf("attach: blob writer: %w", err)
}
src := r
if limit > 0 {
src = io.LimitReader(r, limit+1)
}
n, copyErr := io.Copy(w, src)
if copyErr == nil && limit > 0 && n > limit {
copyErr = ErrTooLarge
}
if copyErr != nil {
// Cancelling the writer's context before Close discards the write.
cancel()
_ = w.Close()
_ = deleteKey(context.WithoutCancel(ctx), bucket, key)
if errors.Is(copyErr, ErrTooLarge) {
return 0, ErrTooLarge
}
return 0, fmt.Errorf("attach: write upload: %w", copyErr)
}
if err := w.Close(); err != nil {
_ = deleteKey(context.WithoutCancel(ctx), bucket, key)
return 0, fmt.Errorf("attach: write upload: %w", err)
}
return n, nil
}
// clientBaseName is the last element of a client file name, with either
// slash style treated as a separator.
func clientBaseName(name string) string {
name = strings.ReplaceAll(name, `\`, "/")
if i := strings.LastIndex(name, "/"); i >= 0 {
name = name[i+1:]
}
return strings.TrimSpace(name)
}
func allowedExtensions(lim Limits) []string {
if len(lim.Extensions) == 0 {
if lim.Image {
return DefaultImageExtensions
}
return DefaultFileExtensions
}
out := make([]string, 0, len(lim.Extensions))
for _, e := range lim.Extensions {
out = append(out, strings.ToLower(strings.TrimPrefix(strings.TrimSpace(e), ".")))
}
return out
}
func baseMediaType(ct string) string {
if ct == "" {
return ""
}
mt, _, err := mime.ParseMediaType(ct)
if err != nil {
return strings.ToLower(strings.TrimSpace(strings.SplitN(ct, ";", 2)[0]))
}
return mt
}
// mimeAllowed reports whether contentType or ext matches one of patterns.
func mimeAllowed(patterns []string, contentType, ext string) bool {
for _, p := range patterns {
p = strings.ToLower(strings.TrimSpace(p))
if p == "" {
continue
}
if !strings.Contains(p, "/") {
if strings.TrimPrefix(p, ".") == ext {
return true
}
continue
}
pType, pSub, _ := strings.Cut(p, "/")
cType, cSub, _ := strings.Cut(contentType, "/")
if (pType == "*" || pType == cType) && (pSub == "*" || pSub == cSub) {
return true
}
}
return false
}
func newDiskName(ext string) (string, error) {
raw := make([]byte, 11)
if _, err := rand.Read(raw); err != nil {
return "", fmt.Errorf("attach: disk name: %w", err)
}
return hex.EncodeToString(raw) + "." + ext, nil
}

View File

@@ -0,0 +1,126 @@
package attach_test
import (
"bytes"
"context"
"errors"
"image"
"image/png"
"io"
"strings"
"testing"
"git.golem15.com/golem15/summercms/modules/lagoon"
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
"gocloud.dev/blob"
"gocloud.dev/blob/memblob"
"gorm.io/gorm"
)
func smokePNG(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 4, 3))); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
func bucketKeys(t *testing.T, bucket *blob.Bucket) []string {
t.Helper()
var keys []string
iter := bucket.List(nil)
for {
obj, err := iter.Next(context.Background())
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
keys = append(keys, obj.Key)
}
return keys
}
// TestStoreSmoke stores a guarded PNG and a body of exactly MaxBytes.
func TestStoreSmoke(t *testing.T) {
if testing.Short() {
t.Skip("requires testcontainers postgres")
}
ctx := t.Context()
gdb := attachGorm(t)
if err := lagoon.Migrate(gdb, nil); err != nil {
t.Fatal(err)
}
bucket := memblob.OpenBucket(nil)
t.Cleanup(func() { _ = bucket.Close() })
data := smokePNG(t)
f, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: `C:\photos\Cover.PNG`, Body: bytes.NewReader(data), Public: false}, attach.Limits{Image: true})
if err != nil {
t.Fatal(err)
}
if f.ID == 0 || f.SortOrder != int(f.ID) {
t.Fatalf("sort_order %d, id %d", f.SortOrder, f.ID)
}
if !strings.HasSuffix(f.DiskName, ".png") || len(f.DiskName) != 26 {
t.Fatalf("disk name %q", f.DiskName)
}
if f.FileName != "Cover.PNG" || f.ContentType != "image/png" || f.FileSize != int64(len(data)) || f.Public() {
t.Fatalf("row %+v", f)
}
var stored attach.File
if err := gdb.First(&stored, f.ID).Error; err != nil {
t.Fatal(err)
}
if stored.SortOrder != int(f.ID) || stored.Public() || stored.AttachmentID != "" {
t.Fatalf("stored row %+v", stored)
}
got, err := bucket.ReadAll(ctx, attach.BlobKey(f.DiskName))
if err != nil || !bytes.Equal(got, data) {
t.Fatalf("blob %v", err)
}
exact := bytes.Repeat([]byte("a"), 64)
g, err := attach.Store(ctx, gdb, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(exact)}, attach.Limits{MaxBytes: 64})
if err != nil {
t.Fatalf("exactly MaxBytes: %v", err)
}
if g.FileSize != 64 || g.Public() || g.ContentType != "text/plain" {
t.Fatalf("row %+v", g)
}
}
// TestStoreSmokeRefusals covers the refusals that happen before any row is
// written: an SVG in image mode and bodies of MaxBytes+1 bytes.
func TestStoreSmokeRefusals(t *testing.T) {
ctx := t.Context()
bucket := memblob.OpenBucket(nil)
t.Cleanup(func() { _ = bucket.Close() })
db := &gorm.DB{}
svg := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>`)
_, err := attach.Store(ctx, db, bucket, attach.Upload{FileName: "x.png", Body: bytes.NewReader(svg)}, attach.Limits{Image: true})
if !errors.Is(err, attach.ErrNotImage) {
t.Fatalf("svg bytes: %v", err)
}
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "x.svg", Body: bytes.NewReader(svg)}, attach.Limits{Image: true})
if !errors.Is(err, attach.ErrFileType) {
t.Fatalf("svg extension: %v", err)
}
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(bytes.Repeat([]byte("a"), 65))}, attach.Limits{MaxBytes: 64})
if !errors.Is(err, attach.ErrTooLarge) {
t.Fatalf("small body: %v", err)
}
// Past the 1 MiB read-ahead the limit is enforced while streaming.
const limit = 2 << 20
_, err = attach.Store(ctx, db, bucket, attach.Upload{FileName: "notes.txt", Body: bytes.NewReader(bytes.Repeat([]byte("a"), limit+1))}, attach.Limits{MaxBytes: limit})
if !errors.Is(err, attach.ErrTooLarge) {
t.Fatalf("streamed body: %v", err)
}
if keys := bucketKeys(t, bucket); len(keys) != 0 {
t.Fatalf("blobs left behind: %v", keys)
}
}

View File

@@ -126,6 +126,19 @@ var encodeImage = defaultEncodeImage
// Thumb returns the public URL of a lazily generated thumbnail. The second
// call for the same dimensions hits the existing blob and does not resize.
// It is PublicURL of ThumbKey; see ThumbKey for the generation rules.
func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode string) (string, error) {
key, err := f.ThumbKey(ctx, bucket, w, h, mode)
if err != nil {
return "", err
}
return PublicURL(key), nil
}
// ThumbKey returns the blob key of a lazily generated thumbnail, generating
// it on first use, so a caller that must not emit a public URL (a protected
// file) can stream the thumbnail itself. The second call for the same
// dimensions hits the existing blob and does not resize.
//
// As WinterCMS's File::makeThumb does, an original that is missing from
// the bucket, does not decode, or declares more than 4096 by 4096 pixels
@@ -133,7 +146,7 @@ var encodeImage = defaultEncodeImage
// thumbnail, and the failure is logged at warn level instead of returned:
// one unusable upload never fails the listings that show it. An invalid
// mode or size, a storage error and an encode failure are still errors.
func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode string) (string, error) {
func (f *File) ThumbKey(ctx context.Context, bucket *blob.Bucket, w, h int, mode string) (string, error) {
if f == nil {
return "", fmt.Errorf("attach: file is nil")
}
@@ -162,7 +175,7 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
return "", fmt.Errorf("attach: thumb exists: %w", err)
}
if exists {
return PublicURL(thumbKey), nil
return thumbKey, nil
}
origKey := part + f.DiskName
r, err := bucket.NewReader(ctx, origKey, nil)
@@ -219,7 +232,7 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
}
return "", closeErr
}
return PublicURL(thumbKey), nil
return thumbKey, nil
}
var errOriginalTooLarge = errors.New("original image is too large")
@@ -239,12 +252,12 @@ var BrokenImagePNG = func() []byte {
}()
// brokenThumb is the catch branch of WinterCMS's File::makeThumb: log the
// reason, store BrokenImagePNG under the thumbnail key and return its URL.
// reason, store BrokenImagePNG under the thumbnail key and return that key.
func brokenThumb(ctx context.Context, bucket *blob.Bucket, f *File, thumbKey string, reason error) (string, error) {
slog.Default().WarnContext(ctx, "attach: thumbnail original is unusable, storing the broken-image picture",
slog.Uint64("file_id", uint64(f.ID)), slog.String("error", reason.Error()))
if err := bucket.WriteAll(ctx, thumbKey, BrokenImagePNG, &blob.WriterOptions{ContentType: "image/png"}); err != nil {
return "", fmt.Errorf("attach: broken-image thumb: %w", err)
}
return PublicURL(thumbKey), nil
return thumbKey, nil
}