feat(12-01): record multipart uploads and match Winter upload URLs

- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the
  thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the
  image size from the header before decoding
- tide requests carry multipart parts (files beside the fixture pinned by
  sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive
  byte-identical bodies
- tide masks the random partition, disk name and file id of url/thumb_url
  upload URLs while still diffing prefix, size, mode and extension, and
  NormalizePublications masks Carbon dates in the published album
This commit is contained in:
Jakub Zych
2026-10-02 11:33:42 +02:00
parent f9b7f2ea33
commit e06e0cc8bf
19 changed files with 870 additions and 34 deletions

View File

@@ -24,7 +24,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he
- Column types: `lagoon.Encrypted` stores AES-256-GCM ciphertext under a key derived from `app.key`, decrypts with previous keys during rotation, and always redacts itself in JSON and string output; `lagoon.Jsonable` stores JSON as TEXT and keeps SQL NULL distinct from an empty value.
- Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns.
- Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports.
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), on-demand thumbnails through `attach.File.Thumb`, static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
## Usage
@@ -151,7 +151,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration {
| `lagoon.WithSoftDeleteCascade` | Runs a cascade inside the parent delete transaction. |
| `lagoon.RegisterJoinTable` | Registers a custom pivot model for a many-to-many field. |
| `lagoon.DecryptLaravelPayload` | Decrypts a Laravel AES-256-CBC payload for data imports. |
| `attach.File` | The `system_files` row model. |
| `attach.File` | The `system_files` row model; `attach.File.URL` is the public URL of the original. |
| `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. |
| `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. |
| `attach.OpenBucket` | Opens the uploads bucket from config. |
| `attach.Publish` | Stores the bucket on the `backpack.App`. |
@@ -198,7 +199,7 @@ storage:
- SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages).
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`).
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails).
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding).
- Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others.
## Testing

View File

@@ -1,6 +1,7 @@
package attach
import (
"bytes"
"context"
"fmt"
"image"
@@ -14,6 +15,10 @@ import (
"github.com/disintegration/imaging"
"gocloud.dev/blob"
// The webp decoder lets image.DecodeConfig and File.Thumb read .webp
// originals. imaging cannot encode webp, so a webp thumbnail holds JPEG
// bytes under the original's .webp name (see defaultEncodeImage).
_ "golang.org/x/image/webp"
)
const (
@@ -67,7 +72,11 @@ func fileExt(diskName string) string {
return strings.ToLower(ext)
}
func publicURL(key string) string {
// PublicURL returns the public URL of a blob key: storage.uploads.
// public_path_prefix and the key joined by exactly one slash. With the
// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
// /storage/app/uploads/public) it is Winter's File::getPath() path.
func PublicURL(key string) string {
prefix := strings.TrimRight(PublicPathPrefix(), "/")
key = strings.TrimLeft(key, "/")
if prefix == "" {
@@ -76,6 +85,15 @@ func publicURL(key string) string {
return prefix + "/" + key
}
// URL returns the public URL of the original file, Winter's File::getPath():
// PublicURL of BlobKey(DiskName).
func (f *File) URL() string {
if f == nil {
return ""
}
return PublicURL(BlobKey(f.DiskName))
}
func defaultResizeImage(src image.Image, w, h int, mode string) image.Image {
switch strings.ToLower(mode) {
case "crop":
@@ -133,21 +151,35 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
return "", fmt.Errorf("attach: thumb exists: %w", err)
}
if exists {
return publicURL(thumbKey), nil
return PublicURL(thumbKey), nil
}
origKey := part + f.DiskName
r, err := bucket.NewReader(ctx, origKey, nil)
if err != nil {
return "", fmt.Errorf("attach: read original: %w", err)
}
src, _, err := image.Decode(io.LimitReader(r, maxThumbSourceBytes))
raw, err := io.ReadAll(io.LimitReader(r, maxThumbSourceBytes))
closeErr := r.Close()
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
return "", fmt.Errorf("attach: read original: %w", err)
}
if closeErr != nil {
return "", closeErr
}
// Check the dimensions from the header before decoding the pixels, so
// a small file that declares a huge image is refused without
// allocating it.
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
}
if int64(cfg.Width)*int64(cfg.Height) > maxThumbSourcePixels {
return "", fmt.Errorf("attach: original image is too large")
}
src, _, err := image.Decode(bytes.NewReader(raw))
if err != nil {
return "", fmt.Errorf("attach: decode original: %w", err)
}
bounds := src.Bounds()
if int64(bounds.Dx())*int64(bounds.Dy()) > maxThumbSourcePixels {
return "", fmt.Errorf("attach: original image is too large")
@@ -173,5 +205,5 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
}
return "", closeErr
}
return publicURL(thumbKey), nil
return PublicURL(thumbKey), nil
}

View File

@@ -0,0 +1,107 @@
package attach
import (
"bytes"
"image"
"image/jpeg"
"os"
"path/filepath"
"testing"
"git.golem15.com/golem15/summercms/modules/compass"
"gocloud.dev/blob"
)
// winterLayoutBucket opens a mem:// bucket with the WinterCMS public prefix
// and restores the framework default afterwards.
func winterLayoutBucket(t *testing.T) *blob.Bucket {
t.Helper()
dir := t.TempDir()
body := "uploads:\n bucket_url: \"mem://\"\n public_path_prefix: \"/storage/app/uploads/public\"\n"
if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
cfg, err := compass.Open(compass.Options{Dir: dir, Env: "development", Environ: []string{"SUMMER_ENV=development"}})
if err != nil {
t.Fatal(err)
}
bucket, err := OpenBucket(t.Context(), cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = bucket.Close()
setPublicPathPrefix(defaultPublicPathPrefix)
})
return bucket
}
func TestFileURLWinterLayout(t *testing.T) {
bucket := winterLayoutBucket(t)
// A WinterCMS disk name: uniqid('', true) without the dot, plus the
// extension.
f := &File{ID: 12, DiskName: "651a2b3c4d5e61234567.png"}
if got, want := f.URL(), "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
t.Fatalf("URL = %q, want %q", got, want)
}
if got, want := PublicURL("/651/a2b/3c4/x.png"), "/storage/app/uploads/public/651/a2b/3c4/x.png"; got != want {
t.Fatalf("PublicURL = %q, want %q", got, want)
}
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), &blob.WriterOptions{ContentType: "image/png"}); err != nil {
t.Fatal(err)
}
thumb, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
if err != nil {
t.Fatal(err)
}
// Winter getThumbFilename: implode('_', [thumb, id, w, h, ox, oy, mode.ext]).
if want := "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png"; thumb != want {
t.Fatalf("thumb = %q, want %q", thumb, want)
}
var nilFile *File
if nilFile.URL() != "" {
t.Fatal("nil file URL must be empty")
}
}
// webpFixture is a 16x12 lossless WebP: blue left half, red right half.
var webpFixture = []byte{
0x52, 0x49, 0x46, 0x46, 0x2a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, 0x50, 0x38, 0x4c,
0x1d, 0x00, 0x00, 0x00, 0x2f, 0x0f, 0xc0, 0x02, 0x00, 0x0f, 0x70, 0x14, 0xfb, 0x53, 0xd0, 0x5e,
0x88, 0x7b, 0xfe, 0x83, 0x07, 0x62, 0xc1, 0x64, 0xfe, 0xd2, 0xbd, 0x21, 0x44, 0xf4, 0x3f, 0x74,
0x01, 0x00,
}
func TestThumbWebP(t *testing.T) {
cfg, format, err := image.DecodeConfig(bytes.NewReader(webpFixture))
if err != nil {
t.Fatalf("DecodeConfig: %v", err)
}
if format != "webp" || cfg.Width != 16 || cfg.Height != 12 {
t.Fatalf("config = %s %dx%d", format, cfg.Width, cfg.Height)
}
bucket := winterLayoutBucket(t)
f := &File{ID: 5, DiskName: "abcdef123456789012345.webp"}
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), webpFixture, &blob.WriterOptions{ContentType: "image/webp"}); err != nil {
t.Fatal(err)
}
url, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
if err != nil {
t.Fatalf("Thumb: %v", err)
}
if want := "/storage/app/uploads/public/abc/def/123/thumb_5_200_200_0_0_crop.webp"; url != want {
t.Fatalf("thumb url = %q, want %q", url, want)
}
raw, err := bucket.ReadAll(t.Context(), "abc/def/123/thumb_5_200_200_0_0_crop.webp")
if err != nil {
t.Fatal(err)
}
// imaging cannot encode webp: the thumb is JPEG bytes under the .webp name.
img, err := jpeg.Decode(bytes.NewReader(raw))
if err != nil {
t.Fatalf("thumb is not JPEG: %v", err)
}
if b := img.Bounds(); b.Dx() != 200 || b.Dy() != 200 {
t.Fatalf("thumb size = %v", b)
}
}