feat(12-01): record multipart uploads and match Winter upload URLs
- attach.PublicURL and (*File).URL build Winter File::getPath() URLs; the thumbnailer decodes webp via golang.org/x/image v0.46.0 and checks the image size from the header before decoding - tide requests carry multipart parts (files beside the fixture pinned by sha256) encoded with the fixed MultipartBoundary, so PHP and Go receive byte-identical bodies - tide masks the random partition, disk name and file id of url/thumb_url upload URLs while still diffing prefix, size, mode and extension, and NormalizePublications masks Carbon dates in the published album
This commit is contained in:
@@ -24,7 +24,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he
|
||||
- Column types: `lagoon.Encrypted` stores AES-256-GCM ciphertext under a key derived from `app.key`, decrypts with previous keys during rotation, and always redacts itself in JSON and string output; `lagoon.Jsonable` stores JSON as TEXT and keeps SQL NULL distinct from an empty value.
|
||||
- Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns.
|
||||
- Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports.
|
||||
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), on-demand thumbnails through `attach.File.Thumb`, static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
|
||||
- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`).
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -151,7 +151,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration {
|
||||
| `lagoon.WithSoftDeleteCascade` | Runs a cascade inside the parent delete transaction. |
|
||||
| `lagoon.RegisterJoinTable` | Registers a custom pivot model for a many-to-many field. |
|
||||
| `lagoon.DecryptLaravelPayload` | Decrypts a Laravel AES-256-CBC payload for data imports. |
|
||||
| `attach.File` | The `system_files` row model. |
|
||||
| `attach.File` | The `system_files` row model; `attach.File.URL` is the public URL of the original. |
|
||||
| `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. |
|
||||
| `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. |
|
||||
| `attach.OpenBucket` | Opens the uploads bucket from config. |
|
||||
| `attach.Publish` | Stores the bucket on the `backpack.App`. |
|
||||
@@ -198,7 +199,7 @@ storage:
|
||||
|
||||
- SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages).
|
||||
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`).
|
||||
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails).
|
||||
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding).
|
||||
- Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others.
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
@@ -14,6 +15,10 @@ import (
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"gocloud.dev/blob"
|
||||
// The webp decoder lets image.DecodeConfig and File.Thumb read .webp
|
||||
// originals. imaging cannot encode webp, so a webp thumbnail holds JPEG
|
||||
// bytes under the original's .webp name (see defaultEncodeImage).
|
||||
_ "golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -67,7 +72,11 @@ func fileExt(diskName string) string {
|
||||
return strings.ToLower(ext)
|
||||
}
|
||||
|
||||
func publicURL(key string) string {
|
||||
// PublicURL returns the public URL of a blob key: storage.uploads.
|
||||
// public_path_prefix and the key joined by exactly one slash. With the
|
||||
// WinterCMS layout (bucket rooted at storage/app/uploads/public, prefix
|
||||
// /storage/app/uploads/public) it is Winter's File::getPath() path.
|
||||
func PublicURL(key string) string {
|
||||
prefix := strings.TrimRight(PublicPathPrefix(), "/")
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if prefix == "" {
|
||||
@@ -76,6 +85,15 @@ func publicURL(key string) string {
|
||||
return prefix + "/" + key
|
||||
}
|
||||
|
||||
// URL returns the public URL of the original file, Winter's File::getPath():
|
||||
// PublicURL of BlobKey(DiskName).
|
||||
func (f *File) URL() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
return PublicURL(BlobKey(f.DiskName))
|
||||
}
|
||||
|
||||
func defaultResizeImage(src image.Image, w, h int, mode string) image.Image {
|
||||
switch strings.ToLower(mode) {
|
||||
case "crop":
|
||||
@@ -133,21 +151,35 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
|
||||
return "", fmt.Errorf("attach: thumb exists: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return publicURL(thumbKey), nil
|
||||
return PublicURL(thumbKey), nil
|
||||
}
|
||||
origKey := part + f.DiskName
|
||||
r, err := bucket.NewReader(ctx, origKey, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: read original: %w", err)
|
||||
}
|
||||
src, _, err := image.Decode(io.LimitReader(r, maxThumbSourceBytes))
|
||||
raw, err := io.ReadAll(io.LimitReader(r, maxThumbSourceBytes))
|
||||
closeErr := r.Close()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
return "", fmt.Errorf("attach: read original: %w", err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", closeErr
|
||||
}
|
||||
// Check the dimensions from the header before decoding the pixels, so
|
||||
// a small file that declares a huge image is refused without
|
||||
// allocating it.
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > maxThumbSourcePixels {
|
||||
return "", fmt.Errorf("attach: original image is too large")
|
||||
}
|
||||
src, _, err := image.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("attach: decode original: %w", err)
|
||||
}
|
||||
bounds := src.Bounds()
|
||||
if int64(bounds.Dx())*int64(bounds.Dy()) > maxThumbSourcePixels {
|
||||
return "", fmt.Errorf("attach: original image is too large")
|
||||
@@ -173,5 +205,5 @@ func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode st
|
||||
}
|
||||
return "", closeErr
|
||||
}
|
||||
return publicURL(thumbKey), nil
|
||||
return PublicURL(thumbKey), nil
|
||||
}
|
||||
|
||||
107
modules/lagoon/attach/url_test.go
Normal file
107
modules/lagoon/attach/url_test.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package attach
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
"gocloud.dev/blob"
|
||||
)
|
||||
|
||||
// winterLayoutBucket opens a mem:// bucket with the WinterCMS public prefix
|
||||
// and restores the framework default afterwards.
|
||||
func winterLayoutBucket(t *testing.T) *blob.Bucket {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
body := "uploads:\n bucket_url: \"mem://\"\n public_path_prefix: \"/storage/app/uploads/public\"\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := compass.Open(compass.Options{Dir: dir, Env: "development", Environ: []string{"SUMMER_ENV=development"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket, err := OpenBucket(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = bucket.Close()
|
||||
setPublicPathPrefix(defaultPublicPathPrefix)
|
||||
})
|
||||
return bucket
|
||||
}
|
||||
|
||||
func TestFileURLWinterLayout(t *testing.T) {
|
||||
bucket := winterLayoutBucket(t)
|
||||
// A WinterCMS disk name: uniqid('', true) without the dot, plus the
|
||||
// extension.
|
||||
f := &File{ID: 12, DiskName: "651a2b3c4d5e61234567.png"}
|
||||
if got, want := f.URL(), "/storage/app/uploads/public/651/a2b/3c4/651a2b3c4d5e61234567.png"; got != want {
|
||||
t.Fatalf("URL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := PublicURL("/651/a2b/3c4/x.png"), "/storage/app/uploads/public/651/a2b/3c4/x.png"; got != want {
|
||||
t.Fatalf("PublicURL = %q, want %q", got, want)
|
||||
}
|
||||
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), testJPEG(t), &blob.WriterOptions{ContentType: "image/png"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
thumb, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Winter getThumbFilename: implode('_', [thumb, id, w, h, ox, oy, mode.ext]).
|
||||
if want := "/storage/app/uploads/public/651/a2b/3c4/thumb_12_200_200_0_0_crop.png"; thumb != want {
|
||||
t.Fatalf("thumb = %q, want %q", thumb, want)
|
||||
}
|
||||
var nilFile *File
|
||||
if nilFile.URL() != "" {
|
||||
t.Fatal("nil file URL must be empty")
|
||||
}
|
||||
}
|
||||
|
||||
// webpFixture is a 16x12 lossless WebP: blue left half, red right half.
|
||||
var webpFixture = []byte{
|
||||
0x52, 0x49, 0x46, 0x46, 0x2a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, 0x50, 0x38, 0x4c,
|
||||
0x1d, 0x00, 0x00, 0x00, 0x2f, 0x0f, 0xc0, 0x02, 0x00, 0x0f, 0x70, 0x14, 0xfb, 0x53, 0xd0, 0x5e,
|
||||
0x88, 0x7b, 0xfe, 0x83, 0x07, 0x62, 0xc1, 0x64, 0xfe, 0xd2, 0xbd, 0x21, 0x44, 0xf4, 0x3f, 0x74,
|
||||
0x01, 0x00,
|
||||
}
|
||||
|
||||
func TestThumbWebP(t *testing.T) {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(webpFixture))
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeConfig: %v", err)
|
||||
}
|
||||
if format != "webp" || cfg.Width != 16 || cfg.Height != 12 {
|
||||
t.Fatalf("config = %s %dx%d", format, cfg.Width, cfg.Height)
|
||||
}
|
||||
bucket := winterLayoutBucket(t)
|
||||
f := &File{ID: 5, DiskName: "abcdef123456789012345.webp"}
|
||||
if err := bucket.WriteAll(t.Context(), BlobKey(f.DiskName), webpFixture, &blob.WriterOptions{ContentType: "image/webp"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url, err := f.Thumb(t.Context(), bucket, 200, 200, "crop")
|
||||
if err != nil {
|
||||
t.Fatalf("Thumb: %v", err)
|
||||
}
|
||||
if want := "/storage/app/uploads/public/abc/def/123/thumb_5_200_200_0_0_crop.webp"; url != want {
|
||||
t.Fatalf("thumb url = %q, want %q", url, want)
|
||||
}
|
||||
raw, err := bucket.ReadAll(t.Context(), "abc/def/123/thumb_5_200_200_0_0_crop.webp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// imaging cannot encode webp: the thumb is JPEG bytes under the .webp name.
|
||||
img, err := jpeg.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("thumb is not JPEG: %v", err)
|
||||
}
|
||||
if b := img.Bounds(); b.Dx() != 200 || b.Dy() != 200 {
|
||||
t.Fatalf("thumb size = %v", b)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user