From ba9c99246ec40bab5886f8b3425e143bde03f6d5 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Fri, 18 Sep 2026 19:55:15 +0200 Subject: [PATCH] feat(05-04): implement attach File, Thumb, blob bucket, system_files migration - Winter-exact thumb filename and 3x3 partition with lazy imaging resize - gocloud.dev/blob fileblob/memblob; empty bucket_url fails boot - lagoon.Migrate runs system_files before every plugin set --- go.mod | 2 + go.sum | 5 ++ lagoon/attach/bucket.go | 81 ++++++++++++++++++++-- lagoon/attach/migrations.go | 50 ++++++++++++-- lagoon/attach/thumb.go | 129 ++++++++++++++++++++++++++++++++++-- lagoon/migrations.go | 11 ++- 6 files changed, 262 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 23a763b..820cf7f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.golem15.com/golem15/summercms go 1.27.0 require ( + github.com/disintegration/imaging v1.6.2 github.com/fsnotify/fsnotify v1.10.1 github.com/go-gormigrate/gormigrate/v2 v2.1.7 github.com/go-playground/validator/v10 v10.30.4 @@ -96,6 +97,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.55.0 // indirect + golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.48.0 // indirect diff --git a/go.sum b/go.sum index f82417b..8929066 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfv github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= @@ -287,6 +289,8 @@ gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -301,6 +305,7 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/lagoon/attach/bucket.go b/lagoon/attach/bucket.go index 31b3008..2ec0a65 100644 --- a/lagoon/attach/bucket.go +++ b/lagoon/attach/bucket.go @@ -3,6 +3,9 @@ package attach import ( "context" "fmt" + "net/url" + "strings" + "sync" "git.golem15.com/golem15/summercms/backpack" "git.golem15.com/golem15/summercms/compass" @@ -11,12 +14,82 @@ import ( _ "gocloud.dev/blob/memblob" ) +const defaultPublicPathPrefix = "/storage/uploads" + +var ( + prefixMu sync.RWMutex + publicPathPrefix = defaultPublicPathPrefix +) + +func setPublicPathPrefix(prefix string) { + prefixMu.Lock() + publicPathPrefix = prefix + prefixMu.Unlock() +} + +// PublicPathPrefix is the URL prefix prepended to partition+filename. +func PublicPathPrefix() string { + prefixMu.RLock() + defer prefixMu.RUnlock() + return publicPathPrefix +} + +func normalizeBucketURL(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("attach: bucket_url: %w", err) + } + switch strings.ToLower(u.Scheme) { + case "mem", "memory": + return "mem://", nil + case "fileblob": + u.Scheme = "file" + fallthrough + case "file": + q := u.Query() + if q.Get("create_dir") == "" { + q.Set("create_dir", "true") + u.RawQuery = q.Encode() + } + return u.String(), nil + default: + return raw, nil + } +} + +// OpenBucket opens storage.uploads.bucket_url (file:// or mem://) and +// records storage.uploads.public_path_prefix. An empty bucket_url fails boot. func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error) { - return nil, fmt.Errorf("attach: bucket not implemented") + if cfg == nil { + return nil, fmt.Errorf("attach: config is nil") + } + raw := strings.TrimSpace(cfg.String("storage.uploads.bucket_url")) + if raw == "" { + return nil, fmt.Errorf("attach: storage.uploads.bucket_url is empty (set SUMMER_STORAGE__UPLOADS__BUCKET_URL)") + } + bucketURL, err := normalizeBucketURL(raw) + if err != nil { + return nil, err + } + bucket, err := blob.OpenBucket(ctx, bucketURL) + if err != nil { + return nil, fmt.Errorf("attach: open bucket %q: %w", raw, err) + } + prefix := strings.TrimSpace(cfg.String("storage.uploads.public_path_prefix")) + if prefix == "" { + prefix = defaultPublicPathPrefix + } + setPublicPathPrefix(prefix) + return bucket, nil } +// Publish stores the opened bucket once on the app, matching lagoon.Publish. func Publish(app *backpack.App, bucket *blob.Bucket) error { - return fmt.Errorf("attach: publish not implemented") + if app == nil { + return fmt.Errorf("attach: app is nil") + } + if bucket == nil { + return fmt.Errorf("attach: bucket is nil") + } + return app.Publish(bucket) } - -func PublicPathPrefix() string { return "" } diff --git a/lagoon/attach/migrations.go b/lagoon/attach/migrations.go index b794dae..8de7d3d 100644 --- a/lagoon/attach/migrations.go +++ b/lagoon/attach/migrations.go @@ -1,7 +1,49 @@ package attach -import "github.com/go-gormigrate/gormigrate/v2" +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) -// Migrations is the framework-owned system_files set. lagoon.Migrate -// runs it before every plugin set (D-14). -var Migrations = []*gormigrate.Migration{} +// Migrations creates Winter's system_files table. Folded from +// modules/system/database/migrations/2013_10_01_000002_Db_System_Files.php +// and 2025_04_10_000031_Db_Add_System_Files_Metadata.php. +var Migrations = []*gormigrate.Migration{ + { + ID: "202609180001_create_system_files", + Migrate: func(tx *gorm.DB) error { + stmts := []string{ + `CREATE TABLE system_files ( + id SERIAL PRIMARY KEY, + disk_name TEXT NOT NULL, + file_name TEXT NOT NULL, + file_size INTEGER NOT NULL, + content_type TEXT NOT NULL, + title TEXT, + description TEXT, + field TEXT, + attachment_id TEXT, + attachment_type TEXT, + is_public BOOLEAN NOT NULL DEFAULT TRUE, + sort_order INTEGER, + metadata TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +)`, + `CREATE INDEX system_files_field_index ON system_files (field)`, + `CREATE INDEX system_files_attachment_id_index ON system_files (attachment_id)`, + `CREATE INDEX system_files_attachment_type_index ON system_files (attachment_type)`, + `CREATE INDEX system_files_attachment_lookup_index ON system_files (attachment_type, attachment_id, field)`, + } + for _, stmt := range stmts { + if err := tx.Exec(stmt).Error; err != nil { + return err + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("DROP TABLE IF EXISTS system_files").Error + }, + }, +} diff --git a/lagoon/attach/thumb.go b/lagoon/attach/thumb.go index a32efb3..f034811 100644 --- a/lagoon/attach/thumb.go +++ b/lagoon/attach/thumb.go @@ -4,22 +4,137 @@ import ( "context" "fmt" "image" + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "io" + "path" + "strings" + "github.com/disintegration/imaging" "gocloud.dev/blob" ) +// ThumbFilename is Winter File::getThumbFilename: thumb______.. func ThumbFilename(id uint, w, h int, offsetX, offsetY int, mode, ext string) string { - return "" + return fmt.Sprintf("thumb_%d_%d_%d_%d_%d_%s.%s", id, w, h, offsetX, offsetY, mode, ext) } +// PartitionDirectory is Winter File::getPartitionDirectory: first 9 chars of +// disk_name split into 3 groups of 3, joined by '/', with a trailing slash. func PartitionDirectory(diskName string) string { - return "" + var groups []string + for i := 0; i < len(diskName) && len(groups) < 3; i += 3 { + end := i + 3 + if end > len(diskName) { + end = len(diskName) + } + groups = append(groups, diskName[i:end]) + } + return strings.Join(groups, "/") + "/" } +// BlobKey is the Winter on-disk key for an original file: partition + disk_name. +func BlobKey(diskName string) string { + return PartitionDirectory(diskName) + diskName +} + +func fileExt(diskName string) string { + ext := strings.TrimPrefix(path.Ext(diskName), ".") + if ext == "" { + return "jpg" + } + return strings.ToLower(ext) +} + +func publicURL(key string) string { + prefix := strings.TrimRight(PublicPathPrefix(), "/") + key = strings.TrimLeft(key, "/") + if prefix == "" { + return "/" + key + } + return prefix + "/" + key +} + +func defaultResizeImage(src image.Image, w, h int, mode string) image.Image { + switch strings.ToLower(mode) { + case "crop": + return imaging.Fill(src, w, h, imaging.Center, imaging.Lanczos) + case "exact": + return imaging.Resize(src, w, h, imaging.Lanczos) + default: + return imaging.Fit(src, w, h, imaging.Lanczos) + } +} + +var resizeImage = defaultResizeImage + +func encodeImage(w io.Writer, img image.Image, ext string) error { + format := imaging.JPEG + switch strings.ToLower(ext) { + case "png": + format = imaging.PNG + case "gif": + format = imaging.GIF + } + return imaging.Encode(w, img, format) +} + +// Thumb returns the public URL of a lazily generated thumbnail. The second +// call for the same dimensions hits the existing blob and does not resize. func (f *File) Thumb(ctx context.Context, bucket *blob.Bucket, w, h int, mode string) (string, error) { - return "", fmt.Errorf("attach: thumb not implemented") -} - -var resizeImage = func(src image.Image, w, h int, mode string) image.Image { - return src + if f == nil { + return "", fmt.Errorf("attach: file is nil") + } + if bucket == nil { + return "", fmt.Errorf("attach: bucket is nil") + } + if mode == "" { + mode = "auto" + } + ext := fileExt(f.DiskName) + thumbName := ThumbFilename(f.ID, w, h, 0, 0, mode, ext) + part := PartitionDirectory(f.DiskName) + thumbKey := part + thumbName + exists, err := bucket.Exists(ctx, thumbKey) + if err != nil { + return "", fmt.Errorf("attach: thumb exists: %w", err) + } + if exists { + return publicURL(thumbKey), nil + } + origKey := part + f.DiskName + r, err := bucket.NewReader(ctx, origKey, nil) + if err != nil { + return "", fmt.Errorf("attach: read original: %w", err) + } + src, _, err := image.Decode(r) + closeErr := r.Close() + if err != nil { + return "", fmt.Errorf("attach: decode original: %w", err) + } + if closeErr != nil { + return "", closeErr + } + resized := resizeImage(src, w, h, mode) + contentType := "image/jpeg" + switch ext { + case "png": + contentType = "image/png" + case "gif": + contentType = "image/gif" + } + wr, err := bucket.NewWriter(ctx, thumbKey, &blob.WriterOptions{ContentType: contentType}) + if err != nil { + return "", fmt.Errorf("attach: thumb writer: %w", err) + } + encErr := encodeImage(wr, resized, ext) + closeErr = wr.Close() + if encErr != nil { + return "", encErr + } + if closeErr != nil { + return "", closeErr + } + return publicURL(thumbKey), nil } diff --git a/lagoon/migrations.go b/lagoon/migrations.go index c355e72..dbd29c9 100644 --- a/lagoon/migrations.go +++ b/lagoon/migrations.go @@ -6,6 +6,7 @@ import ( "strings" "unicode" + "git.golem15.com/golem15/summercms/lagoon/attach" "git.golem15.com/golem15/summercms/pact" "git.golem15.com/golem15/summercms/party" "github.com/go-gormigrate/gormigrate/v2" @@ -56,11 +57,19 @@ func migrator(gdb *gorm.DB, pluginID string, migrations []*gormigrate.Migration) }, migrations), nil } -// Migrate runs each plugin's HasMigrations set in party.Activate order. +// Migrate runs the framework-owned system_files set first, then each +// plugin's HasMigrations set in party.Activate order. func Migrate(gdb *gorm.DB, plugins []party.Plugin) error { if gdb == nil { return fmt.Errorf("lagoon: gorm db is nil") } + m, err := migrator(gdb, "summercms.attach", attach.Migrations) + if err != nil { + return err + } + if err := m.Migrate(); err != nil { + return fmt.Errorf("lagoon: migrate system_files: %w", err) + } for _, p := range plugins { hm, ok := p.(pact.HasMigrations) if !ok {