From 2382040b254e0f2b8b394080c8de35c30a3e7d76 Mon Sep 17 00:00:00 2001 From: Jakub Zych Date: Thu, 15 Oct 2026 18:07:05 +0200 Subject: [PATCH] feat(19.3-03): summer.yaml build.tags and the summercms_s3 tag that links the S3 uploads driver - Manifest.Tags from build.tags (^[a-z0-9_]+$, duplicates dropped), round-tripped by SaveManifest - summer build passes -tags= through goBuildArgs and prints the build tags line - attach: s3blob imported only by s3_on.go (//go:build summercms_s3); an untagged binary refuses an s3:// bucket_url naming the tag, build.tags and -tags - go mod tidy adds the AWS SDK v2 modules as indirect requirements - storage, modules, scaffolding, configuration and scaling docs plus the lagoon README --- .../architecture/go-modules-and-workspaces.md | 7 +- docs/architecture/performance-and-scaling.md | 2 +- docs/console/scaffolding.md | 6 +- docs/services/storage.md | 42 +++++++++- docs/setup/configuration.md | 2 +- go.mod | 20 +++++ go.sum | 6 ++ internal/build/build.go | 16 +++- internal/build/manifest.go | 78 ++++++++++++++++++- internal/build/manifest_tags_smoke_test.go | 70 +++++++++++++++++ modules/lagoon/README.md | 11 +-- modules/lagoon/attach/bucket.go | 28 ++++++- modules/lagoon/attach/infra_smoke_test.go | 60 ++++++++++++++ modules/lagoon/attach/s3_off.go | 7 ++ modules/lagoon/attach/s3_on.go | 9 +++ 15 files changed, 346 insertions(+), 18 deletions(-) create mode 100644 internal/build/manifest_tags_smoke_test.go create mode 100644 modules/lagoon/attach/infra_smoke_test.go create mode 100644 modules/lagoon/attach/s3_off.go create mode 100644 modules/lagoon/attach/s3_on.go diff --git a/docs/architecture/go-modules-and-workspaces.md b/docs/architecture/go-modules-and-workspaces.md index bf57150..1e1d526 100644 --- a/docs/architecture/go-modules-and-workspaces.md +++ b/docs/architecture/go-modules-and-workspaces.md @@ -69,7 +69,7 @@ There is no `go.sum` in the application root: builds go through the workspace, a ## The summer.yaml manifest -The manifest in the application root names the application module, the binary `summer build` writes to `bin/`, and the plugins in activation order: +The manifest in the application root names the application module, the binary `summer build` writes to `bin/`, the plugins in activation order and, optionally, Go build tags: ```yaml module: git.golem15.com/acme/acme @@ -79,10 +79,15 @@ plugins: module: git.golem15.com/acme/acme/plugins/user - id: acme.blog module: git.golem15.com/acme/acme/plugins/blog +build: + tags: + - summercms_s3 ``` `summer build` turns this list into `plugins.gen.go`. The order is the manifest order, adjusted so that each plugin comes after the plugins it requires. Plugins with no dependency between them keep the order you wrote. +`build.tags` lists Go build tags that `summer build` (and `summer dev`) passes to `go build` as `-tags=,`, printing them as `build tags: …`. Each tag must consist of lower-case letters, digits and underscores; a duplicate is ignored. Framework features that pull in large dependencies sit behind a tag, such as `summercms_s3` for S3 uploads buckets (see [Storage](../services/storage.md#s3-and-s3-compatible-buckets)). `summer plugin:add` keeps the list when it rewrites the manifest. A script that runs its own `go build` must pass the same tags. + ## Local plugins in a workspace A plugin you develop inside the application lives in `plugins/` as its own module. `summer make:plugin acme.blog` creates it there, and `summer plugin:add plugins/blog` registers it: diff --git a/docs/architecture/performance-and-scaling.md b/docs/architecture/performance-and-scaling.md index ccce853..1cc0225 100644 --- a/docs/architecture/performance-and-scaling.md +++ b/docs/architecture/performance-and-scaling.md @@ -68,7 +68,7 @@ Some costs come over with the port unchanged, and a few framework behaviours nee The application is one stateless binary. It starts in milliseconds, ships in a small container image and needs no OPcache warm-up after a deploy. - Vertical scaling is more cores. The Go runtime uses them all, so there is no worker count to tune. -- Horizontal scaling is N replicas behind a load balancer that share Postgres. Uploads must be shared too: point `storage.uploads.bucket_url` at a `file://` directory on storage that every replica mounts (see [Storage](../services/storage.md#bucket-urls)). Apply the blacklist and rate-limit notes above before you add the second replica. +- Horizontal scaling is N replicas behind a load balancer that share Postgres. Uploads must be shared too: point `storage.uploads.bucket_url` at a `file://` directory on storage that every replica mounts, or at an S3 or S3-compatible bucket (`s3://`, in a binary built with the `summercms_s3` tag; see [Storage](../services/storage.md#s3-and-s3-compatible-buckets)). Apply the blacklist and rate-limit notes above before you add the second replica. - Centrifugo scales on its own. [lighthouse](../../modules/lighthouse/README.md) only publishes to it and issues connection tokens, so adding application replicas does not change the realtime server; see [The Centrifugo driver](../services/realtime.md#the-centrifugo-driver). A rollout runs the migrations once, then starts each replica behind the reverse proxy: diff --git a/docs/console/scaffolding.md b/docs/console/scaffolding.md index 7f540dd..e4dee7a 100644 --- a/docs/console/scaffolding.md +++ b/docs/console/scaffolding.md @@ -12,8 +12,8 @@ The `summer` tool creates and builds applications and generates the files a plug | Command | Purpose | |---------|---------| -| `summer build` | Reads `summer.yaml`, generates `plugins.gen.go` and `main.go`, and builds the binary into `bin/`. Run it from the application directory or any directory below it. | -| `summer dev` | Builds the application, starts the binary, and rebuilds and restarts it whenever a Go or YAML source, `go.mod`, `go.work`, `.env` or `summer.yaml` changes. | +| `summer build` | Reads `summer.yaml`, generates `plugins.gen.go` and `main.go`, and builds the binary into `bin/`, passing the manifest's `build.tags` to `go build` as `-tags`. Run it from the application directory or any directory below it. | +| `summer dev` | Builds the application as `summer build` does (build tags included), starts the binary, and rebuilds and restarts it whenever a Go or YAML source, `go.mod`, `go.work`, `.env` or `summer.yaml` changes. | ```sh summer build @@ -44,7 +44,7 @@ The application gets these files: |------|----------| | `go.mod` | Module `git.golem15.com/golem15/sm--app`, requiring the framework through a relative `replace`. | | `go.work` | The workspace with the application and its plugin modules, as relative paths. It is committed. | -| `summer.yaml` | The manifest: module path, binary name and plugin list. | +| `summer.yaml` | The manifest: module path, binary name, plugin list and optional `build.tags` (for example `summercms_s3` for an S3 uploads bucket). | | `config/` | `app.yaml`, `database.yaml`, `http.yaml` and `storage.yaml`, enough for `serve` and `route:list`. | | `.env`, `.env.example` | The database DSN, a freshly generated application key and a random admin token secret (`SUMMER_ADMIN__JWT__SECRET`). `.env` is readable by its owner only and never committed; `.env.example` lists the variables without the key and the secret. | | `compose.yaml` | PostgreSQL 16 on `127.0.0.1:54320`, with credentials matching `.env`. | diff --git a/docs/services/storage.md b/docs/services/storage.md index d6d1686..8c8866b 100644 --- a/docs/services/storage.md +++ b/docs/services/storage.md @@ -1,6 +1,6 @@ --- title: Storage -description: Configure the uploads bucket that serve opens, choose file or memory bucket URLs, serve stored files, and size upload routes. +description: Configure the uploads bucket that serve opens, choose file, memory or S3 bucket URLs, serve stored files, and size upload routes. section: services order: 90 --- @@ -14,6 +14,7 @@ WinterCMS stores uploads on a Laravel filesystem disk. SummerCMS stores them in |-----|--------| | `file:///var/lib/acme/uploads` | In a directory on the server. | | `mem://` | In memory, for tests. Everything is lost when the process exits. | +| `s3://acme-uploads?region=eu-central-1` | In an S3 or S3-compatible bucket. The binary must be built with the `summercms_s3` tag; see [S3 and S3-compatible buckets](#s3-and-s3-compatible-buckets). | The keys go in `config/storage.yaml`: @@ -39,6 +40,45 @@ An original then has the URL `/storage/app/uploads/public///{path...}` on `attach.StaticHandlerPublic`: diff --git a/docs/setup/configuration.md b/docs/setup/configuration.md index caed203..d2beec0 100644 --- a/docs/setup/configuration.md +++ b/docs/setup/configuration.md @@ -57,7 +57,7 @@ These are the keys most applications set. Each module's reference page lists all | `cache.driver`, `cache.prefix`, `cache.memory.max_entries` | The cache driver (`memory`, the default, or `redis`), the key prefix (default from `app.name`) and the memory driver's size bound (default 10000). | [cooler](../../modules/cooler/README.md) | | `redis.url` | Redis URL (`redis://`, `rediss://` or `unix://`), required when a Redis cache or rate-limit store is selected. Keep it in `SUMMER_REDIS__URL` when it carries a password. | [cooler](../../modules/cooler/README.md) | | `http.access_log` | Logs one record per request with method, matched route pattern, status and duration. Defaults to `false`. | [surf](../../modules/surf/README.md) | -| `storage.uploads.bucket_url` | Uploads bucket, `file://` or `mem://`. Required by `serve`. | [lagoon](../../modules/lagoon/README.md) | +| `storage.uploads.bucket_url` | Uploads bucket, `file://`, `mem://` or `s3://` (the last only in a binary built with the `summercms_s3` tag from `build.tags` in `summer.yaml`; credentials from the AWS environment). Required by `serve`. | [lagoon](../../modules/lagoon/README.md) | | `storage.uploads.serve` | Mounts `attach.StaticHandlerPublic` under `storage.uploads.public_path_prefix` with `X-Content-Type-Options: nosniff`, a sandboxing CSP and `Content-Disposition: attachment` for anything but a raster image. Defaults to `false`. | [surf](../../modules/surf/README.md) | | `admin.jwt.secret` | Secret for admin tokens. Required as soon as a plugin registers an admin controller. | [cabana](../../modules/cabana/README.md) | | `backend.uri` | Path the admin is served under. Defaults to `/backend`. | [cabana](../../modules/cabana/README.md) | diff --git a/go.mod b/go.mod index c8e754f..104f3dd 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,25 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.9 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.20 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.19 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect + github.com/aws/smithy-go v1.26.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -67,6 +86,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/google/wire v0.7.0 // indirect github.com/googleapis/gax-go/v2 v2.19.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 23e5432..5276e93 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,12 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= +github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI= +github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/internal/build/build.go b/internal/build/build.go index ac2a1f3..145702f 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -46,7 +46,10 @@ func App(ctx context.Context, appDir string, out io.Writer) error { return fmt.Errorf("build: create bin dir: %w", err) } - cmd := exec.CommandContext(ctx, "go", "build", "-o", binRel, ".") + if len(m.Tags) > 0 { + fmt.Fprintf(out, "build tags: %s\n", strings.Join(m.Tags, ",")) + } + cmd := exec.CommandContext(ctx, "go", goBuildArgs(binRel, m.Tags)...) cmd.Dir = appDir cmd.Stdout = out cmd.Stderr = out @@ -59,6 +62,17 @@ func App(ctx context.Context, appDir string, out io.Writer) error { return nil } +// goBuildArgs returns the go command arguments for summer build: build, the +// -tags flag when the manifest lists build tags, then -o binRel and the +// application package. The tags travel as one exec argument, never a shell. +func goBuildArgs(binRel string, tags []string) []string { + args := []string{"build"} + if len(tags) > 0 { + args = append(args, "-tags="+strings.Join(tags, ",")) + } + return append(args, "-o", binRel, ".") +} + func generate(appDir string, m Manifest) error { mainSrc, err := generateMain(m) if err != nil { diff --git a/internal/build/manifest.go b/internal/build/manifest.go index 9c6891a..8a25053 100644 --- a/internal/build/manifest.go +++ b/internal/build/manifest.go @@ -13,13 +13,18 @@ import ( const manifestName = "summer.yaml" -var pluginIDRe = regexp.MustCompile(`^[a-z][a-z0-9]*\.[a-z][a-z0-9]*$`) +var ( + pluginIDRe = regexp.MustCompile(`^[a-z][a-z0-9]*\.[a-z][a-z0-9]*$`) + buildTagRe = regexp.MustCompile(`^[a-z0-9_]+$`) +) -// Manifest is the ordered app plugin list used to generate imports. +// Manifest is the ordered app plugin list used to generate imports, plus +// the Go build tags summer build passes to go build (summer.yaml build.tags). type Manifest struct { Module string Binary string Plugins []Plugin + Tags []string } // Plugin is one compiled plugin module in manifest order. @@ -76,10 +81,67 @@ func LoadManifest(path string) (Manifest, error) { seenMod[p.Module] = struct{}{} m.Plugins = append(m.Plugins, p) } + + tags, err := loadBuildTags(tree["build"]) + if err != nil { + return Manifest{}, err + } + m.Tags = tags return m, nil } -// SaveManifest writes summer.yaml with plugins in slice order. +// loadBuildTags reads the optional build mapping and its tags list. Entries +// are trimmed and validated; duplicates are dropped keeping the first. +func loadBuildTags(raw any) ([]string, error) { + if raw == nil { + return nil, nil + } + section := asMap(raw) + if section == nil { + return nil, fmt.Errorf("build: build must be a mapping (build.tags)") + } + rawTags, ok := section["tags"] + if !ok || rawTags == nil { + return nil, nil + } + list, ok := rawTags.([]any) + if !ok { + return nil, fmt.Errorf("build: build.tags must be a list of tags") + } + var tags []string + seen := make(map[string]struct{}, len(list)) + for i, item := range list { + s, ok := item.(string) + if !ok { + return nil, fmt.Errorf("build: build.tags[%d] must be a string", i) + } + tag := strings.TrimSpace(s) + if err := ValidateBuildTag(tag); err != nil { + return nil, fmt.Errorf("build: build.tags[%d]: %w", i, err) + } + if _, dup := seen[tag]; dup { + continue + } + seen[tag] = struct{}{} + tags = append(tags, tag) + } + return tags, nil +} + +// ValidateBuildTag requires a Go build tag of lowercase letters, digits and +// underscores, so a summer.yaml value can never smuggle a flag into go build. +func ValidateBuildTag(tag string) error { + if tag == "" { + return fmt.Errorf("build tag is empty") + } + if !buildTagRe.MatchString(tag) { + return fmt.Errorf("build tag %q must match ^[a-z0-9_]+$", tag) + } + return nil +} + +// SaveManifest writes summer.yaml with plugins in slice order and, when +// present, the build.tags list. func SaveManifest(path string, m Manifest) error { if err := validateModulePath(m.Module); err != nil { return fmt.Errorf("build: module: %w", err) @@ -101,6 +163,16 @@ func SaveManifest(path string, m Manifest) error { fmt.Fprintf(&b, " - id: %s\n", p.ID) fmt.Fprintf(&b, " module: %s\n", p.Module) } + if len(m.Tags) > 0 { + b.WriteString("build:\n") + b.WriteString(" tags:\n") + for i, tag := range m.Tags { + if err := ValidateBuildTag(tag); err != nil { + return fmt.Errorf("build: build.tags[%d]: %w", i, err) + } + fmt.Fprintf(&b, " - %s\n", tag) + } + } if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { return fmt.Errorf("build: write %s: %w", path, err) } diff --git a/internal/build/manifest_tags_smoke_test.go b/internal/build/manifest_tags_smoke_test.go new file mode 100644 index 0000000..d8c0a03 --- /dev/null +++ b/internal/build/manifest_tags_smoke_test.go @@ -0,0 +1,70 @@ +package build + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestManifestBuildTagsSmoke(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, manifestName) + body := "module: git.golem15.com/acme/acme\n" + + "binary: acme\n" + + "plugins:\n" + + " - id: acme.blog\n" + + " module: git.golem15.com/acme/acme/plugins/blog\n" + + "build:\n" + + " tags:\n" + + " - summercms_s3\n" + + " - \" summercms_s3 \"\n" + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + m, err := LoadManifest(path) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(m.Tags, []string{"summercms_s3"}) { + t.Fatalf("Tags = %q, want [summercms_s3]", m.Tags) + } + + if err := SaveManifest(path, m); err != nil { + t.Fatal(err) + } + again, err := LoadManifest(path) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(again.Tags, m.Tags) || len(again.Plugins) != 1 { + t.Fatalf("round trip = %+v, want tags %q and one plugin", again, m.Tags) + } + + args := goBuildArgs("bin/acme", again.Tags) + tagAt := slices.Index(args, "-tags=summercms_s3") + outAt := slices.Index(args, "-o") + if tagAt < 0 || outAt < 0 || tagAt > outAt { + t.Fatalf("goBuildArgs = %q, want -tags=summercms_s3 before -o", args) + } + if plain := goBuildArgs("bin/acme", nil); !slices.Equal(plain, []string{"build", "-o", "bin/acme", "."}) { + t.Fatalf("goBuildArgs without tags = %q", plain) + } + + for _, bad := range []string{"summercms-s3", "summercms s3"} { + if err := ValidateBuildTag(bad); err == nil { + t.Fatalf("ValidateBuildTag(%q) accepted", bad) + } + raw := "module: git.golem15.com/acme/acme\nbinary: acme\nbuild:\n tags:\n - \"" + bad + "\"\n" + if err := os.WriteFile(path, []byte(raw), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadManifest(path); err == nil || !strings.Contains(err.Error(), "build.tags") { + t.Fatalf("LoadManifest with tag %q: err = %v, want a build.tags error", bad, err) + } + } + if err := SaveManifest(path, Manifest{Module: "git.golem15.com/acme/acme", Binary: "acme", Tags: []string{"a b"}}); err == nil { + t.Fatal("SaveManifest accepted an invalid tag") + } +} diff --git a/modules/lagoon/README.md b/modules/lagoon/README.md index 508e501..b78c215 100644 --- a/modules/lagoon/README.md +++ b/modules/lagoon/README.md @@ -10,7 +10,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he ## Overview -`lagoon` is the WinterCMS models and migrations counterpart: it opens the one `*sql.DB` pool (pgx stdlib driver) that GORM and the rest of the application share, runs each plugin's gormigrate set in its own history table, and ports the Eloquent model conventions that WinterCMS plugins rely on, such as `$fillable`, `$hidden`, `$jsonable`, encrypted casts and Laravel-style validation rules. Its `attach` subpackage ports WinterCMS's `system_files` attachments, storing originals and lazily generated thumbnails in a gocloud.dev blob bucket. The application binary gets the migrate and key commands from `lagoon.RuntimeCommands`. +`lagoon` is the WinterCMS models and migrations counterpart: it opens the one `*sql.DB` pool (pgx stdlib driver) that GORM and the rest of the application share, runs each plugin's gormigrate set in its own history table, and ports the Eloquent model conventions that WinterCMS plugins rely on, such as `$fillable`, `$hidden`, `$jsonable`, encrypted casts and Laravel-style validation rules. Its `attach` subpackage ports WinterCMS's `system_files` attachments, storing originals and lazily generated thumbnails in a gocloud.dev blob bucket (a directory, memory, or S3 in binaries built with the `summercms_s3` tag). The application binary gets the migrate and key commands from `lagoon.RuntimeCommands`. ## Role in the framework @@ -38,7 +38,7 @@ Use cases: - Lifecycle and relations: hook interfaces matching GORM's native method names (`lagoon.HasBeforeCreate`, `lagoon.HasBeforeSave`, `lagoon.HasBeforeDelete`, `lagoon.HasAfterDelete`) plus `lagoon.HasBeforeValidate`; `lagoon.WithSoftDeleteCascade` runs a cascade inside the parent delete; `lagoon.RegisterJoinTable` wires pivot models with business columns. - Deferred binding: WinterCMS's `deferred_bindings` table holds the uploads and related-record changes of a form whose record is not saved yet. Every operation takes a `lagoon.DeferredKey` (the form's session key, the owning backend admin's id and the master record's morph type from `lagoon.MorphType`) and never reads or changes another admin's rows, since each row stores `backend_user_id`. `lagoon.DeferredBind` and `lagoon.DeferredUnbind` port WinterCMS's duplicate and cancel rules: a repeated bind writes nothing, and an unbind of a slave with a pending bind deletes that bind and returns it so the caller can remove what it created. `lagoon.DeferredBindings` reads and locks a session's bindings for the save that commits them, `lagoon.DeferredForget` deletes them once applied, and `lagoon.DeferredSlaves` is the subquery a list uses to include pending rows. A child created under deferral carries the `lagoon.DeferredEnvelope` (`{"created":true,"pivot":{...},"upload_id":"..."}`) in `pivot_data`. `lagoon.PurgeDeferred` removes expired bindings: it deletes an unattached `system_files` row a bind points at, and its blobs only after the commit, deletes a child only when its binding carries the created envelope, keeps records that were only linked, and locks each batch with `FOR UPDATE SKIP LOCKED`. - Imports from Laravel: `lagoon.DecryptLaravelPayload` decrypts Laravel `encrypted` payloads with the old application key, for one-off data imports. -- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded; a missing, undecodable or oversized original gets WinterCMS's broken-image picture, `attach.BrokenImagePNG`, as its thumbnail, as `File::makeThumb` does), storing uploads through `attach.Store` (a server-generated disk name, an extension allow-list with `attach.DefaultImageExtensions` and `attach.DefaultFileExtensions` as defaults, a MIME filter, a size limit enforced while streaming and, in image mode, the `attach.IsAllowedImage` content guard), attachment relation declarations (`attach.Relation`, `attach.HasRelations`), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`). +- Attachments (`attach`): the `attach.File` model for `system_files` rows, WinterCMS-compatible partitioned storage keys (`attach.BlobKey`, `attach.PartitionDirectory`), an S3 bucket option behind the `attach.S3BuildTag` build tag (an untagged binary refuses an `s3://` bucket URL at boot with an error naming the tag), public URLs (`attach.PublicURL` for any key, `attach.File.URL` for an original, matching WinterCMS's `File::getPath()` under the WinterCMS layout), on-demand thumbnails through `attach.File.Thumb` for JPEG, PNG, GIF and WebP originals (a WebP original's thumbnail is JPEG bytes under its `.webp` name, since WebP cannot be encoded; a missing, undecodable or oversized original gets WinterCMS's broken-image picture, `attach.BrokenImagePNG`, as its thumbnail, as `File::makeThumb` does), storing uploads through `attach.Store` (a server-generated disk name, an extension allow-list with `attach.DefaultImageExtensions` and `attach.DefaultFileExtensions` as defaults, a MIME filter, a size limit enforced while streaming and, in image mode, the `attach.IsAllowedImage` content guard), attachment relation declarations (`attach.Relation`, `attach.HasRelations`), static serving with an optional `is_public` gate (`attach.StaticHandlerPublic`), and a two-phase delete that removes blobs only after the database transaction commits (`attach.DeleteForOwner`, `attach.DeleteKeys`). ## Usage @@ -194,7 +194,8 @@ func (p *Plugin) Migrations() []*gormigrate.Migration { | `attach.PublicURL` | Public URL of a blob key under `storage.uploads.public_path_prefix`. | | `attach.BrokenImagePNG` | WinterCMS's broken-image picture, stored as the thumbnail of an unusable original. | | `attach.Owner` | Implemented by models that own attachments; returns the stored morph type name. | -| `attach.OpenBucket` | Opens the uploads bucket from config. | +| `attach.OpenBucket` | Opens the uploads bucket from config; refuses `s3://` in a binary built without `attach.S3BuildTag`. | +| `attach.S3BuildTag` | The build tag, `summercms_s3`, that links the S3 driver; list it under `build.tags` in `summer.yaml`. | | `attach.Publish` | Stores the bucket on the `backpack.App`. | | `attach.StaticHandler` | Serves stored files and thumbnails under a URL prefix. | | `attach.StaticHandlerPublic` | `attach.StaticHandler` plus a 404 for rows that are not public. | @@ -227,7 +228,7 @@ Keys are read from the compass config; the `SUMMER_` environment overlay maps a | `database.dsn` | none (required) | Postgres connection string used by `lagoon.OpenFromApp` and every database command. | | `app.key` | none (required) | Base64 encoding of 32 random bytes; the source of the `lagoon.Encrypted` column key. Generate one with `key:generate`. | | `app.previous_keys` | empty | List of earlier base64 keys still accepted when decrypting, for key rotation. | -| `storage.uploads.bucket_url` | none (required by `attach.OpenBucket`) | Uploads bucket URL, `file://` or `mem://`. | +| `storage.uploads.bucket_url` | none (required by `attach.OpenBucket`) | Uploads bucket URL, `file://`, `mem://` or `s3://` (with `attach.S3BuildTag`; query parameters such as `region`, `endpoint` and `use_path_style` per gocloud.dev, credentials from the AWS environment). | | `storage.uploads.public_path_prefix` | `/storage/uploads` | URL prefix used when building public file and thumbnail URLs. | | `database.deferred_bindings.purge_days` | `5` | Age in days after which `deferred:purge` removes a deferred binding, WinterCMS's `cleanUp(5)`; the `--days` flag overrides it. | | `database.deferred_bindings.purge_at` | `03:00` | Daily `HH:MM` (in `app.timezone`) at which the scheduler runs `deferred:purge`; an empty value disables the framework entry. | @@ -258,7 +259,7 @@ storage: - SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages). - Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`). -- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding). +- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers; the S3 driver `gocloud.dev/blob/s3blob` and the AWS SDK v2 only under the `summercms_s3` build tag), `github.com/disintegration/imaging` (thumbnails), `golang.org/x/image/webp` (WebP decoding). - Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others. ## Testing diff --git a/modules/lagoon/attach/bucket.go b/modules/lagoon/attach/bucket.go index 412ed04..8476aa2 100644 --- a/modules/lagoon/attach/bucket.go +++ b/modules/lagoon/attach/bucket.go @@ -16,6 +16,11 @@ import ( const defaultPublicPathPrefix = "/storage/uploads" +// S3BuildTag is the Go build tag that links the gocloud S3 driver +// (s3blob, imported only by s3_on.go). List it under build.tags in summer.yaml so +// summer build passes it to go build; a raw go build needs -tags summercms_s3. +const S3BuildTag = "summercms_s3" + var ( prefixMu sync.RWMutex publicPathPrefix = defaultPublicPathPrefix @@ -57,8 +62,24 @@ func normalizeBucketURL(raw string) (string, error) { } } -// OpenBucket opens storage.uploads.bucket_url (file:// or mem://) and -// records storage.uploads.public_path_prefix. An empty bucket_url fails boot. +// checkS3Driver refuses an s3:// bucket URL when the binary was built +// without S3BuildTag, naming the tag and where to set it. +func checkS3Driver(bucketURL string) error { + if s3Enabled { + return nil + } + scheme, _, ok := strings.Cut(bucketURL, "://") + if !ok || !strings.EqualFold(scheme, "s3") { + return nil + } + return fmt.Errorf("attach: storage.uploads.bucket_url uses s3:// but this binary was built without the %s build tag; "+ + "list %s under build.tags in summer.yaml and rebuild with summer build (a raw go build needs -tags %s, or GOFLAGS=-tags=%s)", + S3BuildTag, S3BuildTag, S3BuildTag, S3BuildTag) +} + +// OpenBucket opens storage.uploads.bucket_url (file://, mem://, or s3:// in a +// binary built with S3BuildTag) and records +// storage.uploads.public_path_prefix. An empty bucket_url fails boot. func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error) { if cfg == nil { return nil, fmt.Errorf("attach: config is nil") @@ -71,6 +92,9 @@ func OpenBucket(ctx context.Context, cfg *compass.Config) (*blob.Bucket, error) if err != nil { return nil, err } + if err := checkS3Driver(bucketURL); err != nil { + return nil, err + } bucket, err := blob.OpenBucket(ctx, bucketURL) if err != nil { return nil, fmt.Errorf("attach: open bucket %q: %w", raw, err) diff --git a/modules/lagoon/attach/infra_smoke_test.go b/modules/lagoon/attach/infra_smoke_test.go new file mode 100644 index 0000000..1f3919c --- /dev/null +++ b/modules/lagoon/attach/infra_smoke_test.go @@ -0,0 +1,60 @@ +package attach + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "git.golem15.com/golem15/summercms/modules/compass" +) + +// openTestConfig writes storage.yaml into a temp config dir and opens it. +func openTestConfig(t *testing.T, storageYAML string) *compass.Config { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "storage.yaml"), []byte(storageYAML), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := compass.Open(compass.Options{ + Dir: dir, + Env: "development", + Environ: []string{"SUMMER_ENV=development"}, + }) + if err != nil { + t.Fatal(err) + } + return cfg +} + +// TestOpenBucketS3Smoke runs in both tag modes: without summercms_s3 an +// s3:// bucket_url fails boot naming the tag, with it the bucket opens +// (s3blob connects lazily, so no network is needed). +func TestOpenBucketS3Smoke(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "smoke-access-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "smoke-secret-key") + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") + t.Cleanup(func() { setPublicPathPrefix(defaultPublicPathPrefix) }) + + cfg := openTestConfig(t, "uploads:\n bucket_url: \"s3://smoke-bucket?region=us-east-1\"\n") + bucket, err := OpenBucket(t.Context(), cfg) + if !s3Enabled { + if err == nil { + _ = bucket.Close() + t.Fatal("OpenBucket opened s3:// without the summercms_s3 tag") + } + for _, want := range []string{S3BuildTag, "build.tags", "-tags " + S3BuildTag} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error %q does not name %q", err, want) + } + } + return + } + if err != nil { + t.Fatalf("OpenBucket with the summercms_s3 tag: %v", err) + } + if err := bucket.Close(); err != nil { + t.Fatalf("close: %v", err) + } +} diff --git a/modules/lagoon/attach/s3_off.go b/modules/lagoon/attach/s3_off.go new file mode 100644 index 0000000..5ebd85f --- /dev/null +++ b/modules/lagoon/attach/s3_off.go @@ -0,0 +1,7 @@ +//go:build !summercms_s3 + +package attach + +// Without the summercms_s3 tag the S3 driver is not linked; OpenBucket refuses +// an s3:// bucket_url with an error naming the tag. +const s3Enabled = false diff --git a/modules/lagoon/attach/s3_on.go b/modules/lagoon/attach/s3_on.go new file mode 100644 index 0000000..9e2da30 --- /dev/null +++ b/modules/lagoon/attach/s3_on.go @@ -0,0 +1,9 @@ +//go:build summercms_s3 + +package attach + +// Binaries built with the summercms_s3 tag (summer.yaml build.tags) link the +// gocloud S3 driver, so storage.uploads.bucket_url may use s3://. +import _ "gocloud.dev/blob/s3blob" + +const s3Enabled = true