diff --git a/README.md b/README.md index 74a10be..1ef421d 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ Every entry of `featured_images` (on `GET /posts`, `GET /posts/{slug}` and the a - **When they are made.** `POST /_journal/api/v1/posts/{id}/featured-images` stores the original and queues the conga job `golem15.journal.image_variants` (`classes.ImageVariantsArgs`, three attempts) after the row is written; the request itself never resizes or encodes, and a queue that cannot take the job is logged without failing the upload. An admin save of a post queues the job, after the save commits, for its featured images that lack a width. - **What is made.** For each width of `golem15.journal.image_widths` no larger than the original, `classes.GenerateImageVariants` writes the thumbnail (`attach.File.ThumbKey`, mode `exact`, the original's aspect ratio) and its WebP copy (`webp.Variant` of the framework's `modules/lagoon/attach/webp`). A width that has its row is skipped, so the job and the command can run any number of times. A missing or undecodable original, or one over the thumbnailer's 4096 by 4096 pixel limit, is skipped and logged. -- **Backfill.** `journal:image-variants` takes the images that lack a width no larger than their original; `--all` checks every featured image. +- **Backfill.** `journal:image-variants` takes the images that lack a width no larger than their original; `--all` checks every featured image. It runs from the application's CLI too: without a running server it opens `database.dsn` and `storage.uploads.bucket_url` itself. - **Not made.** AVIF (deferred). Images inserted into a post body keep their original file. - **Server.** WebP is encoded in Go. With the system libwebp installed (`libwebp.so` and `libwebpdemux.so`, Debian `libwebp-dev`) it is about five times faster. A binary that mounts this plugin is dynamically linked against glibc, unless it is built with `-tags nodynamic` (see the framework's attachments documentation). diff --git a/console/database.go b/console/database.go new file mode 100644 index 0000000..116cde0 --- /dev/null +++ b/console/database.go @@ -0,0 +1,55 @@ +package console + +import ( + "context" + "errors" + + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/lagoon" + "git.golem15.com/golem15/summercms/modules/lagoon/attach" + "gocloud.dev/blob" + "gorm.io/gorm" +) + +// lookupDB returns the app's database. The generated CLI publishes none for +// plugin commands (only serve and the framework's own commands open one), +// so without a published database it opens database.dsn and publishes it, +// as lagoon's migrate does. done closes a pool this call opened and is a +// no-op otherwise. +func lookupDB(ctx context.Context, app *backpack.App) (*gorm.DB, func(), error) { + if app == nil { + return nil, nil, errors.New("journal: app is nil") + } + if db := mustDB(app); db != nil { + return db, func() {}, nil + } + if lagoon.DSN(app.Config) == "" { + return nil, nil, errors.New("journal: database is not configured (set SUMMER_DATABASE__DSN)") + } + sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app) + if err != nil { + return nil, nil, err + } + if err := lagoon.Publish(app, sqlDB, gdb); err != nil { + _ = sqlDB.Close() + return nil, nil, err + } + return gdb, func() { _ = sqlDB.Close() }, nil +} + +// lookupBucket returns the published uploads bucket, or opens +// storage.uploads.bucket_url when the process has none (the generated +// CLI). done closes a bucket this call opened and is a no-op otherwise. +func lookupBucket(ctx context.Context, app *backpack.App) (*blob.Bucket, func(), error) { + if app == nil { + return nil, nil, errors.New("journal: app is nil") + } + if bucket, ok := app.Lookup[*blob.Bucket](); ok && bucket != nil { + return bucket, func() {}, nil + } + bucket, err := attach.OpenBucket(ctx, app.Config) + if err != nil { + return nil, nil, err + } + return bucket, func() { _ = bucket.Close() }, nil +} diff --git a/console/image_variants.go b/console/image_variants.go index 8f25b22..2f4f6c9 100644 --- a/console/image_variants.go +++ b/console/image_variants.go @@ -2,13 +2,11 @@ package console import ( "context" - "errors" "fmt" "git.golem15.com/golem15/sm-journal-plugin/classes" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/bonfire" - "gocloud.dev/blob" ) // ImageVariants is journal:image-variants: it writes the thumbnails and @@ -31,7 +29,11 @@ queued after the upload commits. This command is the backfill: without --all it takes the images that lack a variant of some width (for example after adding a width), with --all it checks every featured image. A variant that exists is never written again, so the command can run any -number of times.`, +number of times. + +It runs from the application's CLI as well as next to serve: without a +running server it opens database.dsn and storage.uploads.bucket_url +itself.`, Flags: []bonfire.Flag{ {Name: "all", Description: "Check every featured image (default: only those missing a width)", Bare: true}, }, @@ -40,14 +42,16 @@ number of times.`, {Command: "journal:image-variants --all", Description: "Check every featured image"}, }, Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error { - db := mustDB(app) - if db == nil { - return errors.New("journal: database is not configured") + db, done, err := lookupDB(ctx, app) + if err != nil { + return err } - bucket, ok := app.Lookup[*blob.Bucket]() - if !ok || bucket == nil { - return errors.New("journal: the uploads bucket is not configured") + defer done() + bucket, closeBucket, err := lookupBucket(ctx, app) + if err != nil { + return err } + defer closeBucket() ids, err := classes.FeaturedImageIDs(db.WithContext(ctx), appConfig(app), flagOn(in, "all")) if err != nil { return err diff --git a/image_variants_cli_test.go b/image_variants_cli_test.go new file mode 100644 index 0000000..527d28b --- /dev/null +++ b/image_variants_cli_test.go @@ -0,0 +1,99 @@ +package journal_test + +import ( + "bytes" + "context" + "encoding/base64" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "git.golem15.com/golem15/sm-journal-plugin/console" + "git.golem15.com/golem15/sm-journal-plugin/models" + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/compass" + "git.golem15.com/golem15/summercms/modules/lagoon/attach" + "gocloud.dev/blob" + _ "gocloud.dev/blob/fileblob" +) + +// TestImageVariantsCommandCLI pins journal:image-variants in the +// generated CLI, where no database and no bucket are published: the +// command opens database.dsn and storage.uploads.bucket_url itself and +// writes the missing variants; without a DSN it names the setting. +func TestImageVariantsCommandCLI(t *testing.T) { + if testShort() { + t.Skip("Postgres") + } + env := newPostsEnv(t) + var name string + if err := env.gdb.Raw(`SELECT current_database()`).Scan(&name).Error; err != nil { + t.Fatal(err) + } + u, err := url.Parse(adminDSN) + if err != nil { + t.Fatal(err) + } + u.Path = "/" + name + + post := models.Post{Title: "Cli", Slug: "cli", Content: "

x

", Published: true} + if err := env.gdb.Create(&post).Error; err != nil { + t.Fatal(err) + } + dir := t.TempDir() + disk := "cli123def456ghi789jkl0.jpg" + key := attach.BlobKey(disk) + if err := os.MkdirAll(filepath.Join(dir, filepath.Dir(key)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, key), jpegOf(t, 900, 600), 0o644); err != nil { + t.Fatal(err) + } + file := attach.File{DiskName: disk, FileName: "cli.jpg", ContentType: "image/jpeg", Field: "featured_images", AttachmentType: post.MorphName(), AttachmentID: strconv.FormatUint(uint64(post.ID), 10)} + if err := env.gdb.Create(&file).Error; err != nil { + t.Fatal(err) + } + + run := func(environ ...string) (int, string) { + t.Helper() + cfg, err := compass.Open(compass.Options{Dir: t.TempDir(), Environ: append([]string{"SUMMER_ENV=development"}, environ...)}) + if err != nil { + t.Fatal(err) + } + app := backpack.New(cfg) + var buf strings.Builder + root, err := bonfire.NewRootIO("summer", []bonfire.Command{console.ImageVariants(app)}, strings.NewReader(""), &buf, &buf) + if err != nil { + t.Fatal(err) + } + return bonfire.Execute(context.Background(), root, []string{"journal:image-variants"}), buf.String() + } + + if code, out := run(); code == 0 || !strings.Contains(out, "database is not configured (set SUMMER_DATABASE__DSN)") { + t.Fatalf("without a DSN: %d\n%s", code, out) + } + code, out := run("SUMMER_DATABASE__DSN="+u.String(), "SUMMER_STORAGE__UPLOADS__BUCKET_URL=file://"+dir, + "SUMMER_APP__KEY="+base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("k"), 32))) + if code != 0 || !strings.Contains(out, "Checked 1 featured images, wrote 2 variants") { + t.Fatalf("journal:image-variants from the CLI: %d\n%s", code, out) + } + var variants []models.ImageVariant + if err := env.gdb.Where("file_id = ?", file.ID).Order("width ASC").Find(&variants).Error; err != nil { + t.Fatal(err) + } + if len(variants) != 2 || variants[0].Width != 480 || variants[1].Width != 800 { + t.Fatalf("variants = %+v, want 480 and 800 (1200 is wider than the original)", variants) + } + bucket, err := blob.OpenBucket(t.Context(), "file://"+dir) + if err != nil { + t.Fatal(err) + } + defer bucket.Close() + if ok, err := bucket.Exists(t.Context(), variants[1].WebPKey); err != nil || !ok { + t.Fatalf("the CLI's WebP %s is not in the bucket: %v", variants[1].WebPKey, err) + } +}