feat: image variants job and variants in the post API
- golem15_journal_image_variants (migration 202610280002): a thumbnail
per width of golem15.journal.image_widths (480, 800, 1200 by default,
never wider than the original) and its WebP copy, with the original's
size
- conga job golem15.journal.image_variants (three attempts), queued after
a featured image upload and after an admin save of a post whose images
lack a width; the upload request never encodes
- journal:image-variants [--all] backfills
- featured_images[].variants {width, height, url, webp_url, source_width,
source_height} on the post answers (additive key)
This commit is contained in:
Jakub Zych
284
classes/image_variants.go
Normal file
284
classes/image_variants.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package classes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"io"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.golem15.com/golem15/sm-journal-plugin/models"
|
||||
"git.golem15.com/golem15/summercms/modules/backpack"
|
||||
"git.golem15.com/golem15/summercms/modules/compass"
|
||||
"git.golem15.com/golem15/summercms/modules/conga"
|
||||
"git.golem15.com/golem15/summercms/modules/diary"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon/attach"
|
||||
"git.golem15.com/golem15/summercms/modules/lagoon/attach/webp"
|
||||
"gocloud.dev/blob"
|
||||
"gocloud.dev/gcerrors"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// The image variants job and its configuration (2026-10-28 image formats,
|
||||
// D-85, P33-43).
|
||||
const (
|
||||
// ImageVariantsKind is the conga job kind golem15.journal.image_variants.
|
||||
ImageVariantsKind = "golem15.journal.image_variants"
|
||||
// ImageWidthsKey is the config key of the variant widths, in pixels.
|
||||
ImageWidthsKey = "golem15.journal.image_widths"
|
||||
// ImageVariantsMode is the thumbnail mode of a variant: the exact size
|
||||
// of the original's aspect ratio at the variant's width.
|
||||
ImageVariantsMode = "exact"
|
||||
// FeaturedImagesField is the system_files field of a post's featured
|
||||
// images.
|
||||
FeaturedImagesField = "featured_images"
|
||||
// MinImageWidth and MaxImageWidth bound a configured width.
|
||||
MinImageWidth = 16
|
||||
MaxImageWidth = 4096
|
||||
)
|
||||
|
||||
// variantsLog logs the image variants as component journal.
|
||||
var variantsLog = diary.For("journal")
|
||||
|
||||
// DefaultImageWidths are the variant widths without configuration.
|
||||
var DefaultImageWidths = []int{480, 800, 1200}
|
||||
|
||||
// ImageVariantsArgs are the arguments of the image variants job: the
|
||||
// system_files ids of the featured images to make variants of.
|
||||
type ImageVariantsArgs struct {
|
||||
FileIDs []uint `json:"file_ids"`
|
||||
}
|
||||
|
||||
// Kind is the job kind, golem15.journal.image_variants.
|
||||
func (ImageVariantsArgs) Kind() string { return ImageVariantsKind }
|
||||
|
||||
// ImageWidths reads golem15.journal.image_widths: a list of widths or a
|
||||
// comma-separated text (an environment override), each 16 to 4096, sorted
|
||||
// and once each. An unset, empty or invalid value answers
|
||||
// DefaultImageWidths.
|
||||
func ImageWidths(cfg *compass.Config) []int {
|
||||
if cfg == nil {
|
||||
return slices.Clone(DefaultImageWidths)
|
||||
}
|
||||
raw, ok := cfg.Lookup(ImageWidthsKey)
|
||||
if !ok || raw == nil {
|
||||
return slices.Clone(DefaultImageWidths)
|
||||
}
|
||||
var parts []string
|
||||
switch v := raw.(type) {
|
||||
case string:
|
||||
parts = strings.Split(v, ",")
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
parts = append(parts, fmt.Sprint(item))
|
||||
}
|
||||
case []int:
|
||||
for _, n := range v {
|
||||
parts = append(parts, strconv.Itoa(n))
|
||||
}
|
||||
default:
|
||||
parts = []string{fmt.Sprint(v)}
|
||||
}
|
||||
var out []int
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < MinImageWidth || n > MaxImageWidth {
|
||||
return slices.Clone(DefaultImageWidths)
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return slices.Clone(DefaultImageWidths)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
// VariantsResult is what one run of GenerateImageVariants did.
|
||||
type VariantsResult struct {
|
||||
Files int
|
||||
Written int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
// maxHeaderBytes caps the bytes read for an original's size.
|
||||
const maxHeaderBytes = 1 << 20
|
||||
|
||||
// sourceSize reads an original's size from its header; ok is false for a
|
||||
// missing original, one that does not decode, or one over the thumbnailer's
|
||||
// pixel limit (attach would store the broken-image picture for it).
|
||||
func sourceSize(ctx context.Context, bucket *blob.Bucket, f attach.File) (w, h int, ok bool, err error) {
|
||||
r, err := bucket.NewReader(ctx, attach.BlobKey(f.DiskName), nil)
|
||||
if gcerrors.Code(err) == gcerrors.NotFound {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, 0, false, err
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(r, maxHeaderBytes))
|
||||
closeErr := r.Close()
|
||||
if err != nil {
|
||||
return 0, 0, false, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return 0, 0, false, closeErr
|
||||
}
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > attach.MaxImagePixels {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
return cfg.Width, cfg.Height, true, nil
|
||||
}
|
||||
|
||||
// GenerateImageVariants writes the variants of the files ids (D-85): for
|
||||
// each width of widths no larger than the original's, the thumbnail at that
|
||||
// width (attach.File.ThumbKey, mode exact, the original's aspect ratio) and
|
||||
// its WebP copy (webp.Variant), recorded in golem15_journal_image_variants.
|
||||
// A width that already has its row is skipped, so a second run writes
|
||||
// nothing. A missing, undecodable or oversized original is skipped and
|
||||
// logged. A storage or database error is returned (the job retries).
|
||||
func GenerateImageVariants(ctx context.Context, db *gorm.DB, bucket *blob.Bucket, widths []int, ids []uint) (VariantsResult, error) {
|
||||
var out VariantsResult
|
||||
if db == nil || bucket == nil {
|
||||
return out, errors.New("journal: image variants need the database and the uploads bucket")
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var files []attach.File
|
||||
if err := db.WithContext(ctx).Where("id IN ?", ids).Order("id ASC").Find(&files).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, f := range files {
|
||||
out.Files++
|
||||
var have []int
|
||||
if err := db.WithContext(ctx).Model(&models.ImageVariant{}).Where("file_id = ?", f.ID).Pluck("width", &have).Error; err != nil {
|
||||
return out, err
|
||||
}
|
||||
var todo []int
|
||||
for _, w := range widths {
|
||||
if !slices.Contains(have, w) {
|
||||
todo = append(todo, w)
|
||||
}
|
||||
}
|
||||
if len(todo) == 0 {
|
||||
continue
|
||||
}
|
||||
sw, sh, ok, err := sourceSize(ctx, bucket, f)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("journal: read original %d: %w", f.ID, err)
|
||||
}
|
||||
if !ok {
|
||||
variantsLog.Warn("journal: no image variants for an unusable original", "file", f.ID)
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
for _, w := range todo {
|
||||
if w > sw {
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
h := max(1, int((int64(sh)*int64(w)+int64(sw)/2)/int64(sw)))
|
||||
key, err := f.ThumbKey(ctx, bucket, w, h, ImageVariantsMode)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("journal: thumbnail %d at %d: %w", f.ID, w, err)
|
||||
}
|
||||
webpKey, err := webp.Variant(ctx, bucket, &f, w, h, ImageVariantsMode)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("journal: webp %d at %d: %w", f.ID, w, err)
|
||||
}
|
||||
row := models.ImageVariant{FileID: f.ID, Width: w, Height: h, Key: key, WebPKey: webpKey, SourceWidth: sw, SourceHeight: sh, CreatedAt: time.Now().UTC()}
|
||||
res := db.WithContext(ctx).Exec(`INSERT INTO golem15_journal_image_variants (file_id, width, height, key, webp_key, source_width, source_height, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (file_id, width) DO NOTHING`, row.FileID, row.Width, row.Height, row.Key, row.WebPKey, row.SourceWidth, row.SourceHeight, row.CreatedAt)
|
||||
if res.Error != nil {
|
||||
return out, res.Error
|
||||
}
|
||||
out.Written += int(res.RowsAffected)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// featuredFiles is the featured images of the posts, or of every post when
|
||||
// postIDs is nil; missing is true to keep only the images without any
|
||||
// variant row, or without a row for a width no larger than their original.
|
||||
func featuredFiles(db *gorm.DB, postIDs []uint, widths []int, missing bool) ([]uint, error) {
|
||||
q := db.Model(&attach.File{}).Where("attachment_type = ? AND field = ?", (models.Post{}).MorphName(), FeaturedImagesField)
|
||||
if postIDs != nil {
|
||||
keys := make([]string, 0, len(postIDs))
|
||||
for _, id := range postIDs {
|
||||
keys = append(keys, strconv.FormatUint(uint64(id), 10))
|
||||
}
|
||||
q = q.Where("attachment_id IN ?", keys)
|
||||
}
|
||||
if missing {
|
||||
list := make([]string, 0, len(widths))
|
||||
for _, w := range widths {
|
||||
list = append(list, strconv.Itoa(w))
|
||||
}
|
||||
q = q.Where(`NOT EXISTS (SELECT 1 FROM golem15_journal_image_variants v WHERE v.file_id = system_files.id)
|
||||
OR EXISTS (SELECT 1 FROM unnest(?::int[]) AS w(width)
|
||||
WHERE w.width <= (SELECT MAX(v.source_width) FROM golem15_journal_image_variants v WHERE v.file_id = system_files.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM golem15_journal_image_variants v WHERE v.file_id = system_files.id AND v.width = w.width))`, "{"+strings.Join(list, ",")+"}")
|
||||
}
|
||||
var ids []uint
|
||||
err := q.Order("id ASC").Pluck("id", &ids).Error
|
||||
return ids, err
|
||||
}
|
||||
|
||||
// FeaturedImageIDs lists the featured images of every post (all) or only
|
||||
// those missing a variant of a configured width no larger than their
|
||||
// original (or without any variant yet).
|
||||
func FeaturedImageIDs(db *gorm.DB, cfg *compass.Config, all bool) ([]uint, error) {
|
||||
return featuredFiles(db, nil, ImageWidths(cfg), !all)
|
||||
}
|
||||
|
||||
// DispatchImageVariants queues the image variants job for the files ids.
|
||||
// The upload or save that called it has committed; a queue that cannot
|
||||
// take the job is logged and never fails that request (D-85).
|
||||
func DispatchImageVariants(ctx context.Context, app *backpack.App, ids []uint) {
|
||||
if app == nil || len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
m, err := conga.From(app)
|
||||
if err == nil {
|
||||
_, err = m.Dispatch(ctx, nil, ImageVariantsArgs{FileIDs: ids}, conga.DispatchOpts{Label: "Journal image variants"})
|
||||
}
|
||||
if err != nil {
|
||||
variantsLog.Warn("journal: image variants job not queued", "files", len(ids), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchPostImageVariantsAfterCommit queues the image variants job, once
|
||||
// tx commits, for the post's featured images that lack a variant of some
|
||||
// width (the admin save: the uploads it binds commit with it).
|
||||
func DispatchPostImageVariantsAfterCommit(ctx context.Context, tx *gorm.DB, app *backpack.App, postID uint) {
|
||||
if app == nil || postID == 0 {
|
||||
return
|
||||
}
|
||||
lagoon.AfterCommit(ctx, tx, func(ctx context.Context, _ *gorm.DB) {
|
||||
db, ok := app.Lookup[*gorm.DB]()
|
||||
if !ok || db == nil {
|
||||
return
|
||||
}
|
||||
ids, err := featuredFiles(db.WithContext(ctx), []uint{postID}, ImageWidths(app.Config), true)
|
||||
if err != nil {
|
||||
variantsLog.Warn("journal: image variants of a saved post", "post", postID, "error", err)
|
||||
return
|
||||
}
|
||||
DispatchImageVariants(ctx, app, ids)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user