feat(11-01): add the conga job framework on River with transactional dispatch
- Pin River v0.47.0 (riverdatabasesql, rivertype) and tidy the example modules - lagoon.Migrate runs the summercms.conga set: River schema v7 and summer_jobs with the apparatus columns plus an internal river_job_id link - conga.Manager.Dispatch writes the summer_jobs row (status IN_PROGRESS) and the River job on the caller's *sql.Tx; a rollback leaves neither - conga.Job wraps typed job functions so plugins never import River - conga.StartWorker runs one client on riverdatabasesql.NewWithPgxListener with a single-connection LISTEN pool; the final failed attempt sets ERROR - TestListenPickupLatency: 30s poll, pickup under 1s; poll-only control 2s miss
This commit is contained in:
@@ -14,7 +14,7 @@ Postgres data layer: the shared GORM connection, per-plugin migrations, model he
|
||||
|
||||
- One shared pool: `lagoon.Open`, `lagoon.Use` and `lagoon.OpenFromApp` return a `*sql.DB` and a `*gorm.DB` built on that same pool; `lagoon.Publish` makes both available on the `backpack.App`.
|
||||
- Database check at connect time: `lagoon.CheckLocale` refuses a database whose default collation is not the ICU `pl-PL` locale, so ordering matches the database default without per-query `COLLATE`.
|
||||
- Per-plugin migrations: `lagoon.Migrate` runs the framework's `system_files` set (`attach.Migrations`) and backend admin identity set (`lagoon.BackendAdminMigrations`), then every `pact.HasMigrations` set in plugin activation order, each in its own `summer_migrations_<plugin_id>` history table (`lagoon.HistoryTableName`). `lagoon.RollbackLast` and `lagoon.Status` cover rollback and history.
|
||||
- Per-plugin migrations: `lagoon.Migrate` runs the framework's `system_files` set (`attach.Migrations`), backend admin identity set (`lagoon.BackendAdminMigrations`) and job-queue set (`lagoon.QueueMigrations`: River's schema pinned at `lagoon.RiverSchemaVersion`, then the `lagoon.JobsTable` record table, under the `lagoon.QueueHistoryID` history), then every `pact.HasMigrations` set in plugin activation order, each in its own `summer_migrations_<plugin_id>` history table (`lagoon.HistoryTableName`). `lagoon.RollbackLast` and `lagoon.Status` cover rollback and history.
|
||||
- Mass assignment: `lagoon.Fill` copies only allow-listed keys onto a model by GORM column name and silently drops the rest, logging each dropped key once outside production. A `json.Number` (from a decoder using `UseNumber`) fills integer, unsigned and float fields. A value that does not fit its column (a fraction, an exponent or an overflow for an integer field, or a value of the wrong type) is a `lagoon.FillTypeError` naming the key, so a caller can answer it as a validation failure on that field. `lagoon.HasFillable` and `lagoon.HasHidden` are the Go forms of `$fillable` and `$hidden`.
|
||||
- Validation: `lagoon.Validate` accepts Laravel-style rule strings (`required`, `nullable`, `integer`, `numeric`, `between`, `min`, `max`, `in`, `unique`, `boolean`, `email`, `confirmed`, `different`, `mimes`) and returns a field-to-messages map, translated through phrasebook when a translator is given. Unknown rule tokens are an error.
|
||||
- Safe ordering: `lagoon.OrderBy` appends an ORDER BY only for an allow-listed column and an `asc` or `desc` direction.
|
||||
@@ -95,6 +95,10 @@ func (p *Plugin) Migrations() []*gormigrate.Migration {
|
||||
| `lagoon.Status` | Lists applied migration IDs per plugin as `lagoon.StatusRow` values. |
|
||||
| `lagoon.HistoryTableName` | Returns the gormigrate history table for a plugin ID. |
|
||||
| `lagoon.BackendAdminMigrations` | Creates the backend user, role and admin token blacklist tables and seeds the system roles. |
|
||||
| `lagoon.QueueMigrations` | Migrates River's schema to `lagoon.RiverSchemaVersion` and creates the `summer_jobs` job record table. |
|
||||
| `lagoon.QueueHistoryID` | History id of the job-queue set, `summercms.conga`. |
|
||||
| `lagoon.JobsTable` | Name of the job record table, `summer_jobs`. |
|
||||
| `lagoon.RiverSchemaVersion` | The pinned River schema version, 7. |
|
||||
| `lagoon.RuntimeCommands` | Returns the migrate, migrate:rollback, migrate:status and key:generate commands. |
|
||||
| `lagoon.KeyGenerateCommand` | Returns the key:generate command on its own. |
|
||||
| `lagoon.LoadAppKey` | Decodes `app.key` and `app.previous_keys`. |
|
||||
@@ -155,7 +159,7 @@ storage:
|
||||
## Dependencies
|
||||
|
||||
- SummerCMS modules: [backpack](../backpack/README.md), [bonfire](../bonfire/README.md), [compass](../compass/README.md), [pact](../pact/README.md), [party](../party/README.md), [phrasebook](../phrasebook/README.md) (validation messages).
|
||||
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`.
|
||||
- Third-party: `gorm.io/gorm`, `gorm.io/driver/postgres`, `github.com/jackc/pgx/v5` (stdlib driver), `github.com/go-gormigrate/gormigrate/v2`, `github.com/go-playground/validator/v10`, `github.com/riverqueue/river` (its `rivermigrate` and `riverdriver/riverdatabasesql` packages, for the River schema in `lagoon.QueueMigrations`).
|
||||
- Third-party, `attach` only: `gocloud.dev/blob` (file and memory drivers), `github.com/disintegration/imaging` (thumbnails).
|
||||
- Standard library: `database/sql`, `crypto/aes`, `crypto/cipher`, `crypto/hkdf`, `log/slog`, `image`, among others.
|
||||
|
||||
|
||||
@@ -57,8 +57,9 @@ func migrator(gdb *gorm.DB, pluginID string, migrations []*gormigrate.Migration)
|
||||
}, migrations), nil
|
||||
}
|
||||
|
||||
// Migrate runs the framework-owned system_files and backend-admin sets
|
||||
// first, then each plugin's HasMigrations set in party.Activate order.
|
||||
// Migrate runs the framework-owned system_files, backend-admin and job-queue
|
||||
// (QueueMigrations) sets first, then each plugin's HasMigrations set in
|
||||
// party.Activate order.
|
||||
func Migrate(gdb *gorm.DB, plugins []party.Plugin) error {
|
||||
if gdb == nil {
|
||||
return fmt.Errorf("lagoon: gorm db is nil")
|
||||
@@ -77,6 +78,17 @@ func Migrate(gdb *gorm.DB, plugins []party.Plugin) error {
|
||||
if err := admin.Migrate(); err != nil {
|
||||
return fmt.Errorf("lagoon: migrate backend admin: %w", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("lagoon: migrate queue: %w", err)
|
||||
}
|
||||
queue, err := migrator(gdb, QueueHistoryID, QueueMigrations(sqlDB))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := queue.Migrate(); err != nil {
|
||||
return fmt.Errorf("lagoon: migrate queue: %w", err)
|
||||
}
|
||||
for _, p := range plugins {
|
||||
hm, ok := p.(pact.HasMigrations)
|
||||
if !ok {
|
||||
|
||||
86
modules/lagoon/queue_migrations.go
Normal file
86
modules/lagoon/queue_migrations.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package lagoon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"github.com/riverqueue/river/riverdriver/riverdatabasesql"
|
||||
"github.com/riverqueue/river/rivermigrate"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
// QueueHistoryID is the migration history id of the framework job-queue
|
||||
// set. Its history table is summer_migrations_summercms_conga.
|
||||
QueueHistoryID = "summercms.conga"
|
||||
// JobsTable is the framework job record table. Its columns match the
|
||||
// WinterCMS apparatus jobs table plus the internal river_job_id link.
|
||||
JobsTable = "summer_jobs"
|
||||
// RiverSchemaVersion pins the River schema the framework migrates to, so a
|
||||
// River upgrade never applies new DDL without a new framework migration.
|
||||
RiverSchemaVersion = 7
|
||||
)
|
||||
|
||||
// QueueMigrations returns the framework job-queue migration set: River's
|
||||
// schema pinned at RiverSchemaVersion, then the summer_jobs record table.
|
||||
// River's migrator runs each of its own steps in a separate transaction on
|
||||
// sqlDB, because some River migrations cannot share one transaction.
|
||||
func QueueMigrations(sqlDB *sql.DB) []*gormigrate.Migration {
|
||||
return []*gormigrate.Migration{
|
||||
{
|
||||
ID: "202609290001_river_schema",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return migrateRiver(txContext(tx), sqlDB, rivermigrate.DirectionUp, &rivermigrate.MigrateOpts{TargetVersion: RiverSchemaVersion})
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// TargetVersion -1 applies every down step, removing River's schema.
|
||||
return migrateRiver(txContext(tx), sqlDB, rivermigrate.DirectionDown, &rivermigrate.MigrateOpts{TargetVersion: -1})
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "202609290002_summer_jobs",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
return tx.Exec(`CREATE TABLE IF NOT EXISTS summer_jobs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
label VARCHAR(255) NOT NULL,
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
progress_max INTEGER NOT NULL DEFAULT 0,
|
||||
user_id INTEGER NULL,
|
||||
is_admin BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_canceled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
metadata TEXT NOT NULL,
|
||||
river_job_id BIGINT NULL,
|
||||
created_at TIMESTAMPTZ NULL,
|
||||
updated_at TIMESTAMPTZ NULL
|
||||
)`).Error
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
return tx.Exec(`DROP TABLE IF EXISTS summer_jobs`).Error
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func migrateRiver(ctx context.Context, sqlDB *sql.DB, dir rivermigrate.Direction, opts *rivermigrate.MigrateOpts) error {
|
||||
if sqlDB == nil {
|
||||
return fmt.Errorf("lagoon: river migrate: sql db is nil")
|
||||
}
|
||||
m, err := rivermigrate.New(riverdatabasesql.New(sqlDB), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("lagoon: river migrate: %w", err)
|
||||
}
|
||||
if _, err := m.Migrate(ctx, dir, opts); err != nil {
|
||||
return fmt.Errorf("lagoon: river migrate %s: %w", dir, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func txContext(tx *gorm.DB) context.Context {
|
||||
if tx != nil && tx.Statement != nil && tx.Statement.Context != nil {
|
||||
return tx.Statement.Context
|
||||
}
|
||||
return context.Background()
|
||||
}
|
||||
Reference in New Issue
Block a user