168d65cd1b
- queue:work and schedule:run print a startup summary and a shutdown report on a terminal, log them as records otherwise, and return nil after a clean stop - schedule:run --once ends with 'Ran N due entries (M failed)' - summer dev runs bin/<app> serve with the forwarded global options, reports through Output, stops the old server before starting the new one and prints a dev report on Ctrl+C (exit 0) - lagoon.DatabaseLabel describes a DSN without its secrets - Long help and examples for queue:work, schedule:run, dev, docs:serve
376 lines
13 KiB
Go
376 lines
13 KiB
Go
package conga
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.golem15.com/golem15/summercms/modules/backpack"
|
|
"git.golem15.com/golem15/summercms/modules/bonfire"
|
|
"git.golem15.com/golem15/summercms/modules/diary"
|
|
"git.golem15.com/golem15/summercms/modules/lagoon"
|
|
"git.golem15.com/golem15/summercms/modules/pact"
|
|
"git.golem15.com/golem15/summercms/modules/party"
|
|
"github.com/riverqueue/river"
|
|
"github.com/riverqueue/river/rivertype"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// clearBatch is the number of jobs one JobDeleteMany pass removes.
|
|
const clearBatch = 10000
|
|
|
|
// RuntimeCommands returns the queue:work, schedule:run and queue:clear
|
|
// commands for the application binary.
|
|
func RuntimeCommands(app *backpack.App, plugins []party.Plugin) []bonfire.Command {
|
|
return []bonfire.Command{
|
|
{
|
|
Name: "queue:work",
|
|
Description: "Run background job workers in the foreground",
|
|
Long: `Starts one River worker on the shared database pool for every known queue, or
|
|
for the queues named with --queue, and runs jobs until Ctrl+C or SIGTERM. It
|
|
prints a startup summary (environment, queues with their concurrency, database
|
|
host, whether the scheduled queue runs here) on a terminal, and logs it as a
|
|
"startup summary" record of component queue otherwise.
|
|
|
|
On a signal it stops fetching jobs and waits up to 10 seconds for running
|
|
ones; jobs still running then are cancelled and River retries them later.
|
|
It prints a shutdown report (jobs done, failed, handed back to the queue,
|
|
uptime) and exits 0, or 1 when the worker fails to stop.
|
|
|
|
serve runs the same worker in-process unless queue.work_in_serve is false;
|
|
run queue:work as a separate process when it is.`,
|
|
Examples: []bonfire.Example{
|
|
{Command: "queue:work", Description: "Work every known queue"},
|
|
{Command: "queue:work --queue mail --queue default", Description: "Work only the mail and default queues"},
|
|
},
|
|
Flags: []bonfire.Flag{{
|
|
Name: "queue",
|
|
Description: "Queue to work (repeatable; default all known queues)",
|
|
Repeatable: true,
|
|
}},
|
|
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
|
return withDB(ctx, app, func() error {
|
|
return work(ctx, app, plugins, in.Flags("queue"), out)
|
|
})
|
|
},
|
|
},
|
|
{
|
|
Name: "schedule:run",
|
|
Description: "Run the scheduler in the foreground (or once for system cron)",
|
|
Long: `Without --once it runs a worker on the scheduled queue that also enqueues every
|
|
plugin's schedule entries, prints a startup summary (entry count, the next
|
|
due entry) and runs until Ctrl+C or SIGTERM, then prints a report (scheduled
|
|
commands run and failed, uptime) and exits 0.
|
|
|
|
With --once it runs the entries due in the current minute in app.timezone,
|
|
one after another, ends with "Ran N due entries (M failed)" and exits 1 when
|
|
any of them failed. Use it from system cron when no worker runs the
|
|
scheduled queue.`,
|
|
Examples: []bonfire.Example{
|
|
{Command: "schedule:run", Description: "Run the scheduler in the foreground"},
|
|
{Command: "schedule:run --once", Description: "From cron, every minute: run the entries due now and exit"},
|
|
},
|
|
Flags: []bonfire.Flag{{
|
|
Name: "once",
|
|
Description: "Run the entries due this minute and exit (for system cron)",
|
|
Bare: true,
|
|
}},
|
|
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
|
if v, ok := in.Flag("once"); ok {
|
|
if once, _ := strconv.ParseBool(v); once {
|
|
return scheduleRunOnce(ctx, app, plugins, scheduleNow(), out)
|
|
}
|
|
}
|
|
return withDB(ctx, app, func() error {
|
|
return runScheduler(ctx, app, plugins, out)
|
|
})
|
|
},
|
|
},
|
|
{
|
|
Name: "queue:clear",
|
|
Description: "Clear all queued jobs, by deleting all pending jobs.",
|
|
Args: []bonfire.Arg{{
|
|
Name: "queue",
|
|
Description: "The name of the queue to clear (default \"default\")",
|
|
}},
|
|
Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
|
|
queue, _ := in.Argument("queue")
|
|
queue = strings.TrimSpace(queue)
|
|
if queue == "" {
|
|
queue = defaultQueue
|
|
}
|
|
return withDB(ctx, app, func() error {
|
|
return clearQueue(ctx, app, queue, out)
|
|
})
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// conga's console loggers: the startup summaries and shutdown reports of
|
|
// queue:work and schedule:run.
|
|
var (
|
|
queueLog = diary.For("queue")
|
|
scheduleLog = diary.For("schedule")
|
|
)
|
|
|
|
// announce prints rows as a block on a terminal (not under -q) and logs them
|
|
// as one record msg; the record is hidden from a terminal console when the
|
|
// block was printed (diary.HideOnTTY).
|
|
func announce(ctx context.Context, out bonfire.Output, log *slog.Logger, level slog.Level, title, msg string, rows []bonfire.Detail, attrs []slog.Attr) {
|
|
if bonfire.IsTerminal(out) && !bonfire.IsQuiet(out) {
|
|
bonfire.Details(out, title, rows)
|
|
ctx = diary.HideOnTTY(ctx)
|
|
}
|
|
log.LogAttrs(ctx, level, msg, attrs...)
|
|
}
|
|
|
|
// workerReport is the shutdown report of a worker: its job counters, the
|
|
// uptime and the stop error.
|
|
func workerReport(w *Worker, started time.Time, stopErr error) ([]bonfire.Detail, []slog.Attr, slog.Level) {
|
|
st := w.Stats()
|
|
errText := "none"
|
|
level := slog.LevelInfo
|
|
if stopErr != nil {
|
|
errText = stopErr.Error()
|
|
level = slog.LevelError
|
|
}
|
|
up := time.Since(started).Round(time.Second)
|
|
rows := []bonfire.Detail{
|
|
{Label: "Jobs", Value: fmt.Sprintf("%d done, %d failed, %d handed back to the queue", st.Done, st.Failed, st.HandedBack+st.InFlight), Warn: st.Failed > 0},
|
|
{Label: "Uptime", Value: up.String()},
|
|
{Label: "Errors", Value: errText, Warn: stopErr != nil},
|
|
}
|
|
attrs := []slog.Attr{
|
|
slog.Int64("jobs_done", st.Done),
|
|
slog.Int64("jobs_failed", st.Failed),
|
|
slog.Int64("jobs_handed_back", st.HandedBack+st.InFlight),
|
|
slog.Int64("uptime_ms", time.Since(started).Milliseconds()),
|
|
}
|
|
if stopErr != nil {
|
|
attrs = append(attrs, slog.String("errors", errText))
|
|
}
|
|
return rows, attrs, level
|
|
}
|
|
|
|
// queueList is "default (10), mail (2)": each queue with its MaxWorkers.
|
|
func queueList(w *Worker) string {
|
|
parts := make([]string, 0, len(w.queues))
|
|
for _, q := range w.queues {
|
|
parts = append(parts, fmt.Sprintf("%s (%d)", q, w.concurrency[q]))
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func work(ctx context.Context, app *backpack.App, plugins []party.Plugin, queues []string, out bonfire.Output) error {
|
|
started := time.Now()
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
w, err := StartWorker(ctx, app, plugins, WorkerOptions{Queues: queues})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
scheduler := "not in this worker (the " + QueueScheduled + " queue is not selected)"
|
|
if slices.Contains(w.queues, QueueScheduled) {
|
|
scheduler = "included (the " + QueueScheduled + " queue runs scheduled commands)"
|
|
}
|
|
env := "unknown"
|
|
if app.Config != nil && app.Config.Environment() != "" {
|
|
env = app.Config.Environment()
|
|
}
|
|
db := lagoon.DatabaseLabel(lagoon.DSN(app.Config))
|
|
announce(ctx, out, queueLog, slog.LevelInfo, "Worker started", "startup summary", []bonfire.Detail{
|
|
{Label: "Environment", Value: env},
|
|
{Label: "Queues", Value: queueList(w)},
|
|
{Label: "Database", Value: db},
|
|
{Label: "Scheduler", Value: scheduler},
|
|
}, []slog.Attr{
|
|
slog.String("environment", env),
|
|
slog.String("queues", strings.Join(w.queues, ",")),
|
|
slog.String("database", db),
|
|
slog.String("scheduler", scheduler),
|
|
})
|
|
<-ctx.Done()
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
stopErr := w.Stop(stopCtx)
|
|
rows, attrs, level := workerReport(w, started, stopErr)
|
|
announce(context.WithoutCancel(ctx), out, queueLog, level, "Shutdown report", "shutdown report", rows, attrs)
|
|
return stopErr
|
|
}
|
|
|
|
// nextDue is the earliest next run of entries after now, and its command.
|
|
func nextDue(entries []scheduleEntry, now time.Time) (time.Time, string) {
|
|
var at time.Time
|
|
var cmd string
|
|
for _, e := range entries {
|
|
if e.schedule == nil {
|
|
continue
|
|
}
|
|
if n := e.schedule.Next(now); at.IsZero() || n.Before(at) {
|
|
at, cmd = n, e.cmd.Command
|
|
}
|
|
}
|
|
return at, cmd
|
|
}
|
|
|
|
// runScheduler runs a scheduler-only worker (the scheduled queue plus every
|
|
// plugin's periodic jobs) until SIGINT or SIGTERM.
|
|
func runScheduler(ctx context.Context, app *backpack.App, plugins []party.Plugin, out bonfire.Output) error {
|
|
started := time.Now()
|
|
ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
w, err := StartWorker(ctx, app, plugins, WorkerOptions{Queues: []string{QueueScheduled}})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries, _ := scheduleEntries(app, plugins)
|
|
next := "none"
|
|
if at, cmd := nextDue(entries, time.Now()); cmd != "" {
|
|
next = cmd + " at " + at.Format("2006-01-02 15:04 MST")
|
|
}
|
|
announce(ctx, out, scheduleLog, slog.LevelInfo, "Scheduler started", "startup summary", []bonfire.Detail{
|
|
{Label: "Entries", Value: fmt.Sprint(len(entries))},
|
|
{Label: "Next", Value: next},
|
|
}, []slog.Attr{
|
|
slog.Int("entries", len(entries)),
|
|
slog.String("next", next),
|
|
})
|
|
<-ctx.Done()
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
stopErr := w.Stop(stopCtx)
|
|
st := w.Stats()
|
|
rows, attrs, level := workerReport(w, started, stopErr)
|
|
rows[0] = bonfire.Detail{Label: "Runs", Value: fmt.Sprintf("%d scheduled commands (%d failed)", st.Done+st.Failed, st.Failed), Warn: st.Failed > 0}
|
|
announce(context.WithoutCancel(ctx), out, scheduleLog, level, "Shutdown report", "shutdown report", rows, attrs)
|
|
return stopErr
|
|
}
|
|
|
|
// scheduleRunOnce runs, without River, every schedule entry due in the
|
|
// minute of now in app.timezone (Laravel schedule:run for system cron). An
|
|
// unregistered command is warned about and skipped; the returned error is the
|
|
// first command error, after every due entry ran. It ends with "Ran N due
|
|
// entries (M failed)".
|
|
func scheduleRunOnce(ctx context.Context, app *backpack.App, plugins []party.Plugin, now time.Time, out bonfire.Output) error {
|
|
entries, err := scheduleEntries(app, plugins)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
loc, err := appLocation(app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
local := now.In(loc)
|
|
var due []scheduleEntry
|
|
for _, e := range entries {
|
|
if dueAt(e.cmd.Cadence, local) {
|
|
due = append(due, e)
|
|
}
|
|
}
|
|
if len(due) == 0 {
|
|
out.Println("No scheduled commands are ready to run.")
|
|
return nil
|
|
}
|
|
return withAppDB(ctx, app, func() error {
|
|
log := loggerFromApp(app)
|
|
var first error
|
|
ran, failed := 0, 0
|
|
for _, e := range due {
|
|
if _, ok := scheduledCatalog(app, log, e.cmd.Command); !ok {
|
|
out.Warning(fmt.Sprintf("Scheduled command %s is not registered; skipping", e.cmd.Command))
|
|
continue
|
|
}
|
|
out.Printf("Running scheduled command: %s\n", strings.Join(append([]string{e.cmd.Command}, e.cmd.Args...), " "))
|
|
ran++
|
|
if _, err := callScheduled(ctx, app, log, e.cmd.Command, e.cmd.Args, out); err != nil {
|
|
failed++
|
|
if first == nil {
|
|
first = err
|
|
}
|
|
}
|
|
}
|
|
out.Info(fmt.Sprintf("Ran %d due entries (%d failed)", ran, failed))
|
|
return first
|
|
})
|
|
}
|
|
|
|
// dueAt reports whether a cadence is due in the wall-clock minute of t: a
|
|
// daily cadence at its hour and minute, Every(d) of a minute or less on every
|
|
// run, and a longer Every(d) when the minutes since midnight are a multiple
|
|
// of d.
|
|
func dueAt(c pact.Cadence, t time.Time) bool {
|
|
if h, m, ok := c.At(); ok {
|
|
return t.Hour() == h && t.Minute() == m
|
|
}
|
|
d := c.Interval()
|
|
if d <= 0 {
|
|
return false
|
|
}
|
|
if d <= time.Minute {
|
|
return true
|
|
}
|
|
sinceMidnight := time.Duration(t.Hour()*60+t.Minute()) * time.Minute
|
|
return sinceMidnight%d == 0
|
|
}
|
|
|
|
// withAppDB runs fn with the app's database: the published one when there is
|
|
// one, otherwise one opened for the command the way withDB does.
|
|
func withAppDB(ctx context.Context, app *backpack.App, fn func() error) error {
|
|
if gdb, ok := app.Lookup[*gorm.DB](); ok && gdb != nil {
|
|
return fn()
|
|
}
|
|
return withDB(ctx, app, fn)
|
|
}
|
|
|
|
// clearQueue deletes the available, scheduled and retryable jobs of queue in
|
|
// batches until a pass deletes none. River never deletes running jobs.
|
|
func clearQueue(ctx context.Context, app *backpack.App, queue string, out bonfire.Output) error {
|
|
m, err := From(app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
client, err := m.insertClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out.Info(fmt.Sprintf("Clearing queue %q", queue))
|
|
total := 0
|
|
for {
|
|
res, err := client.JobDeleteMany(ctx, river.NewJobDeleteManyParams().
|
|
Queues(queue).
|
|
States(rivertype.JobStateAvailable, rivertype.JobStateScheduled, rivertype.JobStateRetryable).
|
|
First(clearBatch))
|
|
if err != nil {
|
|
return fmt.Errorf("conga: clear queue %q: %w", queue, err)
|
|
}
|
|
if len(res.Jobs) == 0 {
|
|
break
|
|
}
|
|
total += len(res.Jobs)
|
|
}
|
|
out.Info(fmt.Sprintf("Cleared %d jobs", total))
|
|
return nil
|
|
}
|
|
|
|
// withDB opens and publishes the shared pool for a CLI command and closes it
|
|
// when fn returns, the way lagoon's own commands do.
|
|
func withDB(ctx context.Context, app *backpack.App, fn func() error) error {
|
|
sqlDB, gdb, err := lagoon.OpenFromApp(ctx, app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer sqlDB.Close()
|
|
if err := lagoon.Publish(app, sqlDB, gdb); err != nil {
|
|
return err
|
|
}
|
|
return fn()
|
|
}
|