feat(surf): serve startup summary, shutdown report and http request log

- serve prints a startup summary on a terminal (version, environment, URL,
  admin, database host and name, pending migrations, plugins, drivers) and
  logs it as a 'startup summary' record plus 'listening on <addr>'
- on SIGINT/SIGTERM serve drains, prints and logs a shutdown report and
  returns nil (exit 0), or the drain errors (exit 1)
- the request log is on by default under diary component http with
  request_id; the route pattern, never the path, is logged
- lagoon.PendingMigrations, conga.Worker.Stats, diary.HideOnTTY,
  bonfire.IsTerminal and bonfire.Details back the summary and report

modules/conga/worker.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"runtime/debug"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"git.golem15.com/golem15/summercms/modules/backpack"
@@ -38,6 +39,44 @@ type Worker struct {
 	client   *river.Client[*sql.Tx]
 	listener *pgxpool.Pool
 	queues   []string
+	counters *workerCounters
+}
+
+// WorkerStats counts the job attempts a worker ran since it started.
+type WorkerStats struct {
+	// Done is the number of attempts that returned no error.
+	Done int64
+	// Failed is the number of attempts that returned an error, other than
+	// those cancelled by Stop (HandedBack).
+	Failed int64
+	// InFlight is the number of attempts running now.
+	InFlight int64
+	// HandedBack is the number of attempts cancelled by Stop after its
+	// deadline (River's StopAndCancel). River records the cancellation as
+	// an errored attempt, so the job is retried later, or fails for good
+	// when it was its last attempt.
+	HandedBack int64
+}
+
+// workerCounters are the live counters behind WorkerStats.
+type workerCounters struct {
+	done, failed, inFlight, handedBack atomic.Int64
+	// stopping is set by Stop before running jobs are cancelled.
+	stopping atomic.Bool
+}
+
+// Stats returns the worker's attempt counters. A nil worker has none.
+func (w *Worker) Stats() WorkerStats {
+	if w == nil || w.counters == nil {
+		return WorkerStats{}
+	}
+	c := w.counters
+	return WorkerStats{
+		Done:       c.done.Load(),
+		Failed:     c.failed.Load(),
+		InFlight:   c.inFlight.Load(),
+		HandedBack: c.handedBack.Load(),
+	}
 }
 
 // Queues returns the sorted queue names the worker runs.
@@ -152,9 +191,11 @@ func StartWorker(ctx context.Context, app *backpack.App, plugins []party.Plugin,
 		closePool(listener)
 		return nil, fmt.Errorf("conga: start worker: %w", err)
 	}
+	counters := &workerCounters{}
 	m.worker = client
 	m.schedule = table
-	return &Worker{m: m, client: client, listener: listener, queues: sortedQueueNames(queues)}, nil
+	m.counters.Store(counters)
+	return &Worker{m: m, client: client, listener: listener, queues: sortedQueueNames(queues), counters: counters}, nil
 }
 
 // StartServeWorker starts the in-process worker of the serve command on
@@ -175,6 +216,9 @@ func (w *Worker) Stop(ctx context.Context) error {
 	}
 	err := w.client.Stop(ctx)
 	if err != nil && ctx.Err() != nil {
+		if w.counters != nil {
+			w.counters.stopping.Store(true)
+		}
 		hardCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
 		err = w.client.StopAndCancel(hardCtx)
 		cancel()
@@ -235,8 +279,8 @@ func selectQueues(known map[string]int, want []string) (map[string]int, error) {
 	return out, nil
 }
 
-// runAttempt runs one River attempt of a job and applies the summer_jobs
-// rules: the row id goes into ctx; a panic becomes an error; an error on a
+// runAttempt runs one River attempt of a job, counts it in the running
+// worker's WorkerStats and applies the summer_jobs rules: the row id goes into ctx; a panic becomes an error; an error on a
 // STOPPED or canceled row cancels the River job; an error on the final
 // attempt sets StatusError with the error text under metadata "error". An
 // error on an earlier attempt leaves the row as it is so River can retry.
@@ -245,7 +289,22 @@ func (m *Manager) runAttempt(ctx context.Context, row *rivertype.JobRow, fn func
 	if hasRow {
 		ctx = withJobID(ctx, id)
 	}
+	c := m.counters.Load()
+	if c != nil {
+		c.inFlight.Add(1)
+	}
 	err := m.call(ctx, row, fn)
+	if c != nil {
+		c.inFlight.Add(-1)
+		switch {
+		case err == nil:
+			c.done.Add(1)
+		case c.stopping.Load() && ctx.Err() != nil:
+			c.handedBack.Add(1)
+		default:
+			c.failed.Add(1)
+		}
+	}
 	if err == nil || !hasRow {
 		return err
 	}