feat(13-01): queue jobs whose worker ships later while a worker runs

- a kind no plugin registered always inserts through the insert-only River
  client, so Dispatch and Enqueue no longer fail River's unknown-kind check
  while the in-process worker runs
- while a worker runs, such a kind must name a queue no worker serves;
  an empty queue, default, scheduled, a configured queue or a registered
  job's queue is ErrUnregisteredKindQueue and nothing is written
- README and docs/services/jobs.md describe jobs whose worker ships later
This commit is contained in:
Jakub Zych
2026-10-03 06:28:40 +02:00
parent fbdeb20126
commit 55a4092019
4 changed files with 292 additions and 2 deletions

View File

@@ -34,6 +34,11 @@ var (
// ErrUnknownQueue is returned when a worker is asked for a queue that no
// configuration or registered job names.
ErrUnknownQueue = errors.New("conga: unknown queue")
// ErrUnregisteredKindQueue is returned by Dispatch and Enqueue, while a
// worker runs, for a job kind that no plugin registered when the opts
// name no queue or a queue the worker set serves: a worker would fetch
// the job, find no worker for its kind and discard it.
ErrUnregisteredKindQueue = errors.New("conga: a job kind without a registered job needs a queue no worker serves")
)
// Manager is the app-scoped job manager: it registers jobs, dispatches them
@@ -144,7 +149,7 @@ func (m *Manager) Dispatch(ctx context.Context, db *gorm.DB, args pact.JobArgs,
if err != nil {
return 0, err
}
client, err := m.insertClient()
client, err := m.clientFor(args.Kind(), o.Queue)
if err != nil {
return 0, err
}
@@ -214,7 +219,7 @@ func (m *Manager) Enqueue(ctx context.Context, db *gorm.DB, args pact.JobArgs, o
if args == nil {
return fmt.Errorf("conga: enqueue args are nil")
}
client, err := m.insertClient()
client, err := m.clientFor(args.Kind(), o.Queue)
if err != nil {
return err
}
@@ -453,6 +458,41 @@ func (m *Manager) insertOpts(kind, queue string, maxAttempts int, delay time.Dur
return opts, nil
}
// clientFor picks the River client that inserts a job of kind on queue. A
// registered kind uses insertClient as before. A kind with no registered
// job always goes through the insert-only client, which never checks kinds,
// so its insert cannot fail River's unknown-kind check even when a worker
// starts concurrently. While a worker runs the registry is complete, and
// such a kind must name a queue outside every queue a worker serves
// (default, QueueScheduled, the configured queues and every registered
// job's queue), or it would be fetched and discarded.
func (m *Manager) clientFor(kind, queue string) (*river.Client[*sql.Tx], error) {
m.mu.Lock()
_, registered := m.jobs[kind]
working := m.worker != nil
var served map[string]int
if !registered && working {
served = knownQueues(settingsFromApp(m.app), m.jobs)
served[QueueScheduled] = 0
}
m.mu.Unlock()
if registered {
return m.insertClient()
}
if working {
q := strings.TrimSpace(queue)
if q == "" {
return nil, fmt.Errorf("%w: kind %q names no queue", ErrUnregisteredKindQueue, kind)
}
if _, ok := served[q]; ok {
return nil, fmt.Errorf("%w: kind %q on queue %q, which a worker serves", ErrUnregisteredKindQueue, kind, q)
}
}
m.mu.Lock()
defer m.mu.Unlock()
return m.insertOnlyLocked()
}
// insertClient returns the running worker client, else the lazily built
// insert-only client on the shared pool.
func (m *Manager) insertClient() (*river.Client[*sql.Tx], error) {
@@ -461,6 +501,12 @@ func (m *Manager) insertClient() (*river.Client[*sql.Tx], error) {
if m.worker != nil {
return m.worker, nil
}
return m.insertOnlyLocked()
}
// insertOnlyLocked returns the insert-only client, building it on first
// use. It is separate from the worker client and never started.
func (m *Manager) insertOnlyLocked() (*river.Client[*sql.Tx], error) {
if m.inserter != nil {
return m.inserter, nil
}