feat(13-01): queue jobs whose worker ships later while a worker runs
- a kind no plugin registered always inserts through the insert-only River client, so Dispatch and Enqueue no longer fail River's unknown-kind check while the in-process worker runs - while a worker runs, such a kind must name a queue no worker serves; an empty queue, default, scheduled, a configured queue or a registered job's queue is ErrUnregisteredKindQueue and nothing is written - README and docs/services/jobs.md describe jobs whose worker ships later
This commit is contained in:
@@ -34,6 +34,11 @@ var (
|
||||
// ErrUnknownQueue is returned when a worker is asked for a queue that no
|
||||
// configuration or registered job names.
|
||||
ErrUnknownQueue = errors.New("conga: unknown queue")
|
||||
// ErrUnregisteredKindQueue is returned by Dispatch and Enqueue, while a
|
||||
// worker runs, for a job kind that no plugin registered when the opts
|
||||
// name no queue or a queue the worker set serves: a worker would fetch
|
||||
// the job, find no worker for its kind and discard it.
|
||||
ErrUnregisteredKindQueue = errors.New("conga: a job kind without a registered job needs a queue no worker serves")
|
||||
)
|
||||
|
||||
// Manager is the app-scoped job manager: it registers jobs, dispatches them
|
||||
@@ -144,7 +149,7 @@ func (m *Manager) Dispatch(ctx context.Context, db *gorm.DB, args pact.JobArgs,
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
client, err := m.insertClient()
|
||||
client, err := m.clientFor(args.Kind(), o.Queue)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -214,7 +219,7 @@ func (m *Manager) Enqueue(ctx context.Context, db *gorm.DB, args pact.JobArgs, o
|
||||
if args == nil {
|
||||
return fmt.Errorf("conga: enqueue args are nil")
|
||||
}
|
||||
client, err := m.insertClient()
|
||||
client, err := m.clientFor(args.Kind(), o.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -453,6 +458,41 @@ func (m *Manager) insertOpts(kind, queue string, maxAttempts int, delay time.Dur
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
// clientFor picks the River client that inserts a job of kind on queue. A
|
||||
// registered kind uses insertClient as before. A kind with no registered
|
||||
// job always goes through the insert-only client, which never checks kinds,
|
||||
// so its insert cannot fail River's unknown-kind check even when a worker
|
||||
// starts concurrently. While a worker runs the registry is complete, and
|
||||
// such a kind must name a queue outside every queue a worker serves
|
||||
// (default, QueueScheduled, the configured queues and every registered
|
||||
// job's queue), or it would be fetched and discarded.
|
||||
func (m *Manager) clientFor(kind, queue string) (*river.Client[*sql.Tx], error) {
|
||||
m.mu.Lock()
|
||||
_, registered := m.jobs[kind]
|
||||
working := m.worker != nil
|
||||
var served map[string]int
|
||||
if !registered && working {
|
||||
served = knownQueues(settingsFromApp(m.app), m.jobs)
|
||||
served[QueueScheduled] = 0
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if registered {
|
||||
return m.insertClient()
|
||||
}
|
||||
if working {
|
||||
q := strings.TrimSpace(queue)
|
||||
if q == "" {
|
||||
return nil, fmt.Errorf("%w: kind %q names no queue", ErrUnregisteredKindQueue, kind)
|
||||
}
|
||||
if _, ok := served[q]; ok {
|
||||
return nil, fmt.Errorf("%w: kind %q on queue %q, which a worker serves", ErrUnregisteredKindQueue, kind, q)
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.insertOnlyLocked()
|
||||
}
|
||||
|
||||
// insertClient returns the running worker client, else the lazily built
|
||||
// insert-only client on the shared pool.
|
||||
func (m *Manager) insertClient() (*river.Client[*sql.Tx], error) {
|
||||
@@ -461,6 +501,12 @@ func (m *Manager) insertClient() (*river.Client[*sql.Tx], error) {
|
||||
if m.worker != nil {
|
||||
return m.worker, nil
|
||||
}
|
||||
return m.insertOnlyLocked()
|
||||
}
|
||||
|
||||
// insertOnlyLocked returns the insert-only client, building it on first
|
||||
// use. It is separate from the worker client and never started.
|
||||
func (m *Manager) insertOnlyLocked() (*river.Client[*sql.Tx], error) {
|
||||
if m.inserter != nil {
|
||||
return m.inserter, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user