Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 51 additions & 58 deletions manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,10 @@ type Manager struct {
drainMu sync.RWMutex
draining atomic.Bool

// waiters holds one channel per in-flight Await call, so each entry lives
// exactly as long as the Await that added it.
mu sync.Mutex
waiters map[uuid.UUID]*completion
}

// completion signals one in-flight run's terminal status. kickOnce makes
// concurrent Awaits on a StartTx run launch runLocal at most once.
type completion struct {
done chan struct{}
status Status
once sync.Once
kickOnce sync.Once
waiters map[uuid.UUID]map[chan Status]struct{}
}

// NewManager freezes reg and returns a Manager over db. It fails unless the
Expand Down Expand Up @@ -85,7 +78,7 @@ func NewManager(ctx context.Context, db *pgkit.DB, reg *Registry, cfg Config, lo
cfg: cfg,
log: log,
sem: make(chan struct{}, cfg.MaxLocalRuns),
waiters: make(map[uuid.UUID]*completion),
waiters: make(map[uuid.UUID]map[chan Status]struct{}),
}, nil
}

Expand Down Expand Up @@ -145,21 +138,21 @@ func (m *Manager) start(ctx context.Context, workflowName string, input any, opt
return nil, err
}

runID := run.ID
runID, inserted := run.ID, true
if err := pgx.BeginFunc(ctx, m.db.DB.Conn, func(tx pgx.Tx) error {
if cfg.idempotencyKey == "" {
return m.db.WithTx(tx).InsertRun(ctx, run, tasks)
}
id, _, err := m.db.WithTx(tx).UpsertRun(ctx, run, tasks)
runID = id
id, ok, err := m.db.WithTx(tx).UpsertRun(ctx, run, tasks)
runID, inserted = id, ok
return err
}); err != nil {
return nil, fmt.Errorf("workflow: insert run %q: %w", workflowName, err)
}

c := m.register(runID)
m.kick(ctx, runID, c)
return &WorkflowRun{id: runID, mgr: m}, nil
r := &WorkflowRun{id: runID, mgr: m, inserted: inserted}
r.kick(ctx)
return r, nil
}

func (m *Manager) startTx(ctx context.Context, tx pgx.Tx, workflowName string, input any, opts ...StartOption) (*WorkflowRun, error) {
Expand All @@ -169,21 +162,20 @@ func (m *Manager) startTx(ctx context.Context, tx pgx.Tx, workflowName string, i
return nil, err
}

runID := run.ID
runID, inserted := run.ID, true
if cfg.idempotencyKey == "" {
if err := m.db.WithTx(tx).InsertRun(ctx, run, tasks); err != nil {
return nil, fmt.Errorf("workflow: insert run %q: %w", workflowName, err)
}
} else {
id, _, err := m.db.WithTx(tx).UpsertRun(ctx, run, tasks)
id, ok, err := m.db.WithTx(tx).UpsertRun(ctx, run, tasks)
if err != nil {
return nil, fmt.Errorf("workflow: insert run %q: %w", workflowName, err)
}
runID = id
runID, inserted = id, ok
}

m.register(runID)
return &WorkflowRun{id: runID, mgr: m}, nil
return &WorkflowRun{id: runID, mgr: m, inserted: inserted}, nil
}

func (m *Manager) buildRun(ctx context.Context, workflowName string, input any, idempotencyKey string) (*store.WorkflowRun, []*store.TaskRun, error) {
Expand Down Expand Up @@ -238,23 +230,6 @@ func (m *Manager) buildRun(ctx context.Context, workflowName string, input any,
return run, tasks, nil
}

func (m *Manager) register(runID uuid.UUID) *completion {
m.mu.Lock()
defer m.mu.Unlock()
if c, ok := m.waiters[runID]; ok {
return c
}
c := &completion{done: make(chan struct{})}
m.waiters[runID] = c
return c
}

// kick is shared by Start and a StartTx run's first Await, so a run launches
// once whichever fires first.
func (m *Manager) kick(ctx context.Context, runID uuid.UUID, c *completion) {
c.kickOnce.Do(func() { m.launch(ctx, runID) })
}

// launch skips when the semaphore is full or a drain has started; the poller
// covers the run either way. It holds drainMu's read lock from the draining
// check through wg.Go, so a launch is either registered with the WaitGroup
Expand Down Expand Up @@ -304,43 +279,61 @@ func (m *Manager) runLocal(ctx context.Context, runID uuid.UUID) {
m.executePlan(ctx, claimed)
}

func (m *Manager) signal(runID uuid.UUID, s Status) {
func (m *Manager) subscribe(runID uuid.UUID) chan Status {
done := make(chan Status, 1)
m.mu.Lock()
defer m.mu.Unlock()
if m.waiters[runID] == nil {
m.waiters[runID] = make(map[chan Status]struct{})
}
m.waiters[runID][done] = struct{}{}
return done
}

func (m *Manager) unsubscribe(runID uuid.UUID, done chan Status) {
m.mu.Lock()
c, ok := m.waiters[runID]
if ok {
defer m.mu.Unlock()
delete(m.waiters[runID], done)
if len(m.waiters[runID]) == 0 {
delete(m.waiters, runID)
}
}

// signal runs after the terminal status is persisted, so an Await that
// subscribes too late to receive it reads the status instead.
func (m *Manager) signal(runID uuid.UUID, s Status) {
m.mu.Lock()
subs := m.waiters[runID]
delete(m.waiters, runID)
m.mu.Unlock()
if !ok {
return
for done := range subs {
done <- s
}
c.once.Do(func() {
c.status = s
close(c.done)
})
}

// await returns StatusRunning when ctx expires before the run is terminal;
// the run keeps going in the background.
func (m *Manager) await(ctx context.Context, runID uuid.UUID) (Status, error) {
m.mu.Lock()
c := m.waiters[runID]
m.mu.Unlock()
done := m.subscribe(runID)
defer m.unsubscribe(runID, done)

var done <-chan struct{}
if c != nil {
done = c.done
// A StartTx run launches here; for a Start run kickOnce makes this a no-op.
m.kick(ctx, runID, c)
// A duplicate Start hands back a run that may already be terminal, and a
// budget shorter than awaitPollInterval would otherwise never read it.
run, err := m.db.GetRun(ctx, runID)
Comment thread
klaidliadon marked this conversation as resolved.
Resolved
if err != nil {
return awaitPollFailure(ctx, err)
}
if run.Status != StatusRunning {
return run.Status, nil
}

ticker := time.NewTicker(awaitPollInterval)
defer ticker.Stop()

for {
select {
case <-done:
return c.status, nil
case s := <-done:
return s, nil
case <-ticker.C:
run, err := m.db.GetRun(ctx, runID)
if err != nil {
Expand Down
183 changes: 183 additions & 0 deletions manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ func newManager(t *testing.T, db *store.Store, reg *Registry, overrides ...func(
return mgr
}

func requireNoWaiters(t *testing.T, mgr *Manager) {
t.Helper()
mgr.mu.Lock()
defer mgr.mu.Unlock()
require.Empty(t, mgr.waiters)
}

func twoStageRegistry(t *testing.T) *Registry {
t.Helper()
reg := NewRegistry()
Expand Down Expand Up @@ -428,6 +435,182 @@ func TestManager(t *testing.T) {
require.Equal(t, p1.ID(), p2.ID())
})

t.Run("Await reports a duplicate of a finished plan within a sub-poll budget", func(t *testing.T) {
mgr := newManager(t, db, twoStageRegistry(t))
awaitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

p1, err := mgr.start(ctx, "test.twostage", testInput{Seed: "abc"}, WithIdempotencyKey("dup:2"))
require.NoError(t, err)
status, err := p1.Await(awaitCtx)
require.NoError(t, err)
require.Equal(t, StatusSucceeded, status)

p2, err := mgr.start(ctx, "test.twostage", testInput{Seed: "abc"}, WithIdempotencyKey("dup:2"))
require.NoError(t, err)
shortCtx, cancel2 := context.WithTimeout(ctx, awaitPollInterval/4)
defer cancel2()
status, err = p2.Await(shortCtx)
require.NoError(t, err)
require.Equal(t, StatusSucceeded, status)
requireNoWaiters(t, mgr)
})

t.Run("Await reports a duplicate of a parked plan within a sub-poll budget", func(t *testing.T) {
reg := NewRegistry()
require.NoError(t, reg.Register(Define(NewWorkflow[testInput]("test.park"),
Handle(NewTask[None]("test.park"), func(context.Context, *TaskRun[testInput]) (None, error) {
return None{}, Permanent(errors.New("boom"))
}),
)))
mgr := newManager(t, db, reg)
awaitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

p1, err := mgr.start(ctx, "test.park", testInput{Seed: "abc"}, WithIdempotencyKey("dup:3"))
require.NoError(t, err)
status, err := p1.Await(awaitCtx)
require.NoError(t, err)
require.Equal(t, StatusStuck, status)

p2, err := mgr.start(ctx, "test.park", testInput{Seed: "abc"}, WithIdempotencyKey("dup:3"))
require.NoError(t, err)
shortCtx, cancel2 := context.WithTimeout(ctx, awaitPollInterval/4)
defer cancel2()
status, err = p2.Await(shortCtx)
require.NoError(t, err)
require.Equal(t, StatusStuck, status)
})

t.Run("concurrent duplicate starts never launch the plan", func(t *testing.T) {
mgr := newManager(t, db, twoStageRegistry(t))
awaitCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

p1, err := mgr.start(ctx, "test.twostage", testInput{Seed: "abc"}, WithIdempotencyKey("dup:4"))
require.NoError(t, err)
status, err := p1.Await(awaitCtx)
require.NoError(t, err)
require.Equal(t, StatusSucceeded, status)

// launch blocks on drainMu's read lock, so a duplicate that launches
// never returns while the test holds the write lock.
mgr.drainMu.Lock()
var wg sync.WaitGroup
errs := make(chan error, 9)
awaitDup := func(p *WorkflowRun) error {
if p.ID() != p1.ID() {
return fmt.Errorf("duplicate resolved to %s, want %s", p.ID(), p1.ID())
}
status, err := p.Await(awaitCtx)
if err != nil {
return err
}
if status != StatusSucceeded {
return fmt.Errorf("duplicate awaited %v, want %v", status, StatusSucceeded)
}
return nil
}
for range 8 {
wg.Go(func() {
p, err := mgr.start(ctx, "test.twostage", testInput{Seed: "abc"}, WithIdempotencyKey("dup:4"))
if err != nil {
errs <- err
return
}
errs <- awaitDup(p)
})
}
wg.Go(func() {
var p *WorkflowRun
err := pgx.BeginFunc(ctx, db.DB.Conn, func(tx pgx.Tx) error {
var err error
p, err = mgr.startTx(ctx, tx, "test.twostage", testInput{Seed: "abc"}, WithIdempotencyKey("dup:4"))
return err
})
if err != nil {
errs <- err
return
}
errs <- awaitDup(p)
})
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
var blocked bool
select {
case <-done:
case <-time.After(5 * time.Second):
blocked = true
}
mgr.drainMu.Unlock()
<-done
require.False(t, blocked, "a duplicate start launched the plan")
close(errs)
for err := range errs {
require.NoError(t, err)
}
})

t.Run("Await leaves no waiter when no local slot runs the plan", func(t *testing.T) {
mgr := newManager(t, db, twoStageRegistry(t))
for range cap(mgr.sem) {
mgr.sem <- struct{}{}
}
p, err := mgr.start(ctx, "test.twostage", testInput{Seed: "abc"})
require.NoError(t, err)

shortCtx, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
defer cancel()
status, err := p.Await(shortCtx)
require.NoError(t, err)
require.Equal(t, StatusRunning, status)
requireNoWaiters(t, mgr)
})

t.Run("concurrent Awaits on one plan all receive the signal", func(t *testing.T) {
started, release := make(chan struct{}), make(chan struct{})
reg := NewRegistry()
require.NoError(t, reg.Register(Define(NewWorkflow[testInput]("test.gatedfanout"),
Handle(NewTask[None]("test.gatedfanout"), func(context.Context, *TaskRun[testInput]) (None, error) {
close(started)
<-release
return None{}, nil
}),
)))
mgr := newManager(t, db, reg)
p, err := mgr.start(ctx, "test.gatedfanout", testInput{Seed: "abc"})
require.NoError(t, err)
<-started

awaitCtx, cancel := context.WithTimeout(ctx, awaitPollInterval/2)
defer cancel()
type result struct {
status Status
err error
}
results := make(chan result, 2)
for range 2 {
go func() {
status, err := p.Await(awaitCtx)
results <- result{status, err}
}()
}
require.Eventually(t, func() bool {
mgr.mu.Lock()
defer mgr.mu.Unlock()
return len(mgr.waiters[p.ID()]) == 2
}, time.Second, 5*time.Millisecond)
close(release)

for range 2 {
require.Equal(t, result{status: StatusSucceeded}, <-results)
}
requireNoWaiters(t, mgr)
})

t.Run("Await returns Running when the ctx budget expires", func(t *testing.T) {
reg := NewRegistry()
require.NoError(t, reg.Register(Define(NewWorkflow[testInput]("test.slow"),
Expand Down
Loading
Loading