From 9f5a71e137126bc661f43db8aa7221a6c620d6b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Repe=C4=87?= Date: Sun, 4 Oct 2026 11:50:16 +0200 Subject: [PATCH] fix(core): Suspend jobs that register after worker shutdown asked for suspension --- .../__tests__/job-processor.service.test.ts | 44 +++++++++++++++++++ packages/cli/src/scaling/job-processor.ts | 7 +++ 2 files changed, 51 insertions(+) diff --git a/packages/cli/src/scaling/__tests__/job-processor.service.test.ts b/packages/cli/src/scaling/__tests__/job-processor.service.test.ts index b566c567ab6..6f21446afdc 100644 --- a/packages/cli/src/scaling/__tests__/job-processor.service.test.ts +++ b/packages/cli/src/scaling/__tests__/job-processor.service.test.ts @@ -327,6 +327,50 @@ describe('JobProcessor', () => { await processPromise; }); + it('should suspend a suspendable job that registers after suspension was requested', async () => { + const executionPersistence = mock(); + let resolveExecution!: (execution: IExecutionResponse) => void; + executionPersistence.findSingleExecution.mockReturnValue( + new Promise((r) => (resolveExecution = r)), + ); + vi.spyOn(WorkflowExecuteAdditionalData, 'getBase').mockResolvedValue( + mock(), + ); + const jobProcessor = createJobProcessor(executionPersistence); + + let resolveRun!: (run: IRun) => void; + processRunExecutionDataMock.mockReturnValue(new Promise((r) => (resolveRun = r))); + + const job = mock({ + id: 'job-1', + data: { + executionId: 'exec-1', + loadStaticData: false, + streamingEnabled: false, + isMcpExecution: false, + }, + }); + + const processPromise = jobProcessor.processJob(job); + + // Shutdown arrives while the job is still in its preflight reads. + jobProcessor.suspendRunningJobs(); + expect(workflowExecuteSuspendMock).not.toHaveBeenCalled(); + + resolveExecution( + mock({ + mode: 'webhook', + workflowData: { nodes: [], staticData: {} }, + data: mock(), + }), + ); + await vi.waitFor(() => expect(jobProcessor.getRunningJobIds()).toEqual(['job-1'])); + expect(workflowExecuteSuspendMock).toHaveBeenCalledTimes(1); + + resolveRun(successRun()); + await processPromise; + }); + it('should not attach a suspend handle to a non-suspendable job', async () => { const executionPersistence = mock(); executionPersistence.findSingleExecution.mockResolvedValue( diff --git a/packages/cli/src/scaling/job-processor.ts b/packages/cli/src/scaling/job-processor.ts index 84a0bcedec5..9871e3fb916 100644 --- a/packages/cli/src/scaling/job-processor.ts +++ b/packages/cli/src/scaling/job-processor.ts @@ -369,6 +369,9 @@ export class JobProcessor { if (workflowExecute && this.isJobSuspendable(job, execution)) { const suspendable = workflowExecute; runningJob.suspend = () => suspendable.suspend(); + // A job that was still in its preflight reads when shutdown asked the + // running jobs to suspend would otherwise run to completion unasked. + if (this.suspensionRequested) runningJob.suspend(); } this.runningJobs[job.id] = runningJob; @@ -589,6 +592,9 @@ export class JobProcessor { * streaming responses cannot migrate mid-stream, and MCP executions are * pinned to their session. */ + /** Set once at shutdown; jobs that register afterwards suspend right away. */ + private suspensionRequested = false; + private isJobSuspendable(job: Job, execution: IExecutionResponse): boolean { const isProductionMode = ['webhook', 'trigger', 'retry'].includes(execution.mode); @@ -602,6 +608,7 @@ export class JobProcessor { /** Ask every suspendable running job to stop at its next node boundary. */ suspendRunningJobs() { + this.suspensionRequested = true; const executionIds: string[] = []; for (const runningJob of Object.values(this.runningJobs)) {