From 2bc3265e63858c20cca9fe6b1ebb617d754f0cbb Mon Sep 17 00:00:00 2001 From: Jaakko Husso Date: Tue, 13 Oct 2026 11:04:18 +0000 Subject: [PATCH] perf(ai-builder): Trust a matching prebaked bundle manifest in the sandbox (no-changelog) (#40820) Co-authored-by: Claude Opus 5.5 --- packages/@n8n/instance-ai/docs/sandboxing.md | 7 + packages/@n8n/instance-ai/src/index.ts | 1 + .../materialize-knowledge-base.test.ts | 5 +- .../materialize-knowledge-base.ts | 5 + .../materialize-runtime-skills.test.ts | 36 ++++ .../src/skills/materialize-runtime-skills.ts | 21 +- .../prebaked-workspace-bundle.test.ts | 184 +++++++++++++++--- .../workspace/__tests__/sandbox-setup.test.ts | 48 +++++ .../workspace/prebaked-workspace-bundle.ts | 52 ++--- .../src/workspace/sandbox-setup.ts | 8 +- .../instance-ai-sandbox.service.test.ts | 23 +++ .../__tests__/instance-ai.service.test.ts | 11 ++ .../instance-ai/instance-ai.service.ts | 1 + .../sandbox/instance-ai-sandbox.service.ts | 8 +- 14 files changed, 351 insertions(+), 59 deletions(-) diff --git a/packages/@n8n/instance-ai/docs/sandboxing.md b/packages/@n8n/instance-ai/docs/sandboxing.md index c804085c39c..cd22889aa6f 100644 --- a/packages/@n8n/instance-ai/docs/sandboxing.md +++ b/packages/@n8n/instance-ai/docs/sandboxing.md @@ -128,6 +128,13 @@ and installed dependencies. The node-type catalog is written after sandbox creation because it is instance-specific and too large for the Daytona image request. +The knowledge base and runtime skills each have a manifest with a content +hash. Setup compares the manifest hash with the expected hash. When the +hashes do not match, setup writes the bundle again and writes the manifest +last. A matching manifest is trusted, and setup reads no other bundle file. +Each cached sandbox entry keeps the result, so later runs do not read the +manifest again. + When `N8N_INSTANCE_AI_SANDBOX_LINK_SDK` is enabled for local development, Instance AI packs and installs the local `@n8n/utils`, `@n8n/errors`, `n8n-workflow`, and `@n8n/workflow-sdk` packages. Build those packages before diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index 5a4a17de891..d8154d8e8a3 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -534,6 +534,7 @@ export const createSandbox: typeof CreateWorkspaceMod.createSandbox = lazyFuncti export const createWorkspace: typeof CreateWorkspaceMod.createWorkspace = lazyFunction( () => loadCreateWorkspace().createWorkspace, ); +export type { VerifiedWorkspaceBundles } from './workspace/prebaked-workspace-bundle'; export type SnapshotManager = SnapshotManagerMod.SnapshotManager; export const SnapshotManager: typeof SnapshotManagerMod.SnapshotManager = lazyClass( () => loadSnapshotManager().SnapshotManager, diff --git a/packages/@n8n/instance-ai/src/knowledge-base/__tests__/materialize-knowledge-base.test.ts b/packages/@n8n/instance-ai/src/knowledge-base/__tests__/materialize-knowledge-base.test.ts index 37e69e5f446..8a96a3cb13e 100644 --- a/packages/@n8n/instance-ai/src/knowledge-base/__tests__/materialize-knowledge-base.test.ts +++ b/packages/@n8n/instance-ai/src/knowledge-base/__tests__/materialize-knowledge-base.test.ts @@ -299,7 +299,7 @@ describe('materializeKnowledgeBaseIntoWorkspace', () => { expect(writes.size).toBe(0); }); - it('rematerializes when prebaked manifest exists but payload is incomplete', async () => { + it('trusts a matching prebaked manifest without checking each payload file', async () => { const bundle = await buildKnowledgeBaseWorkspaceBundle({ root: ROOT, logger: mockLogger }); const files = new Map([ [bundle.manifestPath, bundle.files.get(bundle.manifestPath) ?? ''], @@ -313,7 +313,6 @@ describe('materializeKnowledgeBaseIntoWorkspace', () => { }); expect(materialized.contentHash).toBe(bundle.contentHash); - expect(writes.has(bundle.indexPath)).toBe(true); - expect(writes.size).toBeGreaterThan(1); + expect(writes.size).toBe(0); }); }); diff --git a/packages/@n8n/instance-ai/src/knowledge-base/materialize-knowledge-base.ts b/packages/@n8n/instance-ai/src/knowledge-base/materialize-knowledge-base.ts index c4489f563a4..8aa5e345ce9 100644 --- a/packages/@n8n/instance-ai/src/knowledge-base/materialize-knowledge-base.ts +++ b/packages/@n8n/instance-ai/src/knowledge-base/materialize-knowledge-base.ts @@ -25,6 +25,7 @@ import { computeWorkspaceContentHash } from '../workspace/compute-workspace-cont import { loadPrebakedWorkspaceBundle, materializeWorkspaceBundle, + type VerifiedWorkspaceBundles, } from '../workspace/prebaked-workspace-bundle'; import type { SandboxWorkspace } from '../workspace/sandbox-fs'; import { stringifyWorkspaceJson, withTrailingNewline } from '../workspace/workspace-file-content'; @@ -100,6 +101,7 @@ export interface BuildKnowledgeBaseWorkspaceBundleOptions { interface MaterializeKnowledgeBaseOptions extends BuildKnowledgeBaseWorkspaceBundleOptions { workspace: SandboxWorkspace; + verifiedBundles?: VerifiedWorkspaceBundles; } function addTemplatesToKnowledgeBaseFiles( @@ -307,6 +309,7 @@ export async function loadPrebakedKnowledgeBaseBundle( return await loadPrebakedWorkspaceBundle({ workspace: options.workspace, + verifiedBundles: options.verifiedBundles, manifestPath: bundle.manifestPath, expectedHash: bundle.contentHash, hashField: 'contentHash', @@ -343,10 +346,12 @@ export async function materializeKnowledgeBaseIntoWorkspace( async () => { return await materializeWorkspaceBundle({ workspace: options.workspace, + verifiedBundles: options.verifiedBundles, resourceLabel: KNOWLEDGE_BASE_FILE_LABEL, logger: options.logger, loadPrebaked: async () => await loadPrebakedKnowledgeBaseBundle(options), buildBundle: async () => await buildKnowledgeBaseWorkspaceBundle(options), + bundleHash: (bundle) => bundle.contentHash, materializedLogMessage: 'Materialized knowledge base into workspace', materializedLogContext: (bundle) => ({ root: options.root, diff --git a/packages/@n8n/instance-ai/src/skills/__tests__/materialize-runtime-skills.test.ts b/packages/@n8n/instance-ai/src/skills/__tests__/materialize-runtime-skills.test.ts index 6ededbeee70..f9817d7aaf2 100644 --- a/packages/@n8n/instance-ai/src/skills/__tests__/materialize-runtime-skills.test.ts +++ b/packages/@n8n/instance-ai/src/skills/__tests__/materialize-runtime-skills.test.ts @@ -319,6 +319,42 @@ describe('materializeRuntimeSkillsIntoWorkspace', () => { expect(text).toContain(skillPath); }); + it('checks a prebaked bundle once per sandbox and reads only the manifest', async () => { + const source = loadInstanceAiRuntimeSkillSource(); + const { workspace, writes, readFile, writeFile } = createMockWorkspace(); + const root = '/home/daytona/workspace'; + const bundle = await buildRuntimeSkillWorkspaceBundle({ source, root, logger: mockLogger }); + if (!bundle) throw new Error('Expected runtime skill bundle'); + for (const [path, content] of bundle.files) { + writes.set(path, content); + } + const verifiedBundles = new Map(); + const runSkillLoad = async () => { + const runtimeSource = createLazyWorkspaceRuntimeSkillSource({ + logger: mockLogger, + source, + workspace, + resolveVerifiedBundles: async () => await Promise.resolve(verifiedBundles), + }); + return await createSkillLoadTool(runtimeSource).handler?.( + { skillId: 'data-table-manager' }, + {}, + ); + }; + + await runSkillLoad(); + + expect(readFile).toHaveBeenCalledTimes(1); + expect(readFile).toHaveBeenCalledWith(bundle.manifestPath, expect.anything()); + + readFile.mockClear(); + const result = await runSkillLoad(); + + expect(readFile).not.toHaveBeenCalled(); + expect(writeFile).not.toHaveBeenCalled(); + expect(skillLoadText(result)).toContain('[Skill: "data-table-manager"]'); + }); + it('falls back to live materialization when the prebaked manifest is stale', async () => { const source = loadInstanceAiRuntimeSkillSource(); const { workspace, writes, writeFile } = createMockWorkspace(); diff --git a/packages/@n8n/instance-ai/src/skills/materialize-runtime-skills.ts b/packages/@n8n/instance-ai/src/skills/materialize-runtime-skills.ts index 8b8673f82fe..775f4e15b2f 100644 --- a/packages/@n8n/instance-ai/src/skills/materialize-runtime-skills.ts +++ b/packages/@n8n/instance-ai/src/skills/materialize-runtime-skills.ts @@ -20,6 +20,7 @@ import { traceSandboxOperation } from '../tracing/sandbox-tracing'; import { loadPrebakedWorkspaceBundle, materializeWorkspaceBundle, + type VerifiedWorkspaceBundles, } from '../workspace/prebaked-workspace-bundle'; import { stringifyWorkspaceJson, withTrailingNewline } from '../workspace/workspace-file-content'; import { WORKSPACE_MANIFEST_FILE } from '../workspace/workspace-manifest'; @@ -70,6 +71,7 @@ interface BuildRuntimeSkillWorkspaceBundleOptions { interface MaterializeRuntimeSkillsOptions { source: RuntimeSkillSource; workspace: Workspace; + verifiedBundles?: VerifiedWorkspaceBundles; root: string; workspaceRoot?: string; logger: Logger; @@ -78,6 +80,8 @@ interface MaterializeRuntimeSkillsOptions { interface LazyWorkspaceRuntimeSkillSourceOptions { source: RuntimeSkillSource; workspace: Workspace | undefined; + /** Resolves the verified bundles of the sandbox behind `workspace`. */ + resolveVerifiedBundles?: () => Promise; logger: Logger; } @@ -387,6 +391,7 @@ const RUNTIME_SKILL_FILE_LABEL = 'Runtime skill file'; export async function loadPrebakedRuntimeSkillsBundle({ source, workspace, + verifiedBundles, root, workspaceRoot = root, logger, @@ -398,6 +403,7 @@ export async function loadPrebakedRuntimeSkillsBundle({ return await loadPrebakedWorkspaceBundle({ workspace, + verifiedBundles, manifestPath, expectedHash: source.registry.skillsHash, hashField: 'skillsHash', @@ -592,6 +598,7 @@ export async function buildRuntimeSkillWorkspaceBundle({ export async function materializeRuntimeSkillsIntoWorkspace({ source, workspace, + verifiedBundles, root, logger, }: MaterializeRuntimeSkillsOptions): Promise { @@ -608,10 +615,17 @@ export async function materializeRuntimeSkillsIntoWorkspace({ return await materializeWorkspaceBundle({ workspace, + verifiedBundles, resourceLabel: RUNTIME_SKILL_FILE_LABEL, logger, loadPrebaked: async () => - await loadPrebakedRuntimeSkillsBundle({ source, workspace, root, logger }), + await loadPrebakedRuntimeSkillsBundle({ + source, + workspace, + verifiedBundles, + root, + logger, + }), buildBundle: async () => { const bundle = await buildRuntimeSkillWorkspaceBundle({ source, root, logger }); if (!bundle) { @@ -619,6 +633,7 @@ export async function materializeRuntimeSkillsIntoWorkspace({ } return bundle; }, + bundleHash: (bundle) => bundle.skillsHash, materializedLogMessage: 'Materialized runtime skills into workspace', materializedLogContext: (bundle) => ({ root, @@ -635,6 +650,7 @@ export async function materializeRuntimeSkillsIntoWorkspace({ export function createLazyWorkspaceRuntimeSkillSource({ source, workspace, + resolveVerifiedBundles, logger, }: LazyWorkspaceRuntimeSkillSourceOptions): RuntimeSkillSource { if (!workspace || source.registry.skills.length === 0) return source; @@ -664,7 +680,8 @@ export function createLazyWorkspaceRuntimeSkillSource({ materializePromise ??= (async () => { const root = await getWorkspaceRoot(runtimeWorkspace); - const options = { source, workspace: runtimeWorkspace, root, logger }; + const verifiedBundles = await resolveVerifiedBundles?.(); + const options = { source, workspace: runtimeWorkspace, verifiedBundles, root, logger }; const result = await materializeRuntimeSkillsIntoWorkspace(options); if (result) { materialized = result; diff --git a/packages/@n8n/instance-ai/src/workspace/__tests__/prebaked-workspace-bundle.test.ts b/packages/@n8n/instance-ai/src/workspace/__tests__/prebaked-workspace-bundle.test.ts index 2613c3af0b6..ffee1101d83 100644 --- a/packages/@n8n/instance-ai/src/workspace/__tests__/prebaked-workspace-bundle.test.ts +++ b/packages/@n8n/instance-ai/src/workspace/__tests__/prebaked-workspace-bundle.test.ts @@ -1,6 +1,7 @@ import { loadPrebakedWorkspaceBundle, materializeWorkspaceBundle, + type VerifiedWorkspaceBundles, } from '../prebaked-workspace-bundle'; import type { SandboxWorkspace } from '../sandbox-fs'; import { stringifyWorkspaceJson } from '../workspace-file-content'; @@ -9,38 +10,35 @@ const ROOT = '/home/daytona/workspace'; const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() } as never; -function createSandboxWorkspace(files: Map): { - workspace: SandboxWorkspace; - writes: Map; -} { +function createSandboxWorkspace(files: Map) { const writes = new Map(); + const writeFile = vi.fn(async (path: string, content: string | Buffer) => { + writes.set(path, Buffer.isBuffer(content) ? content.toString('utf-8') : content); + await Promise.resolve(); + }); + const executeCommand = vi.fn(async (command: string) => { + const readMatch = /^cat '([^']+)' 2>\/dev\/null$/.exec(command); + if (readMatch) { + const content = files.get(readMatch[1]); + return await Promise.resolve( + content === undefined + ? { exitCode: 1, stdout: '', stderr: 'missing' } + : { exitCode: 0, stdout: content, stderr: '' }, + ); + } + + return await Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); + }); const workspace: SandboxWorkspace = { filesystem: { provider: 'local', - writeFile: vi.fn(async (path: string, content: string | Buffer) => { - writes.set(path, Buffer.isBuffer(content) ? content.toString('utf-8') : content); - await Promise.resolve(); - }), + writeFile, mkdir: vi.fn(async () => await Promise.resolve()), }, - sandbox: { - executeCommand: vi.fn(async (command: string) => { - const readMatch = /^cat '([^']+)' 2>\/dev\/null$/.exec(command); - if (readMatch) { - const content = files.get(readMatch[1]); - return await Promise.resolve( - content === undefined - ? { exitCode: 1, stdout: '', stderr: 'missing' } - : { exitCode: 0, stdout: content, stderr: '' }, - ); - } - - return await Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); - }), - }, + sandbox: { executeCommand }, }; - return { workspace, writes }; + return { workspace, writes, writeFile, executeCommand }; } describe('loadPrebakedWorkspaceBundle', () => { @@ -106,8 +104,8 @@ describe('loadPrebakedWorkspaceBundle', () => { expect(result).toBeUndefined(); }); - it('returns undefined when the manifest hash matches but a payload file is missing', async () => { - const { workspace } = createSandboxWorkspace( + it('trusts a matching manifest without a read of each payload file', async () => { + const { workspace, executeCommand } = createSandboxWorkspace( new Map([ [ manifestPath, @@ -132,7 +130,8 @@ describe('loadPrebakedWorkspaceBundle', () => { buildBundle: () => bundle, }); - expect(result).toBeUndefined(); + expect(result).toBe(bundle); + expect(executeCommand).toHaveBeenCalledTimes(1); }); }); @@ -158,6 +157,7 @@ describe('materializeWorkspaceBundle', () => { resourceLabel: 'Test bundle file', loadPrebaked: async () => await Promise.resolve(undefined), buildBundle: () => bundle, + bundleHash: (built) => built.contentHash, materializedLogMessage: 'materialized', materializedLogContext: () => ({ root: ROOT }), }); @@ -187,6 +187,7 @@ describe('materializeWorkspaceBundle', () => { resourceLabel: 'Test bundle file', loadPrebaked: async () => await Promise.resolve(undefined), buildBundle: () => bundle, + bundleHash: (built) => built.contentHash, materializedLogMessage: 'materialized', materializedLogContext: () => ({ root: ROOT }), }); @@ -223,6 +224,7 @@ describe('materializeWorkspaceBundle', () => { buildBundle: () => bundle, }), buildBundle: () => bundle, + bundleHash: (built) => built.contentHash, materializedLogMessage: 'materialized', materializedLogContext: () => ({ root: ROOT }), }); @@ -231,3 +233,131 @@ describe('materializeWorkspaceBundle', () => { expect(writes.size).toBe(0); }); }); + +describe('prebaked bundle checks with verified bundles', () => { + const manifestPath = `${ROOT}/bundle/.manifest.json`; + const filePaths = [`${ROOT}/bundle/a.txt`, `${ROOT}/bundle/b.txt`]; + const manifest = stringifyWorkspaceJson({ schemaVersion: 1, contentHash: 'abc123' }); + const bundle = { + rootDir: `${ROOT}/bundle`, + manifestPath, + files: new Map([ + ...filePaths.map((path): [string, string] => [path, 'content\n']), + [manifestPath, manifest], + ]), + contentHash: 'abc123', + }; + + function readPaths(executeCommand: ReturnType['executeCommand']) { + return executeCommand.mock.calls.flatMap(([command]) => { + const match = /^cat '([^']+)' 2>\/dev\/null$/.exec(command); + return match ? [match[1]] : []; + }); + } + + async function materialize( + workspace: SandboxWorkspace, + verifiedBundles: VerifiedWorkspaceBundles, + ) { + return await materializeWorkspaceBundle({ + logger: mockLogger, + workspace, + verifiedBundles, + resourceLabel: 'Test bundle file', + loadPrebaked: async () => + await loadPrebakedWorkspaceBundle({ + logger: mockLogger, + workspace, + verifiedBundles, + manifestPath, + expectedHash: bundle.contentHash, + hashField: 'contentHash', + schemaVersion: 1, + resourceLabel: 'Test bundle file', + invalidManifestLogMessage: 'invalid', + staleManifestLogMessage: 'stale', + staleManifestLogKeys: { expected: 'expectedHash', actual: 'actualHash' }, + successLogMessage: 'success', + successLogContext: () => ({ root: ROOT }), + buildBundle: () => bundle, + }), + buildBundle: () => bundle, + bundleHash: (built) => built.contentHash, + materializedLogMessage: 'materialized', + materializedLogContext: () => ({ root: ROOT }), + }); + } + + it('reads only the manifest when the manifest hash matches', async () => { + const { workspace, writes, executeCommand } = createSandboxWorkspace(new Map(bundle.files)); + const verifiedBundles: VerifiedWorkspaceBundles = new Map(); + + await expect(materialize(workspace, verifiedBundles)).resolves.toBe(bundle); + + expect(readPaths(executeCommand)).toEqual([manifestPath]); + expect(writes.size).toBe(0); + expect(verifiedBundles.get(manifestPath)).toBe(bundle.contentHash); + }); + + it('makes no sandbox reads after the bundle is verified for the sandbox', async () => { + const { workspace, executeCommand } = createSandboxWorkspace(new Map(bundle.files)); + const verifiedBundles: VerifiedWorkspaceBundles = new Map(); + await materialize(workspace, verifiedBundles); + executeCommand.mockClear(); + + await expect(materialize(workspace, verifiedBundles)).resolves.toBe(bundle); + + expect(readPaths(executeCommand)).toEqual([]); + }); + + it('checks the sandbox again when the expected hash changes', async () => { + const { workspace, writes, executeCommand } = createSandboxWorkspace(new Map(bundle.files)); + const verifiedBundles: VerifiedWorkspaceBundles = new Map([[manifestPath, 'older-hash']]); + + await materialize(workspace, verifiedBundles); + + expect(readPaths(executeCommand)).toEqual([manifestPath]); + expect(writes.size).toBe(0); + expect(verifiedBundles.get(manifestPath)).toBe(bundle.contentHash); + }); + + it('writes the bundle again when the manifest is stale', async () => { + const { workspace, writes } = createSandboxWorkspace( + new Map([ + ...filePaths.map((path): [string, string] => [path, 'old\n']), + [manifestPath, stringifyWorkspaceJson({ schemaVersion: 1, contentHash: 'stale' })], + ]), + ); + const verifiedBundles: VerifiedWorkspaceBundles = new Map(); + + await materialize(workspace, verifiedBundles); + + expect([...writes.keys()].sort()).toEqual([manifestPath, ...filePaths].sort()); + expect(verifiedBundles.get(manifestPath)).toBe(bundle.contentHash); + }); + + it('writes the bundle again when the manifest is missing', async () => { + const { workspace, writes } = createSandboxWorkspace(new Map()); + const verifiedBundles: VerifiedWorkspaceBundles = new Map(); + + await materialize(workspace, verifiedBundles); + + expect([...writes.keys()].sort()).toEqual([manifestPath, ...filePaths].sort()); + expect(verifiedBundles.get(manifestPath)).toBe(bundle.contentHash); + }); + + it('does not record the bundle when a write fails', async () => { + const { workspace, writeFile, executeCommand } = createSandboxWorkspace(new Map()); + writeFile.mockRejectedValue(new Error('disk full')); + executeCommand.mockResolvedValue({ + exitCode: 1, + stdout: '', + stderr: 'disk full', + }); + const verifiedBundles: VerifiedWorkspaceBundles = new Map(); + + await expect(materialize(workspace, verifiedBundles)).rejects.toThrow(); + + expect(verifiedBundles.size).toBe(0); + }); +}); diff --git a/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts b/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts index e0ad25e17bf..08eaa19ff09 100644 --- a/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts +++ b/packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts @@ -67,6 +67,7 @@ import { jsonParse } from 'n8n-workflow'; import type { Mock } from 'vitest'; import { makeBuilderTemplatesTarGz } from '../../knowledge-base/__tests__/builder-templates-archive.fixtures'; +import { buildKnowledgeBaseWorkspaceBundle } from '../../knowledge-base/materialize-knowledge-base'; import type { InstanceAiContext, SearchableNodeDescription } from '../../types'; import type { BuilderTemplatesBundle } from '../builder-templates-service'; import type { SandboxWorkspace } from '../sandbox-fs'; @@ -402,6 +403,53 @@ describe('setupSandboxWorkspace', () => { expect(writtenPaths.some((p) => p.includes('/knowledge-base/templates/'))).toBe(true); }); + it('reads only the knowledge base manifest when the prebaked bundle matches', async () => { + const runInSandbox: RunInSandboxMock = + vi.fn< + ( + ...args: [SandboxWorkspace, string, string?] + ) => Promise<{ exitCode: number; stdout: string; stderr: string }> + >(); + runInSandbox.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + const readFileViaSandbox: ReadFileViaSandboxMock = + vi.fn<(...args: [SandboxWorkspace, string]) => Promise>(); + readFileViaSandbox.mockResolvedValue(null); + const setupSandboxWorkspace = loadSetupSandboxWorkspaceWithFsMocks( + runInSandbox, + readFileViaSandbox, + ); + const context = createSetupContext(); + const knowledgeBase = await buildKnowledgeBaseWorkspaceBundle({ + root: '/sandbox', + logger: context.logger, + }); + const files = new Map(knowledgeBase.files); + files.set('/sandbox/.sandbox-initialized', '2024-01-01T00:00:00.000Z'); + const writeFile = vi.fn< + (...args: [string, string | Buffer, { recursive?: boolean }?]) => Promise + >(async () => await Promise.resolve()); + const readFile = vi.fn(async (path: string) => { + const content = files.get(path); + if (content === undefined) return await Promise.reject(new Error(`ENOENT: ${path}`)); + return await Promise.resolve(content); + }); + const verifiedBundles = new Map(); + + const initialized = await setupSandboxWorkspace( + createLocalWorkspace(writeFile, undefined, readFile), + context, + { verifiedBundles }, + ); + + expect(initialized).toBe(false); + expect(readFile.mock.calls.map(([path]) => path)).toEqual([ + '/sandbox/.sandbox-initialized', + knowledgeBase.manifestPath, + ]); + expect(writeFile).not.toHaveBeenCalled(); + expect(verifiedBundles.get(knowledgeBase.manifestPath)).toBe(knowledgeBase.contentHash); + }); + it('materializes knowledge-base templates on the local provider when a bundle is available', async () => { const runInSandbox: RunInSandboxMock = vi.fn< diff --git a/packages/@n8n/instance-ai/src/workspace/prebaked-workspace-bundle.ts b/packages/@n8n/instance-ai/src/workspace/prebaked-workspace-bundle.ts index 1450918d488..d6173930661 100644 --- a/packages/@n8n/instance-ai/src/workspace/prebaked-workspace-bundle.ts +++ b/packages/@n8n/instance-ai/src/workspace/prebaked-workspace-bundle.ts @@ -8,8 +8,16 @@ import { import { parseVersionedWorkspaceManifest } from './workspace-manifest'; import { traceSandboxOperation, sandboxFileBytes } from '../tracing/sandbox-tracing'; +/** + * Bundle hash by manifest path, for bundles that one sandbox already has. + * Keep it as long as the sandbox, so that repeated checks in that sandbox do + * not read the manifest again. + */ +export type VerifiedWorkspaceBundles = Map; + export interface LoadPrebakedWorkspaceBundleOptions { workspace: WorkspaceFileTarget; + verifiedBundles?: VerifiedWorkspaceBundles; manifestPath: string; expectedHash: string; hashField: string; @@ -27,12 +35,18 @@ export interface LoadPrebakedWorkspaceBundleOptions { buildBundle: () => Promise | TBundle | undefined; } -/** Reuse a workspace bundle when its manifest hash matches the expected value. */ +/** + * Reuse a workspace bundle when its manifest hash matches the expected value. + * A matching manifest is trusted without a read of each bundle file: images + * and snapshots bake the files with the manifest, and a live write puts the + * manifest last. + */ export async function loadPrebakedWorkspaceBundle }>( options: LoadPrebakedWorkspaceBundleOptions, ): Promise { let decision = 'reused'; let actualHash: string | undefined; + const { verifiedBundles } = options; return await traceSandboxOperation( 'check-bundle', { @@ -43,6 +57,14 @@ export async function loadPrebakedWorkspaceBundle { + if (verifiedBundles?.get(options.manifestPath) === options.expectedHash) { + decision = 'cached'; + actualHash = options.expectedHash; + const bundle = await options.buildBundle(); + if (!bundle) decision = 'empty'; + return bundle; + } + const manifestRaw = await readWorkspaceFile(options.workspace, options.manifestPath, { logger: options.logger, resourceLabel: options.resourceLabel, @@ -81,29 +103,7 @@ export async function loadPrebakedWorkspaceBundle path !== options.manifestPath, - ); - const existenceChecks = await Promise.all( - payloadPaths.map(async (path) => ({ - path, - exists: - (await readWorkspaceFile(options.workspace, path, { - logger: options.logger, - resourceLabel: options.resourceLabel, - })) !== null, - })), - ); - const missingPath = existenceChecks.find((check) => !check.exists)?.path; - if (missingPath) { - decision = 'incomplete'; - options.logger.debug('Ignoring incomplete prebaked workspace bundle', { - manifestPath: options.manifestPath, - missingPath, - }); - return undefined; - } - + verifiedBundles?.set(options.manifestPath, options.expectedHash); options.logger.debug(options.successLogMessage, options.successLogContext(bundle)); return bundle; }, @@ -114,10 +114,13 @@ export interface MaterializeWorkspaceBundleOptions< TBundle extends { files: Map; manifestPath: string }, > { workspace: WorkspaceFileTarget; + verifiedBundles?: VerifiedWorkspaceBundles; resourceLabel: string; logger: Logger; loadPrebaked: () => Promise; buildBundle: () => Promise | TBundle; + /** Hash that the bundle manifest records. */ + bundleHash: (bundle: TBundle) => string; materializedLogMessage: string; materializedLogContext: (bundle: TBundle) => Record; } @@ -168,6 +171,7 @@ export async function materializeWorkspaceBundle< }); } + options.verifiedBundles?.set(bundle.manifestPath, options.bundleHash(bundle)); options.logger.debug(options.materializedLogMessage, options.materializedLogContext(bundle)); return bundle; }, diff --git a/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts b/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts index 9d37a93b559..734afe9bf25 100644 --- a/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts +++ b/packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts @@ -39,6 +39,7 @@ import { packSandboxLinkedWorkspacePackages, type WorkspacePackageTarball, } from './pack-workspace-sdk'; +import type { VerifiedWorkspaceBundles } from './prebaked-workspace-bundle'; import { runInSandbox, readFileViaSandbox, @@ -471,11 +472,13 @@ async function materializeKnowledgeBaseStep( workspace: SandboxWorkspace, root: string, context: InstanceAiContext, + verifiedBundles: VerifiedWorkspaceBundles | undefined, ): Promise { await setupStep('materialize-knowledge-base', async () => { const templatesBundle = (await context.templatesService?.getBundle()) ?? null; await materializeKnowledgeBaseIntoWorkspace({ workspace, + verifiedBundles, root, logger: context.logger, templatesArchive: templatesBundle?.archive ?? null, @@ -494,6 +497,7 @@ async function materializeKnowledgeBaseStep( export async function setupSandboxWorkspace( workspace: SandboxWorkspace, context: InstanceAiContext, + options: { verifiedBundles?: VerifiedWorkspaceBundles } = {}, ): Promise { return await traceSandboxOperation( 'initialize-workspace', @@ -511,7 +515,7 @@ export async function setupSandboxWorkspace( async () => await readWorkspaceFile(workspace, markerFile), ); if (marker !== null) { - await materializeKnowledgeBaseStep(workspace, root, context); + await materializeKnowledgeBaseStep(workspace, root, context, options.verifiedBundles); return false; } @@ -538,7 +542,7 @@ export async function setupSandboxWorkspace( 'write-workspace-files', async () => await writeWorkspaceFiles(workspace, root, files), ); - await materializeKnowledgeBaseStep(workspace, root, context); + await materializeKnowledgeBaseStep(workspace, root, context, options.verifiedBundles); // npm install (must run after package.json is in place) await setupStep('install-dependencies', async () => { diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts index d2b1df266ec..02ae424676b 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts @@ -529,6 +529,29 @@ describe('InstanceAiSandboxService', () => { expect(setupSandboxWorkspace).toHaveBeenCalledTimes(1); }); + it('passes the verified bundles of the sandbox to workspace setup', async () => { + const { service } = createSandboxService({ + config: { sandboxEnabled: true, sandboxProvider: 'daytona' }, + }); + const workspace = { init: vi.fn(async () => {}), destroy: vi.fn(async () => {}) }; + (createSandbox as Mock).mockResolvedValue({ id: 'sandbox-1' }); + (createWorkspace as Mock).mockReturnValue(workspace); + (setupSandboxWorkspace as Mock).mockResolvedValue(undefined); + + const entry = await service.getOrCreateWorkspace( + 'thread-1', + fakeUser, + {} as InstanceAiContext, + ); + + expect(entry?.verifiedBundles).toEqual(new Map()); + const setupOptions = (setupSandboxWorkspace as Mock).mock.calls[0]?.[2] as { + verifiedBundles: unknown; + }; + expect((setupSandboxWorkspace as Mock).mock.calls[0]?.[0]).toBe(workspace); + expect(setupOptions.verifiedBundles).toBe(entry?.verifiedBundles); + }); + it('assigns a deterministic thread-scoped UUID for the n8n-sandbox provider', async () => { const n8nSandboxConfig: Overrides['config'] = { sandboxEnabled: true, diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts index 84ba2650d32..e242d337192 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts @@ -1175,6 +1175,17 @@ describe('InstanceAiService — runtime workspace setup', () => { expect(workspace.init).toHaveBeenCalledTimes(1); expect(setupSandboxWorkspace).toHaveBeenCalledTimes(1); + // Runtime skills and workspace setup share the verified bundles of the sandbox. + const { resolveVerifiedBundles } = (createLazyWorkspaceRuntimeSkillSource as Mock).mock + .calls[0]?.[0] as { resolveVerifiedBundles: () => Promise }; + const verifiedBundles = await resolveVerifiedBundles(); + expect(verifiedBundles).toEqual(new Map()); + const setupOptions = (setupSandboxWorkspace as Mock).mock.calls[0]?.[2] as { + verifiedBundles: unknown; + }; + expect((setupSandboxWorkspace as Mock).mock.calls[0]?.[0]).toBe(workspace); + expect(setupOptions.verifiedBundles).toBe(verifiedBundles); + (createLazyRuntimeWorkspace as Mock).mockClear(); (createLazyWorkspaceRuntimeSkillSource as Mock).mockClear(); (createSandbox as Mock).mockClear(); diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index 33f40aa2cdc..c291b9e0a73 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -2919,6 +2919,7 @@ export class InstanceAiService { runtimeSkills = createLazyWorkspaceRuntimeSkillSource({ source: allRuntimeSkills, workspace: runtimeSkillWorkspace, + resolveVerifiedBundles: async () => (await getSandboxEntry())?.verifiedBundles, logger: this.logger, }); } diff --git a/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts b/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts index 68c6d493b3c..cfa7c891b87 100644 --- a/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts +++ b/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts @@ -14,6 +14,7 @@ import { type Logger, type ManagedBackgroundTask, type SandboxConfig, + type VerifiedWorkspaceBundles, } from '@n8n/instance-ai'; import type { ErrorReporter } from 'n8n-core'; import { OperationalError, UnexpectedError } from 'n8n-workflow'; @@ -36,6 +37,8 @@ export type RuntimeSandboxEntry = { sandbox: NonNullable>>; workspace: NonNullable>; configFingerprint: string; + /** Prebaked bundles already verified in this sandbox, shared across runs. */ + verifiedBundles: VerifiedWorkspaceBundles; setupComplete: boolean; setupPromise: Promise | undefined; expiresAt: number; @@ -419,7 +422,9 @@ export class InstanceAiSandboxService { ): Promise { if (entry.setupComplete) return; - entry.setupPromise ??= setupSandboxWorkspace(entry.workspace, context) + entry.setupPromise ??= setupSandboxWorkspace(entry.workspace, context, { + verifiedBundles: entry.verifiedBundles, + }) .then(() => { entry.setupComplete = true; }) @@ -481,6 +486,7 @@ export class InstanceAiSandboxService { sandbox, workspace, configFingerprint: cacheState.fingerprint, + verifiedBundles: new Map(), setupComplete: false, setupPromise: undefined, expiresAt: this.nextSandboxExpiry(),