mirror of
https://github.com/n8n-io/n8n.git
synced 2026-06-05 02:59:27 +02:00
feat(core): Add templates to knowledge base (no-changelog) (#31642)
This commit is contained in:
parent
ab849d3fa8
commit
08180344ea
|
|
@ -181,7 +181,8 @@ The agent package — framework-agnostic business logic.
|
|||
- **Agent factory** (`agent/`) — creates orchestrator instances with tools, memory, MCP, and tool search
|
||||
- **Sub-agent factory** (`agent/`) — creates stateless sub-agents with mandatory protocol and tool subsets
|
||||
- **Orchestration tools** (`tools/orchestration/`) — `plan`, `delegate`, `update-tasks`, `cancel-background-task`, `correct-background-task`, `verify-built-workflow`, `report-verification-verdict`, `apply-workflow-credentials`
|
||||
- **Domain tools** (`tools/`) — native tools across workflows, executions, credentials, nodes, data tables, workspace, web research, filesystem, templates, and best practices
|
||||
- **Domain tools** (`tools/`) — native tools across workflows, executions, credentials, nodes, data tables, workspace, and web research
|
||||
- **Knowledge base** (`knowledge-base/`, `workspace/`) — best-practices guides and curated templates materialized in the builder sandbox for workspace tools to read
|
||||
- **Runtime** (`runtime/`) — stream execution engine, resumable streams with HITL suspension, background task manager, run state registry
|
||||
- **Planned tasks** (`planned-tasks/`) — task graph coordination, dependency resolution, scheduled execution
|
||||
- **Workflow loop** (`workflow-loop/`) — deterministic build→verify→debug state machine for workflow builder agents
|
||||
|
|
|
|||
|
|
@ -637,12 +637,23 @@ See `docs/filesystem-access.md`.
|
|||
|
||||
---
|
||||
|
||||
## Template Tools (2)
|
||||
## Knowledge Base (sandbox workspace)
|
||||
|
||||
| Tool | Description |
|
||||
Best-practices guides and curated workflow templates are materialized under
|
||||
`<workspace_root>/knowledge-base/` when a builder sandbox is available. Agents
|
||||
read them with workspace tools — there is no dedicated `get-best-practices` or
|
||||
template-search tool.
|
||||
|
||||
| Path | Description |
|
||||
|------|-------------|
|
||||
| `search-template-structures` | Search workflow templates by structure pattern |
|
||||
| `search-template-parameters` | Search templates by parameter values |
|
||||
| `knowledge-base/index.json` | Combined catalog of technique guides and curated templates |
|
||||
| `knowledge-base/best-practices/index.json` | Catalog of workflow technique guides |
|
||||
| `knowledge-base/best-practices/*.md` | Best-practices documentation per technique |
|
||||
| `knowledge-base/templates/index.json` | Catalog of curated SDK workflow examples |
|
||||
| `knowledge-base/templates/*.ts` | Template workflow source files |
|
||||
|
||||
Use `workspace_read_file` and `workspace_grep` (or shell equivalents in the
|
||||
sandbox) to consult these before planning or building non-trivial workflows.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -651,7 +662,6 @@ See `docs/filesystem-access.md`.
|
|||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `ask-user` | Suspend and request user input (single/multi-select or text) |
|
||||
| `get-best-practices` | Get workflow building best practices for common patterns |
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -671,7 +681,7 @@ everything; sub-agents receive only what they need.
|
|||
| Workspace tools | ✅ | ✅ (via delegate) | ❌ |
|
||||
| Filesystem tools | ✅ (conditional) | ✅ (via delegate) | ❌ |
|
||||
| Web research tools | ✅ | ✅ (via delegate) | ❌ |
|
||||
| Template / best practices | ✅ | ✅ (via delegate) | ✅ (builder) |
|
||||
| Knowledge base (best practices & templates via workspace) | ✅ | ✅ (via delegate) | ✅ (builder) |
|
||||
| Sandbox tools (`submit-workflow`, `materialize-node-type`, `write-sandbox-file`) | ❌ | ❌ | ✅ (builder only) |
|
||||
| MCP tools | ✅ | ❌ | ❌ |
|
||||
| Computer Use browser tools | ✅ (direct, via credential skill when setting up credentials) | ❌ | ❌ |
|
||||
|
|
|
|||
|
|
@ -110,6 +110,8 @@ describe('getSystemPrompt', () => {
|
|||
expect(prompt).toContain('call `plan` immediately');
|
||||
expect(prompt).toContain('Do not load the `workflow-builder` skill');
|
||||
expect(prompt).toContain('workflow tasks include any data table names');
|
||||
expect(prompt).toContain('The planner sub-agent discovers credentials and data tables');
|
||||
expect(prompt).not.toContain('discovers credentials, data tables, and best practices');
|
||||
});
|
||||
|
||||
it('routes standalone data-table work through direct tools and the skill', () => {
|
||||
|
|
@ -279,18 +281,37 @@ describe('getSystemPrompt', () => {
|
|||
});
|
||||
|
||||
describe('sandbox workspace', () => {
|
||||
it('includes sandbox workspace guidance when sandboxWorkspaceAvailable is true', () => {
|
||||
const prompt = getSystemPrompt({ sandboxWorkspaceAvailable: true });
|
||||
|
||||
expect(prompt).toContain('## Sandbox workspace');
|
||||
expect(prompt).toContain('knowledge-base/best-practices/index.json');
|
||||
expect(prompt).toContain('workspace_execute_command');
|
||||
});
|
||||
|
||||
it('omits sandbox workspace guidance when sandboxWorkspaceAvailable is false', () => {
|
||||
const prompt = getSystemPrompt({ sandboxWorkspaceAvailable: false });
|
||||
it('omits sandbox workspace guidance when no runtime workspace is attached', () => {
|
||||
const prompt = getSystemPrompt({});
|
||||
|
||||
expect(prompt).not.toContain('## Sandbox workspace');
|
||||
expect(prompt).not.toContain('workspace_read_file');
|
||||
expect(prompt).not.toContain('Consult the knowledge base before planning or building');
|
||||
});
|
||||
|
||||
it('includes sandbox workspace and knowledge-base guidance when workspaceRoot is provided', () => {
|
||||
const prompt = getSystemPrompt({
|
||||
workspaceRoot: '/home/daytona/workspace',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('## Sandbox workspace');
|
||||
expect(prompt).toContain('knowledge-base/index.json');
|
||||
expect(prompt).toContain('knowledge-base/best-practices/index.json');
|
||||
expect(prompt).toContain('knowledge-base/templates/index.json');
|
||||
expect(prompt).not.toContain('knowledge-base/templates/index.txt');
|
||||
expect(prompt).toContain('workspace_execute_command');
|
||||
expect(prompt).toContain('Consult the knowledge base before planning or building');
|
||||
expect(prompt).not.toContain('knowledge-base/best-practices/*.md');
|
||||
});
|
||||
|
||||
it('includes the resolved workspace root when workspaceRoot is provided', () => {
|
||||
const prompt = getSystemPrompt({
|
||||
workspaceRoot: '/home/daytona/workspace',
|
||||
});
|
||||
|
||||
expect(prompt).toContain('Workspace root: `/home/daytona/workspace`');
|
||||
expect(prompt).toContain('/home/daytona/workspace/knowledge-base/index.json');
|
||||
expect(prompt).not.toContain('<workspace_root>');
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,6 @@ export async function createInstanceAgent(options: CreateInstanceAgentOptions):
|
|||
});
|
||||
const hasDeferrableTools = !options.disableDeferredTools && deferredTools.size > 0;
|
||||
const runtimeTools = hasDeferrableTools ? coreTools : tracedOrchestratorTools;
|
||||
const sandboxWorkspaceAvailable = Boolean(orchestrationContext?.workspace);
|
||||
const systemPrompt = getSystemPrompt({
|
||||
webhookBaseUrl: orchestrationContext?.webhookBaseUrl,
|
||||
formBaseUrl: orchestrationContext?.formBaseUrl,
|
||||
|
|
@ -136,7 +135,10 @@ export async function createInstanceAgent(options: CreateInstanceAgentOptions):
|
|||
timeZone: options.timeZone,
|
||||
browserAvailable: browserToolNames.size > 0,
|
||||
branchReadOnly: context.branchReadOnly,
|
||||
sandboxWorkspaceAvailable,
|
||||
workspaceRoot:
|
||||
orchestrationContext?.workspace && orchestrationContext.workspaceRoot
|
||||
? orchestrationContext.workspaceRoot
|
||||
: undefined,
|
||||
});
|
||||
|
||||
const telemetry = orchestrationContext?.tracing?.getTelemetry?.({
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@
|
|||
* near-duplicate copies across files.
|
||||
*/
|
||||
|
||||
import { N8N_SANDBOX_WORKSPACE_ROOT } from '@/workspace/sandbox-setup';
|
||||
|
||||
export const SUBAGENT_OUTPUT_CONTRACT = `## Output Discipline
|
||||
- You report to a parent agent, not a human. Be terse.
|
||||
- Do not narrate ("I'll search for…", "Let me look up…") — just do the work.
|
||||
|
|
@ -21,17 +19,32 @@ export const UNTRUSTED_CONTENT_DOCTRINE =
|
|||
export const ASK_USER_FALLBACK =
|
||||
'If you are stuck or need information only a human can provide (e.g. a chat ID, external resource name, account label), use the `ask-user` tool. Do not retry the same failing approach more than twice — ask the user instead. Never solicit API keys, tokens, or other secrets through `ask-user` — route credential collection through credential setup or Computer Use browser credential capture instead.';
|
||||
|
||||
export const SANDBOX_WORKSPACE_SECTION = `## Sandbox workspace
|
||||
const WORKSPACE_ROOT_PLACEHOLDER = '<workspace_root>';
|
||||
|
||||
function substituteWorkspaceRoot(text: string, workspaceRoot?: string): string {
|
||||
if (!workspaceRoot) return text;
|
||||
return text.replaceAll(WORKSPACE_ROOT_PLACEHOLDER, workspaceRoot);
|
||||
}
|
||||
|
||||
export function getSandboxWorkspaceSection(workspaceRoot?: string): string {
|
||||
const pathHint = workspaceRoot
|
||||
? `\nWorkspace root: \`${workspaceRoot}\`. Paths below are under this root — pass them to \`workspace_read_file\`, \`workspace_list_files\`, and \`workspace_execute_command\` as shown (relative paths like \`knowledge-base/...\` also work).\n`
|
||||
: '';
|
||||
|
||||
const section = `## Sandbox workspace
|
||||
${pathHint}
|
||||
A thread-scoped sandbox workspace is available via \`workspace_read_file\`, \`workspace_list_files\`, and \`workspace_execute_command\` (use \`grep\` or \`rg\` to search). The workspace is created on first use and includes baked-in reference material:
|
||||
|
||||
- \`${N8N_SANDBOX_WORKSPACE_ROOT}/knowledge-base/best-practices/index.json\` — index of workflow technique guides; read the linked \`.md\` files for full documentation
|
||||
- \`${N8N_SANDBOX_WORKSPACE_ROOT}/knowledge-base/best-practices/*.md\` — n8n workflow design best practices (scheduling, forms, data persistence, web apps, etc.)
|
||||
- \`${N8N_SANDBOX_WORKSPACE_ROOT}/node-types/index.txt\` — searchable catalog of available n8n nodes
|
||||
- \`${N8N_SANDBOX_WORKSPACE_ROOT}/workflows/*.json\` — existing workflows on this instance (when synced)
|
||||
- Curated template examples under the workspace root (when present)
|
||||
- \`<workspace_root>/knowledge-base/index.json\` — combined catalog of technique guides and curated workflow templates
|
||||
- \`<workspace_root>/knowledge-base/best-practices/index.json\` — workflow technique guides (read the linked \`.md\` files)
|
||||
- \`<workspace_root>/knowledge-base/templates/index.json\` — curated SDK workflow examples (read the linked \`.ts\` source files)
|
||||
- \`<workspace_root>/node-types/index.txt\` — searchable catalog of available n8n nodes
|
||||
- \`<workspace_root>/workflows/*.json\` — existing workflows on this instance (when synced)
|
||||
|
||||
**Consult the best-practices knowledge base early and often.** Before planning or building a workflow — and whenever a request touches a technique you have not already reviewed in this thread — read \`${N8N_SANDBOX_WORKSPACE_ROOT}/knowledge-base/best-practices/index.json\` and \`grep\`/\`rg\` plus \`workspace_read_file\` the linked \`.md\` guides for each relevant technique. These guides reflect current n8n patterns and supersede your training priors, so prefer them over assumptions. Default to checking the knowledge base; skip it only for trivial mechanical changes where you have already reviewed the relevant guidance in this thread.`;
|
||||
**Consult the knowledge base before planning or building.** Read \`<workspace_root>/knowledge-base/index.json\` (or the section indexes under \`best-practices/\` and \`templates/\`), then \`workspace_read_file\` the relevant \`.md\` guides for each technique the request involves and matching \`.ts\` template files for structural patterns. Skip only for trivial mechanical edits you have already reviewed in this thread.`;
|
||||
|
||||
return substituteWorkspaceRoot(section, workspaceRoot);
|
||||
}
|
||||
|
||||
export const PLACEHOLDERS_RULE = `## Placeholders
|
||||
Use \`placeholder('descriptive hint')\` for values that cannot be safely picked without the user:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
import { DateTime } from 'luxon';
|
||||
|
||||
import { N8N_SANDBOX_WORKSPACE_ROOT } from '@/workspace/sandbox-setup';
|
||||
|
||||
import { getComputerUsePrompt } from './computer-use-prompt';
|
||||
import { SECRET_ASK_GUARDRAIL } from './credential-guardrails.prompt';
|
||||
import { SANDBOX_WORKSPACE_SECTION, UNTRUSTED_CONTENT_DOCTRINE } from './shared-prompts';
|
||||
import { getSandboxWorkspaceSection, UNTRUSTED_CONTENT_DOCTRINE } from './shared-prompts';
|
||||
import type { LocalGatewayStatus } from '../types';
|
||||
|
||||
interface SystemPromptOptions {
|
||||
|
|
@ -19,8 +17,8 @@ interface SystemPromptOptions {
|
|||
browserAvailable?: boolean;
|
||||
/** When true, the instance is in read-only mode (source control branchReadOnly). */
|
||||
branchReadOnly?: boolean;
|
||||
/** When true, a lazy thread-scoped sandbox workspace is attached for file/command tools. */
|
||||
sandboxWorkspaceAvailable?: boolean;
|
||||
/** Absolute or host-relative sandbox workspace root for `<workspace_root>` paths in prompts. */
|
||||
workspaceRoot?: string;
|
||||
}
|
||||
|
||||
export function getDateTimeSection(timeZone?: string): string {
|
||||
|
|
@ -85,18 +83,13 @@ export function getSystemPrompt(options: SystemPromptOptions = {}): string {
|
|||
timeZone,
|
||||
browserAvailable,
|
||||
branchReadOnly,
|
||||
sandboxWorkspaceAvailable,
|
||||
workspaceRoot,
|
||||
} = options;
|
||||
|
||||
const buildKnowledgeBaseNudge = sandboxWorkspaceAvailable
|
||||
? `
|
||||
|
||||
**Consult the best-practices knowledge base before building or editing.** Direct single-workflow edits skip the planner, so they also skip its knowledge-base discovery step — do it yourself. Before writing SDK code, \`grep\`/\`rg\` \`${N8N_SANDBOX_WORKSPACE_ROOT}/knowledge-base/best-practices/index.json\` and \`workspace_read_file\` the linked \`.md\` guides for any technique the change involves (scheduling, forms, data persistence, web apps, error handling, batching, pagination, AI agents, etc.). These guides reflect current n8n patterns and supersede your training priors. Skip this only for trivial mechanical edits where you have already reviewed the relevant guidance in this thread.`
|
||||
: '';
|
||||
|
||||
return `You are the n8n Instance Agent — an AI assistant embedded in an n8n instance. You help users build, run, debug, and manage workflows through natural language.
|
||||
${getDateTimeSection(timeZone)}
|
||||
${webhookBaseUrl && formBaseUrl ? getInstanceInfoSection(webhookBaseUrl, formBaseUrl) : ''}
|
||||
${workspaceRoot ? `\n${getSandboxWorkspaceSection(workspaceRoot)}\n` : ''}
|
||||
|
||||
You have access to workflow, execution, and credential tools plus a specialized workflow-builder skill. You also have delegation capabilities for complex tasks, and may have access to MCP tools for extended capabilities.
|
||||
|
||||
|
|
@ -104,7 +97,7 @@ You have access to workflow, execution, and credential tools plus a specialized
|
|||
|
||||
Route by **what you are touching**, not by how risky the change feels:
|
||||
|
||||
1. **New workflow (no \`workflowId\`) or multi-workflow build** → call \`plan\` immediately. Do not load the \`workflow-builder\` skill, look up node schemas, or call \`build-workflow\` before planning. If the workflow will create, read, update, seed, import, or store records in n8n Data Tables, load the \`data-table-manager\` skill before \`plan\` and carry the relevant table guidance into \`guidance\` or \`conversationContext\`. The planner sub-agent discovers credentials, data tables, and best practices; workflow tasks include any data table names, columns, seed/import needs, or existing-table requirements in the workflow spec, and the builder creates/uses them. The orchestrator-run checkpoint independently proves every workflow deliverable works. Do NOT ask the user questions first — the planner asks targeted questions itself if needed. Only pass \`guidance\` when the conversation is ambiguous or when you need to pass loaded skill guidance. When \`plan\` returns, tasks are already dispatched.
|
||||
1. **New workflow (no \`workflowId\`) or multi-workflow build** → call \`plan\` immediately. Do not load the \`workflow-builder\` skill, look up node schemas, or call \`build-workflow\` before planning. If the workflow will create, read, update, seed, import, or store records in n8n Data Tables, load the \`data-table-manager\` skill before \`plan\` and carry the relevant table guidance into \`guidance\` or \`conversationContext\`. The planner sub-agent discovers credentials and data tables; workflow tasks include any data table names, columns, seed/import needs, or existing-table requirements in the workflow spec, and the builder creates/uses them. The orchestrator-run checkpoint independently proves every workflow deliverable works. Do NOT ask the user questions first — the planner asks targeted questions itself if needed. Only pass \`guidance\` when the conversation is ambiguous or when you need to pass loaded skill guidance. When \`plan\` returns, tasks are already dispatched.
|
||||
|
||||
2. **Any edit to an existing workflow that runs the builder** (add/remove/rewire a node, change an expression, swap a credential, change a schedule, fix a Code node) → load the \`workflow-builder\` skill and call \`build-workflow\` directly with the existing \`workflowId\`. The tool asks for approval before saving when required. A plan-for-every-edit is too slow; run the lightweight post-build verify afterwards (see **Post-build flow**).
|
||||
|
||||
|
|
@ -126,7 +119,7 @@ When \`credentials(action="setup")\` returns \`needsBrowserSetup=true\`, load th
|
|||
|
||||
Never use \`delegate\` to build, patch, fix, or update workflows — workflow building happens in the orchestrator with the \`workflow-builder\` skill and the workflow build tools.
|
||||
|
||||
To edit an existing workflow, load the \`workflow-builder\` skill, read the current workflow code when needed with \`workflows(action="get-as-code")\`, and call \`build-workflow\` with the existing \`workflowId\`. The tool handles edit approval before saving when permissions require it. Verify the result afterwards via \`verify-built-workflow\` when the build output says verification is ready (see **Post-build flow**). Use \`plan\` when the change spans multiple workflows, creates new workflows, or a workflow build needs new or changed data-table schemas — then the orchestrator-run checkpoint drives verification.${buildKnowledgeBaseNudge}
|
||||
To edit an existing workflow, load the \`workflow-builder\` skill, read the current workflow code when needed with \`workflows(action="get-as-code")\`, and call \`build-workflow\` with the existing \`workflowId\`. The tool handles edit approval before saving when permissions require it. Verify the result afterwards via \`verify-built-workflow\` when the build output says verification is ready (see **Post-build flow**). Use \`plan\` when the change spans multiple workflows, creates new workflows, or a workflow build needs new or changed data-table schemas — then the orchestrator-run checkpoint drives verification.
|
||||
|
||||
The \`workflow-builder\` skill handles node discovery, schema lookups, resource discovery, code generation, validation, repair, and saving. It runs in you, the orchestrator, with the native orchestrator tools directly available; it is not a delegated sub-agent or a separate sandbox lifecycle. For planned workflow builds, follow the build task spec exactly. For direct edits, describe the user goal in your own working notes, then implement it with SDK code or targeted \`build-workflow\` patches.
|
||||
|
||||
|
|
@ -200,7 +193,6 @@ Examples: search "credential" for the credentials tool, search "file" for filesy
|
|||
You have the \`research\` tool with \`web-search\` and \`fetch-url\` actions. Use them directly for most questions. Use \`plan\` with \`research\` tasks only for broad detached synthesis (comparing services, broad surveys across 3+ doc pages).
|
||||
|
||||
${UNTRUSTED_CONTENT_DOCTRINE}
|
||||
${sandboxWorkspaceAvailable ? `\n${SANDBOX_WORKSPACE_SECTION}\n` : ''}
|
||||
${getComputerUsePrompt({ browserAvailable, localGateway })}
|
||||
|
||||
${
|
||||
|
|
|
|||
|
|
@ -420,6 +420,8 @@ export const setupSandboxWorkspace: typeof SandboxSetupMod.setupSandboxWorkspace
|
|||
() => loadSandboxSetup().setupSandboxWorkspace,
|
||||
);
|
||||
export type BuilderTemplatesService = BuilderTemplatesServiceMod.BuilderTemplatesService;
|
||||
export { createScopedWorkspace } from './workspace/scoped-workspace';
|
||||
export { getPromptWorkspaceRoot } from './workspace/sandbox-setup';
|
||||
export const BuilderTemplatesService: typeof BuilderTemplatesServiceMod.BuilderTemplatesService =
|
||||
lazyClass(() => loadBuilderTemplatesService().BuilderTemplatesService);
|
||||
export const builderTemplatesOptionsFromEnv: typeof BuilderTemplatesServiceMod.builderTemplatesOptionsFromEnv =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
import { buildTemplatesIndexFromArchive } from '../build-templates-index';
|
||||
|
||||
describe('buildTemplatesIndexFromArchive', () => {
|
||||
it('uses index.json when present', () => {
|
||||
const extracted = new Map([
|
||||
[
|
||||
'index.json',
|
||||
JSON.stringify({
|
||||
entries: [
|
||||
{
|
||||
id: 'slack-daily-summary',
|
||||
description: 'Daily Slack summary',
|
||||
file: 'templates/slack-daily-summary.ts',
|
||||
techniques: ['notification'],
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
['slack-daily-summary.ts', 'export default {};'],
|
||||
]);
|
||||
|
||||
expect(buildTemplatesIndexFromArchive(extracted)).toEqual({
|
||||
entries: [
|
||||
{
|
||||
id: 'slack-daily-summary',
|
||||
description: 'Daily Slack summary',
|
||||
file: 'templates/slack-daily-summary.ts',
|
||||
techniques: ['notification'],
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('converts CDN index.txt into structured entries', () => {
|
||||
const extracted = new Map([
|
||||
['index.txt', 'example.ts | Example template'],
|
||||
['example.ts', 'export default {};'],
|
||||
]);
|
||||
|
||||
expect(buildTemplatesIndexFromArchive(extracted)).toEqual({
|
||||
entries: [
|
||||
{
|
||||
id: 'example',
|
||||
description: 'Example template',
|
||||
file: 'templates/example.ts',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('derives entries from template files when no catalog is present', () => {
|
||||
const extracted = new Map([
|
||||
['alpha.ts', 'export default {};'],
|
||||
['beta.ts', 'export default {};'],
|
||||
]);
|
||||
|
||||
expect(buildTemplatesIndexFromArchive(extracted)).toEqual({
|
||||
entries: [
|
||||
{ id: 'alpha', description: 'alpha', file: 'templates/alpha.ts' },
|
||||
{ id: 'beta', description: 'beta', file: 'templates/beta.ts' },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import { gzipSync } from 'node:zlib';
|
||||
|
||||
export type BuilderTemplatesTarEntry = {
|
||||
name: string;
|
||||
content?: string;
|
||||
typeFlag?: string;
|
||||
linkName?: string;
|
||||
};
|
||||
|
||||
function writeTarOctal(buffer: Buffer, offset: number, length: number, value: number): void {
|
||||
const octal = value
|
||||
.toString(8)
|
||||
.padStart(length - 1, '0')
|
||||
.slice(-(length - 1));
|
||||
buffer.write(octal, offset, length - 1, 'ascii');
|
||||
buffer[offset + length - 1] = 0;
|
||||
}
|
||||
|
||||
function writeTarChecksum(buffer: Buffer, checksum: number): void {
|
||||
const octal = checksum.toString(8).padStart(6, '0').slice(-6);
|
||||
buffer.write(octal, 148, 6, 'ascii');
|
||||
buffer[154] = 0;
|
||||
buffer[155] = 0x20;
|
||||
}
|
||||
|
||||
/** Build a gzip-wrapped tar archive matching the n8n-sdk-templates bundle shape. */
|
||||
export function makeBuilderTemplatesTarGz(entries: BuilderTemplatesTarEntry[]): Buffer {
|
||||
const blocks: Buffer[] = [];
|
||||
for (const entry of entries) {
|
||||
const content = Buffer.from(entry.content ?? '', 'utf-8');
|
||||
const typeFlag = entry.typeFlag ?? '0';
|
||||
const size = typeFlag === '0' ? content.byteLength : 0;
|
||||
const header = Buffer.alloc(512);
|
||||
|
||||
header.write(entry.name, 0, 100, 'utf-8');
|
||||
writeTarOctal(header, 100, 8, 0o644);
|
||||
writeTarOctal(header, 108, 8, 0);
|
||||
writeTarOctal(header, 116, 8, 0);
|
||||
writeTarOctal(header, 124, 12, size);
|
||||
writeTarOctal(header, 136, 12, 0);
|
||||
header.fill(0x20, 148, 156);
|
||||
header.write(typeFlag, 156, 1, 'ascii');
|
||||
if (entry.linkName) header.write(entry.linkName, 157, 100, 'utf-8');
|
||||
header.write('ustar', 257, 5, 'ascii');
|
||||
header.write('00', 263, 2, 'ascii');
|
||||
|
||||
const checksum = header.reduce((sum, byte) => sum + byte, 0);
|
||||
writeTarChecksum(header, checksum);
|
||||
blocks.push(header);
|
||||
|
||||
if (size > 0) {
|
||||
blocks.push(content);
|
||||
const padding = (512 - (size % 512)) % 512;
|
||||
if (padding > 0) blocks.push(Buffer.alloc(padding));
|
||||
}
|
||||
}
|
||||
blocks.push(Buffer.alloc(1024));
|
||||
return gzipSync(Buffer.concat(blocks));
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import {
|
||||
extractBuilderTemplatesArchive,
|
||||
validateBuilderTemplatesArchive,
|
||||
} from '../extract-builder-templates-archive';
|
||||
import { makeBuilderTemplatesTarGz } from './builder-templates-archive.fixtures';
|
||||
|
||||
const VALID_JSON_INDEX_ARCHIVE = makeBuilderTemplatesTarGz([
|
||||
{
|
||||
name: 'index.json',
|
||||
content: JSON.stringify({
|
||||
entries: [
|
||||
{
|
||||
id: 'slack-daily-summary',
|
||||
description: 'Daily Slack',
|
||||
file: 'templates/slack-daily-summary.ts',
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ name: 'slack-daily-summary.ts', content: 'export default {};' },
|
||||
]);
|
||||
|
||||
const VALID_CDN_INDEX_TXT_ARCHIVE = makeBuilderTemplatesTarGz([
|
||||
{ name: 'index.txt', content: 'slack-daily-summary.ts | Daily Slack' },
|
||||
{ name: 'slack-daily-summary.ts', content: 'export default {};' },
|
||||
]);
|
||||
|
||||
describe('extractBuilderTemplatesArchive', () => {
|
||||
it('extracts index.json and template files from a valid archive', () => {
|
||||
const extracted = extractBuilderTemplatesArchive(VALID_JSON_INDEX_ARCHIVE);
|
||||
|
||||
expect(extracted?.get('index.json')).toContain('slack-daily-summary');
|
||||
expect(extracted?.get('slack-daily-summary.ts')).toBe('export default {};');
|
||||
});
|
||||
|
||||
it('accepts CDN archives that still ship index.txt', () => {
|
||||
const extracted = extractBuilderTemplatesArchive(VALID_CDN_INDEX_TXT_ARCHIVE);
|
||||
|
||||
expect(extracted?.get('index.txt')).toBe('slack-daily-summary.ts | Daily Slack');
|
||||
expect(extracted?.get('slack-daily-summary.ts')).toBe('export default {};');
|
||||
expect(validateBuilderTemplatesArchive(VALID_CDN_INDEX_TXT_ARCHIVE)).toBeNull();
|
||||
});
|
||||
|
||||
it.each<[string, Buffer]>([
|
||||
['absolute path', makeBuilderTemplatesTarGz([{ name: '/escape.ts', content: 'x' }])],
|
||||
['parent traversal', makeBuilderTemplatesTarGz([{ name: '../escape.ts', content: 'x' }])],
|
||||
['nested path', makeBuilderTemplatesTarGz([{ name: 'nested/template.ts', content: 'x' }])],
|
||||
['malformed gzip', Buffer.from('not-a-gzip-archive')],
|
||||
])('returns null for invalid archive: %s', (_label, archive) => {
|
||||
expect(extractBuilderTemplatesArchive(archive)).toBeNull();
|
||||
expect(validateBuilderTemplatesArchive(archive)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -3,11 +3,14 @@ import { jsonParse } from 'n8n-workflow';
|
|||
import type { SandboxWorkspace } from '../../workspace/sandbox-fs';
|
||||
import {
|
||||
buildKnowledgeBaseWorkspaceBundle,
|
||||
KNOWLEDGE_BASE_INDEX_FILE,
|
||||
KNOWLEDGE_BASE_MANIFEST_FILE,
|
||||
KNOWLEDGE_BASE_TEMPLATES_DIR,
|
||||
loadPrebakedKnowledgeBaseBundle,
|
||||
materializeKnowledgeBaseIntoWorkspace,
|
||||
SANDBOX_KNOWLEDGE_BASE_DIR,
|
||||
} from '../materialize-knowledge-base';
|
||||
import { makeBuilderTemplatesTarGz } from './builder-templates-archive.fixtures';
|
||||
|
||||
const ROOT = '/home/daytona/workspace';
|
||||
|
||||
|
|
@ -46,7 +49,7 @@ function createSandboxWorkspace(files: Map<string, string>): {
|
|||
}
|
||||
|
||||
describe('buildKnowledgeBaseWorkspaceBundle', () => {
|
||||
it('builds best-practice markdown files, index, and manifest', () => {
|
||||
it('builds best-practice markdown files, section indexes, root index, and manifest v4', () => {
|
||||
const bundle = buildKnowledgeBaseWorkspaceBundle({ root: ROOT });
|
||||
|
||||
expect(bundle.rootDir).toBe(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}`);
|
||||
|
|
@ -57,19 +60,139 @@ describe('buildKnowledgeBaseWorkspaceBundle', () => {
|
|||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/best-practices/index.json`),
|
||||
).toBeDefined();
|
||||
expect(
|
||||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_MANIFEST_FILE}`),
|
||||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_INDEX_FILE}`),
|
||||
).toBeDefined();
|
||||
const manifest = jsonParse<{
|
||||
schemaVersion: number;
|
||||
contentHash: string;
|
||||
}>(
|
||||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_MANIFEST_FILE}`) ??
|
||||
'',
|
||||
);
|
||||
expect(manifest.schemaVersion).toBe(4);
|
||||
expect(manifest.contentHash).toBe(bundle.contentHash);
|
||||
expect(bundle.contentHash).toMatch(/^[a-f0-9]{12}$/);
|
||||
|
||||
const index = jsonParse<{ entries: Array<{ id: string; hasDocumentation: boolean }> }>(
|
||||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/best-practices/index.json`) ?? '',
|
||||
);
|
||||
expect(index.entries.some((entry) => entry.id === 'scheduling' && entry.hasDocumentation)).toBe(
|
||||
true,
|
||||
);
|
||||
const bestPracticesIndex = jsonParse<{
|
||||
entries: Array<{ id: string; hasDocumentation: boolean }>;
|
||||
}>(bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/best-practices/index.json`) ?? '');
|
||||
expect(
|
||||
index.entries.some((entry) => entry.id === 'monitoring' && !entry.hasDocumentation),
|
||||
bestPracticesIndex.entries.some(
|
||||
(entry) => entry.id === 'scheduling' && entry.hasDocumentation,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
bestPracticesIndex.entries.some(
|
||||
(entry) => entry.id === 'monitoring' && !entry.hasDocumentation,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
const rootIndex = jsonParse<{
|
||||
bestPractices: { indexFile: string; entries: Array<{ id: string }> };
|
||||
templates: { indexFile: string; entries: unknown[] };
|
||||
}>(
|
||||
bundle.files.get(`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_INDEX_FILE}`) ?? '',
|
||||
);
|
||||
expect(rootIndex.bestPractices.indexFile).toBe('best-practices/index.json');
|
||||
expect(rootIndex.templates.indexFile).toBe('templates/index.json');
|
||||
expect(rootIndex.templates.entries).toEqual([]);
|
||||
expect(rootIndex.bestPractices.entries.some((entry) => entry.id === 'scheduling')).toBe(true);
|
||||
expect(bundle.indexPath).toBe(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_INDEX_FILE}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('materializes CDN index.txt archives as templates/index.json', () => {
|
||||
const archive = makeBuilderTemplatesTarGz([
|
||||
{ name: 'index.txt', content: 'example.ts | Example template' },
|
||||
{ name: 'example.ts', content: 'export default {};' },
|
||||
]);
|
||||
const bundle = buildKnowledgeBaseWorkspaceBundle({
|
||||
root: ROOT,
|
||||
templatesArchive: archive,
|
||||
});
|
||||
|
||||
const templatesIndex = jsonParse<{ entries: Array<{ id: string; description: string }> }>(
|
||||
bundle.files.get(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_TEMPLATES_DIR}/index.json`,
|
||||
) ?? '',
|
||||
);
|
||||
expect(templatesIndex.entries).toEqual([
|
||||
{
|
||||
id: 'example',
|
||||
description: 'Example template',
|
||||
file: 'templates/example.ts',
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
bundle.files.has(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_TEMPLATES_DIR}/index.txt`,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('materializes templates as index.json and includes them in the root index', () => {
|
||||
const withoutTemplates = buildKnowledgeBaseWorkspaceBundle({ root: ROOT });
|
||||
const archive = makeBuilderTemplatesTarGz([
|
||||
{
|
||||
name: 'index.json',
|
||||
content: JSON.stringify({
|
||||
entries: [
|
||||
{
|
||||
id: 'example',
|
||||
description: 'Example template',
|
||||
file: 'templates/example.ts',
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ name: 'example.ts', content: 'export default {};' },
|
||||
]);
|
||||
const withTemplates = buildKnowledgeBaseWorkspaceBundle({
|
||||
root: ROOT,
|
||||
templatesArchive: archive,
|
||||
});
|
||||
|
||||
const templatesIndex = jsonParse<{
|
||||
entries: Array<{ id: string; description: string; file: string }>;
|
||||
}>(
|
||||
withTemplates.files.get(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_TEMPLATES_DIR}/index.json`,
|
||||
) ?? '',
|
||||
);
|
||||
expect(templatesIndex.entries).toEqual([
|
||||
{
|
||||
id: 'example',
|
||||
description: 'Example template',
|
||||
file: 'templates/example.ts',
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
withTemplates.files.get(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_TEMPLATES_DIR}/example.ts`,
|
||||
),
|
||||
).toBe('export default {};\n');
|
||||
expect(
|
||||
withTemplates.files.has(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_TEMPLATES_DIR}/index.json`,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
const rootIndex = jsonParse<{
|
||||
templates: { entries: Array<{ id: string }> };
|
||||
}>(
|
||||
withTemplates.files.get(
|
||||
`${ROOT}/${SANDBOX_KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_INDEX_FILE}`,
|
||||
) ?? '',
|
||||
);
|
||||
expect(rootIndex.templates.entries).toEqual([
|
||||
{
|
||||
id: 'example',
|
||||
description: 'Example template',
|
||||
file: 'templates/example.ts',
|
||||
},
|
||||
]);
|
||||
expect(withTemplates.contentHash).not.toBe(withoutTemplates.contentHash);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,148 @@
|
|||
import { jsonParse } from 'n8n-workflow';
|
||||
import { join as posixJoin } from 'node:path/posix';
|
||||
import { z } from 'zod';
|
||||
|
||||
export const KNOWLEDGE_BASE_TEMPLATES_DIR = 'templates';
|
||||
|
||||
export interface KnowledgeBaseTemplateEntry {
|
||||
id: string;
|
||||
description: string;
|
||||
file: string;
|
||||
version?: string;
|
||||
techniques?: string[];
|
||||
}
|
||||
|
||||
export interface KnowledgeBaseTemplatesIndex {
|
||||
entries: KnowledgeBaseTemplateEntry[];
|
||||
}
|
||||
|
||||
const templateEntrySchema = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string(),
|
||||
file: z.string().optional(),
|
||||
version: z.string().optional(),
|
||||
techniques: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
const templatesIndexSchema = z.object({
|
||||
entries: z.array(templateEntrySchema),
|
||||
});
|
||||
|
||||
const PIPE_DELIMITED_INDEX_LINE = /^([a-zA-Z0-9][a-zA-Z0-9._-]*\.ts)\s*\|\s*(.+)$/;
|
||||
|
||||
function templateFilenameFromId(id: string): string {
|
||||
return `${id}.ts`;
|
||||
}
|
||||
|
||||
function templateFilePath(filename: string): string {
|
||||
return posixJoin(KNOWLEDGE_BASE_TEMPLATES_DIR, filename);
|
||||
}
|
||||
|
||||
function normalizeTemplateEntry(
|
||||
entry: z.infer<typeof templateEntrySchema>,
|
||||
availableFilenames: Set<string>,
|
||||
): KnowledgeBaseTemplateEntry | null {
|
||||
const filename = entry.file
|
||||
? entry.file.replace(/^templates\//, '').replace(/^\//, '')
|
||||
: templateFilenameFromId(entry.id);
|
||||
|
||||
if (!filename.endsWith('.ts') || !availableFilenames.has(filename)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalized: KnowledgeBaseTemplateEntry = {
|
||||
id: entry.id,
|
||||
description: entry.description,
|
||||
file: templateFilePath(filename),
|
||||
};
|
||||
if (entry.version !== undefined) normalized.version = entry.version;
|
||||
if (entry.techniques !== undefined) normalized.techniques = entry.techniques;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/** Parse the pipe-delimited catalog shipped by current CDN archives (`index.txt`). */
|
||||
function parsePipeDelimitedTemplatesCatalog(content: string): KnowledgeBaseTemplateEntry[] {
|
||||
const entries: KnowledgeBaseTemplateEntry[] = [];
|
||||
|
||||
for (const line of content.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
const match = PIPE_DELIMITED_INDEX_LINE.exec(trimmed);
|
||||
if (!match) continue;
|
||||
|
||||
const filename = match[1];
|
||||
entries.push({
|
||||
id: filename.replace(/\.ts$/, ''),
|
||||
description: match[2].trim(),
|
||||
file: templateFilePath(filename),
|
||||
});
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parseTemplatesIndexJson(content: string): z.infer<typeof templatesIndexSchema> | null {
|
||||
try {
|
||||
const parsed: unknown = jsonParse(content);
|
||||
const result = templatesIndexSchema.safeParse(parsed);
|
||||
return result.success ? result.data : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function listTemplateFilenames(extracted: Map<string, string>): Set<string> {
|
||||
const filenames = new Set<string>();
|
||||
for (const name of extracted.keys()) {
|
||||
if (name.endsWith('.ts')) filenames.add(name);
|
||||
}
|
||||
return filenames;
|
||||
}
|
||||
|
||||
function buildEntriesFromTemplateFiles(
|
||||
availableFilenames: Set<string>,
|
||||
): KnowledgeBaseTemplateEntry[] {
|
||||
return [...availableFilenames].sort().map((filename) => ({
|
||||
id: filename.replace(/\.ts$/, ''),
|
||||
description: filename.replace(/\.ts$/, ''),
|
||||
file: templateFilePath(filename),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a normalized templates index from an extracted n8n-sdk-templates archive.
|
||||
* Uses `index.json` when present, converts CDN `index.txt` when needed, otherwise
|
||||
* derives entries from `.ts` files. Only `index.json` is written to the workspace.
|
||||
*/
|
||||
export function buildTemplatesIndexFromArchive(
|
||||
extracted: Map<string, string>,
|
||||
): KnowledgeBaseTemplatesIndex {
|
||||
const availableFilenames = listTemplateFilenames(extracted);
|
||||
|
||||
const indexJson = extracted.get('index.json');
|
||||
if (indexJson) {
|
||||
const parsed = parseTemplatesIndexJson(indexJson);
|
||||
if (parsed) {
|
||||
const entries = parsed.entries
|
||||
.map((entry) => normalizeTemplateEntry(entry, availableFilenames))
|
||||
.filter((entry): entry is KnowledgeBaseTemplateEntry => entry !== null);
|
||||
if (entries.length > 0) {
|
||||
return { entries };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const indexTxt = extracted.get('index.txt');
|
||||
if (indexTxt) {
|
||||
const entries = parsePipeDelimitedTemplatesCatalog(indexTxt).filter((entry) => {
|
||||
const filename = entry.file.replace(`${KNOWLEDGE_BASE_TEMPLATES_DIR}/`, '');
|
||||
return availableFilenames.has(filename);
|
||||
});
|
||||
if (entries.length > 0) {
|
||||
return { entries };
|
||||
}
|
||||
}
|
||||
|
||||
return { entries: buildEntriesFromTemplateFiles(availableFilenames) };
|
||||
}
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
const TAR_BLOCK_SIZE = 512;
|
||||
const TAR_TYPE_REGULAR = '0';
|
||||
const TEMPLATE_ENTRY_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*\.ts$/;
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isAllowedTemplateEntryName(name: string): boolean {
|
||||
if (name === 'index.json' || name === 'index.txt') return true;
|
||||
return TEMPLATE_ENTRY_PATTERN.test(name);
|
||||
}
|
||||
|
||||
function isZeroBlock(block: Buffer): boolean {
|
||||
return block.every((byte) => byte === 0);
|
||||
}
|
||||
|
||||
function readTarString(block: Buffer, start: number, length: number): string {
|
||||
const field = block.subarray(start, start + length);
|
||||
const nullIndex = field.indexOf(0);
|
||||
return field.subarray(0, nullIndex === -1 ? field.length : nullIndex).toString('utf-8');
|
||||
}
|
||||
|
||||
function parseTarOctal(block: Buffer, start: number, length: number): number | null {
|
||||
const raw = readTarString(block, start, length).trim();
|
||||
if (!/^[0-7]+$/.test(raw)) return null;
|
||||
const parsed = Number.parseInt(raw, 8);
|
||||
return Number.isSafeInteger(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
type TarWalkResult = { ok: true; entries: Map<string, string> } | { ok: false; error: string };
|
||||
|
||||
function walkBuilderTemplatesTar(tar: Buffer, collectContent: boolean): TarWalkResult {
|
||||
const entries = new Map<string, string>();
|
||||
let offset = 0;
|
||||
|
||||
while (offset + TAR_BLOCK_SIZE <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + TAR_BLOCK_SIZE);
|
||||
if (isZeroBlock(header)) return { ok: true, entries };
|
||||
|
||||
const name = readTarString(header, 0, 100);
|
||||
const prefix = readTarString(header, 345, 155);
|
||||
const entryName = prefix ? `${prefix}/${name}` : name;
|
||||
const typeFlag = readTarString(header, 156, 1);
|
||||
const size = parseTarOctal(header, 124, 12);
|
||||
|
||||
if (size === null) {
|
||||
return { ok: false, error: `invalid size for archive entry "${entryName}"` };
|
||||
}
|
||||
if (typeFlag !== '' && typeFlag !== TAR_TYPE_REGULAR) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `unsupported archive entry type "${typeFlag}" for "${entryName}"`,
|
||||
};
|
||||
}
|
||||
if (!isAllowedTemplateEntryName(entryName)) {
|
||||
return { ok: false, error: `unsupported archive entry path "${entryName}"` };
|
||||
}
|
||||
|
||||
const dataStart = offset + TAR_BLOCK_SIZE;
|
||||
if (collectContent && size > 0) {
|
||||
entries.set(entryName, tar.subarray(dataStart, dataStart + size).toString('utf-8'));
|
||||
}
|
||||
|
||||
const dataBlocks = Math.ceil(size / TAR_BLOCK_SIZE);
|
||||
offset += TAR_BLOCK_SIZE + dataBlocks * TAR_BLOCK_SIZE;
|
||||
}
|
||||
|
||||
return offset === tar.length
|
||||
? { ok: true, entries }
|
||||
: { ok: false, error: 'trailing partial tar header' };
|
||||
}
|
||||
|
||||
function gunzipArchive(archive: Buffer): { tar: Buffer } | { error: string } {
|
||||
try {
|
||||
return { tar: gunzipSync(archive) };
|
||||
} catch (error) {
|
||||
return { error: `failed to gunzip archive: ${getErrorMessage(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the exact archive shape published by n8n-sdk-templates: a gzip-wrapped tar
|
||||
* with only regular top-level files (`index.json` or CDN `index.txt`, and `<slug>.ts`).
|
||||
* The workspace never materializes `index.txt` — it is converted to `index.json`.
|
||||
*/
|
||||
export function validateBuilderTemplatesArchive(archive: Buffer): string | null {
|
||||
const gunzipped = gunzipArchive(archive);
|
||||
if ('error' in gunzipped) return gunzipped.error;
|
||||
|
||||
const walked = walkBuilderTemplatesTar(gunzipped.tar, false);
|
||||
return walked.ok ? null : walked.error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract template files from a validated n8n-sdk-templates archive on the host.
|
||||
* Returns null when the archive is invalid.
|
||||
*/
|
||||
export function extractBuilderTemplatesArchive(archive: Buffer): Map<string, string> | null {
|
||||
const gunzipped = gunzipArchive(archive);
|
||||
if ('error' in gunzipped) return null;
|
||||
|
||||
const walked = walkBuilderTemplatesTar(gunzipped.tar, true);
|
||||
return walked.ok ? walked.entries : null;
|
||||
}
|
||||
|
|
@ -7,6 +7,13 @@ import {
|
|||
import { join as posixJoin } from 'node:path/posix';
|
||||
|
||||
import type { Logger } from '../logger';
|
||||
import {
|
||||
buildTemplatesIndexFromArchive,
|
||||
KNOWLEDGE_BASE_TEMPLATES_DIR,
|
||||
type KnowledgeBaseTemplateEntry,
|
||||
} from './build-templates-index';
|
||||
export { KNOWLEDGE_BASE_TEMPLATES_DIR };
|
||||
import { extractBuilderTemplatesArchive } from './extract-builder-templates-archive';
|
||||
import { computeWorkspaceContentHash } from '../workspace/compute-workspace-content-hash';
|
||||
import {
|
||||
loadPrebakedWorkspaceBundle,
|
||||
|
|
@ -18,8 +25,9 @@ import { WORKSPACE_MANIFEST_FILE } from '../workspace/workspace-manifest';
|
|||
|
||||
export const SANDBOX_KNOWLEDGE_BASE_DIR = 'knowledge-base';
|
||||
export const KNOWLEDGE_BASE_BEST_PRACTICES_DIR = 'best-practices';
|
||||
export const KNOWLEDGE_BASE_INDEX_FILE = 'index.json';
|
||||
export const KNOWLEDGE_BASE_MANIFEST_FILE = WORKSPACE_MANIFEST_FILE;
|
||||
export const KNOWLEDGE_BASE_MANIFEST_SCHEMA_VERSION = 1;
|
||||
export const KNOWLEDGE_BASE_MANIFEST_SCHEMA_VERSION = 4;
|
||||
|
||||
export interface KnowledgeBaseIndexEntry {
|
||||
id: BestPracticesGuideId;
|
||||
|
|
@ -33,6 +41,17 @@ export interface KnowledgeBaseBestPracticesIndex {
|
|||
entries: KnowledgeBaseIndexEntry[];
|
||||
}
|
||||
|
||||
export interface KnowledgeBaseRootIndex {
|
||||
bestPractices: {
|
||||
indexFile: string;
|
||||
entries: KnowledgeBaseIndexEntry[];
|
||||
};
|
||||
templates: {
|
||||
indexFile: string;
|
||||
entries: KnowledgeBaseTemplateEntry[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface KnowledgeBaseWorkspaceManifest {
|
||||
schemaVersion: typeof KNOWLEDGE_BASE_MANIFEST_SCHEMA_VERSION;
|
||||
contentHash: string;
|
||||
|
|
@ -46,20 +65,55 @@ export interface KnowledgeBaseWorkspaceBundle {
|
|||
contentHash: string;
|
||||
}
|
||||
|
||||
interface MaterializeKnowledgeBaseOptions {
|
||||
workspace: SandboxWorkspace;
|
||||
export interface BuildKnowledgeBaseWorkspaceBundleOptions {
|
||||
root: string;
|
||||
templatesArchive?: Buffer | null;
|
||||
logger?: Logger;
|
||||
}
|
||||
|
||||
export function buildKnowledgeBaseWorkspaceBundle({
|
||||
root,
|
||||
}: {
|
||||
root: string;
|
||||
}): KnowledgeBaseWorkspaceBundle {
|
||||
interface MaterializeKnowledgeBaseOptions extends BuildKnowledgeBaseWorkspaceBundleOptions {
|
||||
workspace: SandboxWorkspace;
|
||||
}
|
||||
|
||||
function addTemplatesToKnowledgeBaseFiles(
|
||||
files: Map<string, string>,
|
||||
rootDir: string,
|
||||
templatesArchive: Buffer,
|
||||
logger?: Logger,
|
||||
): KnowledgeBaseTemplateEntry[] {
|
||||
const extracted = extractBuilderTemplatesArchive(templatesArchive);
|
||||
if (!extracted) {
|
||||
logger?.warn('[knowledge-base] rejected templates archive during bundle build', {
|
||||
archiveBytes: templatesArchive.byteLength,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
|
||||
const templatesIndex = buildTemplatesIndexFromArchive(extracted);
|
||||
const templatesDir = posixJoin(rootDir, KNOWLEDGE_BASE_TEMPLATES_DIR);
|
||||
|
||||
for (const [name, content] of extracted) {
|
||||
if (!name.endsWith('.ts')) continue;
|
||||
files.set(posixJoin(templatesDir, name), withTrailingNewline(content));
|
||||
}
|
||||
|
||||
const templatesIndexPath = posixJoin(templatesDir, KNOWLEDGE_BASE_INDEX_FILE);
|
||||
files.set(templatesIndexPath, stringifyWorkspaceJson(templatesIndex));
|
||||
|
||||
// The decompressed archive has been copied into `files`; release the
|
||||
// intermediate map so the duplicate copy isn't held until GC.
|
||||
extracted.clear();
|
||||
|
||||
return templatesIndex.entries;
|
||||
}
|
||||
|
||||
export function buildKnowledgeBaseWorkspaceBundle(
|
||||
options: BuildKnowledgeBaseWorkspaceBundleOptions,
|
||||
): KnowledgeBaseWorkspaceBundle {
|
||||
const { root, templatesArchive = null, logger } = options;
|
||||
const rootDir = posixJoin(root, SANDBOX_KNOWLEDGE_BASE_DIR);
|
||||
const files = new Map<string, string>();
|
||||
const entries: KnowledgeBaseIndexEntry[] = [];
|
||||
const bestPracticeEntries: KnowledgeBaseIndexEntry[] = [];
|
||||
|
||||
for (const guideId of Object.values(WorkflowTechnique)) {
|
||||
const doc = bestPracticesRegistry[guideId];
|
||||
|
|
@ -77,12 +131,33 @@ export function buildKnowledgeBaseWorkspaceBundle({
|
|||
entry.version = doc.version;
|
||||
}
|
||||
|
||||
entries.push(entry);
|
||||
bestPracticeEntries.push(entry);
|
||||
}
|
||||
|
||||
const indexPath = posixJoin(rootDir, KNOWLEDGE_BASE_BEST_PRACTICES_DIR, 'index.json');
|
||||
const index: KnowledgeBaseBestPracticesIndex = { entries };
|
||||
files.set(indexPath, stringifyWorkspaceJson(index));
|
||||
const bestPracticesIndexPath = posixJoin(
|
||||
rootDir,
|
||||
KNOWLEDGE_BASE_BEST_PRACTICES_DIR,
|
||||
KNOWLEDGE_BASE_INDEX_FILE,
|
||||
);
|
||||
const bestPracticesIndex: KnowledgeBaseBestPracticesIndex = { entries: bestPracticeEntries };
|
||||
files.set(bestPracticesIndexPath, stringifyWorkspaceJson(bestPracticesIndex));
|
||||
|
||||
const templateEntries = templatesArchive
|
||||
? addTemplatesToKnowledgeBaseFiles(files, rootDir, templatesArchive, logger)
|
||||
: [];
|
||||
|
||||
const rootIndexPath = posixJoin(rootDir, KNOWLEDGE_BASE_INDEX_FILE);
|
||||
const rootIndex: KnowledgeBaseRootIndex = {
|
||||
bestPractices: {
|
||||
indexFile: posixJoin(KNOWLEDGE_BASE_BEST_PRACTICES_DIR, KNOWLEDGE_BASE_INDEX_FILE),
|
||||
entries: bestPracticeEntries,
|
||||
},
|
||||
templates: {
|
||||
indexFile: posixJoin(KNOWLEDGE_BASE_TEMPLATES_DIR, KNOWLEDGE_BASE_INDEX_FILE),
|
||||
entries: templateEntries,
|
||||
},
|
||||
};
|
||||
files.set(rootIndexPath, stringifyWorkspaceJson(rootIndex));
|
||||
|
||||
const manifestPath = posixJoin(rootDir, KNOWLEDGE_BASE_MANIFEST_FILE);
|
||||
const contentHash = computeWorkspaceContentHash(files);
|
||||
|
|
@ -95,7 +170,7 @@ export function buildKnowledgeBaseWorkspaceBundle({
|
|||
return {
|
||||
rootDir,
|
||||
manifestPath,
|
||||
indexPath,
|
||||
indexPath: rootIndexPath,
|
||||
files,
|
||||
contentHash,
|
||||
};
|
||||
|
|
@ -103,21 +178,19 @@ export function buildKnowledgeBaseWorkspaceBundle({
|
|||
|
||||
const KNOWLEDGE_BASE_FILE_LABEL = 'Knowledge base file';
|
||||
|
||||
export async function loadPrebakedKnowledgeBaseBundle({
|
||||
workspace,
|
||||
root,
|
||||
logger,
|
||||
}: MaterializeKnowledgeBaseOptions): Promise<KnowledgeBaseWorkspaceBundle | undefined> {
|
||||
const bundle = buildKnowledgeBaseWorkspaceBundle({ root });
|
||||
export async function loadPrebakedKnowledgeBaseBundle(
|
||||
options: MaterializeKnowledgeBaseOptions,
|
||||
): Promise<KnowledgeBaseWorkspaceBundle | undefined> {
|
||||
const bundle = buildKnowledgeBaseWorkspaceBundle(options);
|
||||
|
||||
return await loadPrebakedWorkspaceBundle({
|
||||
workspace,
|
||||
workspace: options.workspace,
|
||||
manifestPath: bundle.manifestPath,
|
||||
expectedHash: bundle.contentHash,
|
||||
hashField: 'contentHash',
|
||||
schemaVersion: KNOWLEDGE_BASE_MANIFEST_SCHEMA_VERSION,
|
||||
resourceLabel: KNOWLEDGE_BASE_FILE_LABEL,
|
||||
logger,
|
||||
logger: options.logger,
|
||||
invalidManifestLogMessage: 'Ignoring invalid prebaked knowledge base manifest',
|
||||
staleManifestLogMessage: 'Ignoring stale prebaked knowledge base manifest',
|
||||
staleManifestLogKeys: {
|
||||
|
|
@ -126,7 +199,7 @@ export async function loadPrebakedKnowledgeBaseBundle({
|
|||
},
|
||||
successLogMessage: 'Using prebaked knowledge base from workspace',
|
||||
successLogContext: (loadedBundle) => ({
|
||||
root,
|
||||
root: options.root,
|
||||
knowledgeBaseRoot: loadedBundle.rootDir,
|
||||
contentHash: loadedBundle.contentHash,
|
||||
fileCount: loadedBundle.files.size,
|
||||
|
|
@ -143,7 +216,7 @@ export async function materializeKnowledgeBaseIntoWorkspace(
|
|||
resourceLabel: KNOWLEDGE_BASE_FILE_LABEL,
|
||||
logger: options.logger,
|
||||
loadPrebaked: async () => await loadPrebakedKnowledgeBaseBundle(options),
|
||||
buildBundle: () => buildKnowledgeBaseWorkspaceBundle({ root: options.root }),
|
||||
buildBundle: () => buildKnowledgeBaseWorkspaceBundle(options),
|
||||
materializedLogMessage: 'Materialized knowledge base into workspace',
|
||||
materializedLogContext: (bundle) => ({
|
||||
root: options.root,
|
||||
|
|
@ -153,3 +226,8 @@ export async function materializeKnowledgeBaseIntoWorkspace(
|
|||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export type {
|
||||
KnowledgeBaseTemplateEntry,
|
||||
KnowledgeBaseTemplatesIndex,
|
||||
} from './build-templates-index';
|
||||
|
|
|
|||
|
|
@ -5,34 +5,21 @@ import {
|
|||
NATIVE_NODE_PREFERENCE,
|
||||
} from '@n8n/workflow-sdk/prompts/node-selection';
|
||||
|
||||
import { N8N_SANDBOX_WORKSPACE_ROOT } from '@/workspace/sandbox-setup';
|
||||
|
||||
import { SANDBOX_WORKSPACE_SECTION, SUBAGENT_OUTPUT_CONTRACT } from '../../agent/shared-prompts';
|
||||
import { getSandboxWorkspaceSection, SUBAGENT_OUTPUT_CONTRACT } from '../../agent/shared-prompts';
|
||||
|
||||
interface PlannerAgentPromptOptions {
|
||||
sandboxWorkspaceAvailable?: boolean;
|
||||
workspaceRoot?: string;
|
||||
}
|
||||
|
||||
const PLANNER_DISCOVER_WITH_SANDBOX = `2. **Discover** — check what exists and learn best practices. Expect 3–6 tool calls for a typical request:
|
||||
- Read \`${N8N_SANDBOX_WORKSPACE_ROOT}/knowledge-base/best-practices/index.json\`, then grep/read the linked \`.md\` guides for each relevant technique (e.g. "form_input", "scheduling", "data_persistence", "web_app")
|
||||
const PLANNER_DISCOVER_SECTION = `2. **Discover** — check what exists and learn best practices. Expect 3–6 tool calls for a typical request:
|
||||
- \`nodes(action="suggested")\` for the relevant categories
|
||||
- \`data-tables(action="list")\` to check for existing tables
|
||||
- \`credentials(action="list")\` if the request involves external services
|
||||
- Skip searches for nodes you already know exist (webhooks, schedule triggers, data tables, code, set, filter, etc.)`;
|
||||
|
||||
const PLANNER_DISCOVER_WITHOUT_SANDBOX = `2. **Discover** — check what exists. Expect 2–4 tool calls for a typical request:
|
||||
- \`nodes(action="suggested")\` for the relevant categories
|
||||
- \`data-tables(action="list")\` to check for existing tables
|
||||
- \`credentials(action="list")\` if the request involves external services
|
||||
- Use the Node Selection Reference below for best-practice patterns (sandbox knowledge base is unavailable)
|
||||
- Skip searches for nodes you already know exist (webhooks, schedule triggers, data tables, code, set, filter, etc.)`;
|
||||
|
||||
export function getPlannerAgentPrompt(options: PlannerAgentPromptOptions = {}): string {
|
||||
const { sandboxWorkspaceAvailable = false } = options;
|
||||
const sandboxSection = sandboxWorkspaceAvailable ? `\n${SANDBOX_WORKSPACE_SECTION}\n` : '';
|
||||
const discoverSection = sandboxWorkspaceAvailable
|
||||
? PLANNER_DISCOVER_WITH_SANDBOX
|
||||
: PLANNER_DISCOVER_WITHOUT_SANDBOX;
|
||||
const { workspaceRoot } = options;
|
||||
const sandboxSection = workspaceRoot ? `\n${getSandboxWorkspaceSection(workspaceRoot)}\n` : '';
|
||||
|
||||
return `You are the n8n Workflow Planner — you design solution architectures. You do NOT build workflows.
|
||||
|
||||
|
|
@ -58,7 +45,7 @@ ${sandboxSection}
|
|||
- **Use credential-backed resource investigation only when it changes the plan.** You may call \`credentials(action="list")\` so a later resource lookup can validate a resource that affects the architecture (for example checking whether a named Slack channel exists). Do not turn that into a credential-choice question unless the multiple-credentials rule above applies.
|
||||
- **List your assumptions** on your first \`add-plan-item\` call. The user reviews the plan before execution and can reject/correct.
|
||||
|
||||
${discoverSection}
|
||||
${PLANNER_DISCOVER_SECTION}
|
||||
|
||||
## Node Selection Reference
|
||||
|
||||
|
|
|
|||
|
|
@ -176,10 +176,8 @@ function buildPlannerSubAgent(
|
|||
context: OrchestrationContext,
|
||||
tracedPlannerTools: ReturnType<typeof traceSubAgentTools>,
|
||||
subAgentId: string,
|
||||
plannerPrompt: string,
|
||||
) {
|
||||
const plannerPrompt = getPlannerAgentPrompt({
|
||||
sandboxWorkspaceAvailable: Boolean(context.workspace),
|
||||
});
|
||||
const subAgent = new Agent('Workflow Planner Agent')
|
||||
.model(context.modelId)
|
||||
.instructions(plannerPrompt, {
|
||||
|
|
@ -232,15 +230,25 @@ export class PlannerRunCoordinator {
|
|||
|
||||
private readonly subAgent: ReturnType<typeof buildPlannerSubAgent>;
|
||||
|
||||
private readonly plannerPrompt: string;
|
||||
|
||||
// Held as a field so finishTrace/failTrace can finalise the span whether the
|
||||
// run ends in handleTerminalResult or in the handler's catch.
|
||||
private traceRun: Awaited<ReturnType<typeof startSubAgentTrace>>;
|
||||
|
||||
constructor(private readonly context: OrchestrationContext) {
|
||||
this.subAgentId = `agent-planner-${context.runId}`;
|
||||
this.plannerPrompt = getPlannerAgentPrompt({
|
||||
workspaceRoot: context.workspace && context.workspaceRoot ? context.workspaceRoot : undefined,
|
||||
});
|
||||
this.plannerTools = buildPlannerTools(context, this.accumulator);
|
||||
this.tracedPlannerTools = traceSubAgentTools(context, this.plannerTools, 'planner');
|
||||
this.subAgent = buildPlannerSubAgent(context, this.tracedPlannerTools, this.subAgentId);
|
||||
this.subAgent = buildPlannerSubAgent(
|
||||
context,
|
||||
this.tracedPlannerTools,
|
||||
this.subAgentId,
|
||||
this.plannerPrompt,
|
||||
);
|
||||
}
|
||||
|
||||
/** First-call leg: persist the in-flight user message, brief the planner,
|
||||
|
|
@ -289,13 +297,10 @@ export class PlannerRunCoordinator {
|
|||
kind: 'planner',
|
||||
inputs: { guidance, messageCount: messages.length },
|
||||
});
|
||||
const plannerPrompt = getPlannerAgentPrompt({
|
||||
sandboxWorkspaceAvailable: Boolean(context.workspace),
|
||||
});
|
||||
mergeTraceRunInputs(
|
||||
this.traceRun,
|
||||
buildAgentTraceInputs({
|
||||
systemPrompt: plannerPrompt,
|
||||
systemPrompt: this.plannerPrompt,
|
||||
tools: tracedPlannerTools,
|
||||
modelId: context.modelId,
|
||||
}),
|
||||
|
|
@ -350,13 +355,10 @@ export class PlannerRunCoordinator {
|
|||
kind: 'planner',
|
||||
inputs: { resumed: true },
|
||||
});
|
||||
const plannerPrompt = getPlannerAgentPrompt({
|
||||
sandboxWorkspaceAvailable: Boolean(context.workspace),
|
||||
});
|
||||
mergeTraceRunInputs(
|
||||
this.traceRun,
|
||||
buildAgentTraceInputs({
|
||||
systemPrompt: plannerPrompt,
|
||||
systemPrompt: this.plannerPrompt,
|
||||
tools: tracedPlannerTools,
|
||||
modelId: context.modelId,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -723,11 +723,7 @@ export interface InstanceAiContext {
|
|||
nodeService: InstanceAiNodeService;
|
||||
dataTableService: InstanceAiDataTableService;
|
||||
webResearchService?: InstanceAiWebResearchService;
|
||||
/**
|
||||
* Curated workflow-template provider for the sandbox setup. When absent or
|
||||
* when the service returns an empty bundle, the sandbox is created without
|
||||
* an `examples/` directory and the agent operates without template hints.
|
||||
*/
|
||||
/** Curated workflow-template provider — materializes `knowledge-base/templates/` in the sandbox. */
|
||||
templatesService?: BuilderTemplatesService;
|
||||
workspaceService?: InstanceAiWorkspaceService;
|
||||
/**
|
||||
|
|
@ -1238,6 +1234,8 @@ export interface OrchestrationContext {
|
|||
schedulePlannedTasks?: () => Promise<void>;
|
||||
/** Shared runtime workspace for the current orchestration context. */
|
||||
workspace?: Workspace;
|
||||
/** Absolute or host-relative sandbox workspace root for `<workspace_root>` paths in prompts. */
|
||||
workspaceRoot?: string;
|
||||
/** Directories containing node type definition files (.ts) for materializing into sandbox */
|
||||
nodeDefinitionDirs?: string[];
|
||||
/** Native memory store — used to retrieve thread message history for sub-agents. */
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { jsonParse } from 'n8n-workflow';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import type { Mock } from 'vitest';
|
||||
|
||||
import { makeBuilderTemplatesTarGz } from '../../knowledge-base/__tests__/builder-templates-archive.fixtures';
|
||||
import type { InstanceAiContext, SearchableNodeDescription } from '../../types';
|
||||
import type { BuilderTemplatesBundle } from '../builder-templates-service';
|
||||
import type { SandboxWorkspace } from '../sandbox-fs';
|
||||
import type { setupSandboxWorkspace as setupSandboxWorkspaceFunction } from '../sandbox-setup';
|
||||
import { formatNodeCatalogLine, getWorkspaceRoot } from '../sandbox-setup';
|
||||
import { formatNodeCatalogLine, getPromptWorkspaceRoot, getWorkspaceRoot } from '../sandbox-setup';
|
||||
|
||||
type SetupSandboxWorkspace = typeof setupSandboxWorkspaceFunction;
|
||||
type LinkWorkspaceSdkIfEnabled = (
|
||||
|
|
@ -225,10 +225,51 @@ describe('setupSandboxWorkspace', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('never writes examples/ on the local provider even when a bundle is available', async () => {
|
||||
// Local provider is for SDK dev iteration; the agent operates fine without
|
||||
// the curated reference set, so setupSandboxWorkspace must not pay the
|
||||
// per-file/archive write cost here.
|
||||
it('upgrades the knowledge base when sandbox was initialized before templates existed', async () => {
|
||||
const runInSandbox: RunInSandboxMock =
|
||||
vi.fn<
|
||||
(
|
||||
...args: [SandboxWorkspace, string, string?]
|
||||
) => Promise<{ exitCode: number; stdout: string; stderr: string }>
|
||||
>();
|
||||
runInSandbox.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
|
||||
const readFileViaSandbox: ReadFileViaSandboxMock =
|
||||
vi.fn<(...args: [SandboxWorkspace, string]) => Promise<string | null>>();
|
||||
readFileViaSandbox.mockImplementation(async (_workspace, path) => {
|
||||
await Promise.resolve();
|
||||
if (path === '/sandbox/.sandbox-initialized') {
|
||||
return '2024-01-01T00:00:00.000Z';
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const setupSandboxWorkspace = await loadSetupSandboxWorkspaceWithFsMocks(
|
||||
runInSandbox,
|
||||
readFileViaSandbox,
|
||||
);
|
||||
const writeFile = vi.fn<
|
||||
(...args: [string, string | Buffer, { recursive?: boolean }?]) => Promise<void>
|
||||
>(async () => {});
|
||||
|
||||
const bundle: BuilderTemplatesBundle = {
|
||||
archive: makeBuilderTemplatesTarGz([{ name: 'example-workflow.ts', content: 'export {}' }]),
|
||||
version: 'test-sha',
|
||||
};
|
||||
const initialized = await setupSandboxWorkspace(
|
||||
createLocalWorkspace(writeFile),
|
||||
createSetupContext(bundle),
|
||||
);
|
||||
|
||||
expect(initialized).toBe(false);
|
||||
expect(runInSandbox).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'npm install --ignore-scripts',
|
||||
'/sandbox',
|
||||
);
|
||||
const writtenPaths = writeFile.mock.calls.map(([path]) => path);
|
||||
expect(writtenPaths.some((p) => p.includes('/knowledge-base/templates/'))).toBe(true);
|
||||
});
|
||||
|
||||
it('materializes knowledge-base templates on the local provider when a bundle is available', async () => {
|
||||
const runInSandbox: RunInSandboxMock =
|
||||
vi.fn<
|
||||
(
|
||||
|
|
@ -248,17 +289,13 @@ describe('setupSandboxWorkspace', () => {
|
|||
>(async () => {});
|
||||
|
||||
const bundle: BuilderTemplatesBundle = {
|
||||
archive: Buffer.from('opaque-archive-bytes'),
|
||||
archive: makeBuilderTemplatesTarGz([{ name: 'example-workflow.ts', content: 'export {}' }]),
|
||||
version: 'test-sha',
|
||||
};
|
||||
await setupSandboxWorkspace(createLocalWorkspace(writeFile), createSetupContext(bundle));
|
||||
|
||||
const writtenPaths = writeFile.mock.calls.map(([path]) => path);
|
||||
expect(writtenPaths.some((p) => p.includes('/examples/'))).toBe(false);
|
||||
expect(writtenPaths.some((p) => p.endsWith('.templates.tar.gz'))).toBe(false);
|
||||
// `tar` must not be exec'd on the local provider either.
|
||||
const tarInvocations = runInSandbox.mock.calls.filter(([, cmd]) => cmd.includes('tar -xzf'));
|
||||
expect(tarInvocations).toEqual([]);
|
||||
expect(writtenPaths.some((p) => p.includes('/knowledge-base/templates/'))).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects setup file paths that escape the workspace root', async () => {
|
||||
|
|
@ -481,259 +518,6 @@ describe('getWorkspaceRoot', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('writeCuratedExamples', () => {
|
||||
afterEach(() => {
|
||||
vi.doUnmock('../sandbox-fs');
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
type WriteCuratedExamples = (
|
||||
workspace: SandboxWorkspace,
|
||||
bundle: BuilderTemplatesBundle | null,
|
||||
logger?: { debug?: Mock; warn?: Mock },
|
||||
) => Promise<void>;
|
||||
|
||||
type FsMocks = {
|
||||
runInSandbox: RunInSandboxMock;
|
||||
writeFileViaSandbox: Mock<
|
||||
(...args: [SandboxWorkspace, string, string | Buffer]) => Promise<void>
|
||||
>;
|
||||
};
|
||||
|
||||
async function loadWriteCuratedExamples(): Promise<{ fn: WriteCuratedExamples; fs: FsMocks }> {
|
||||
const runInSandbox: RunInSandboxMock =
|
||||
vi.fn<
|
||||
(
|
||||
...args: [SandboxWorkspace, string, string?]
|
||||
) => Promise<{ exitCode: number; stdout: string; stderr: string }>
|
||||
>();
|
||||
runInSandbox.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
|
||||
const writeFileViaSandbox = vi.fn<
|
||||
(...args: [SandboxWorkspace, string, string | Buffer]) => Promise<void>
|
||||
>(async () => {});
|
||||
vi.resetModules();
|
||||
vi.doMock('../sandbox-fs', () => ({
|
||||
runInSandbox,
|
||||
readFileViaSandbox: vi.fn().mockResolvedValue(null),
|
||||
writeFileViaSandbox,
|
||||
escapeSingleQuotes: (value: string) => value.replace(/'/g, "'\\''"),
|
||||
}));
|
||||
|
||||
const loaded = (await import('../sandbox-setup')) as {
|
||||
writeCuratedExamples: WriteCuratedExamples;
|
||||
};
|
||||
return { fn: loaded.writeCuratedExamples, fs: { runInSandbox, writeFileViaSandbox } };
|
||||
}
|
||||
|
||||
function makeDaytonaWorkspace() {
|
||||
const filesystem = {
|
||||
provider: 'daytona' as const,
|
||||
writeFile: vi.fn<(...args: [string, Buffer, { recursive?: boolean }?]) => Promise<void>>(
|
||||
async () => {},
|
||||
),
|
||||
mkdir: vi.fn<(...args: [string, { recursive?: boolean }?]) => Promise<void>>(async () => {}),
|
||||
};
|
||||
const workspace = { filesystem } as unknown as SandboxWorkspace;
|
||||
return { workspace, filesystem };
|
||||
}
|
||||
|
||||
function makeShellOnlyWorkspace(): SandboxWorkspace {
|
||||
// No filesystem property → forces the writeFileViaSandbox fallback.
|
||||
return {} as unknown as SandboxWorkspace;
|
||||
}
|
||||
|
||||
type TarEntry = {
|
||||
name: string;
|
||||
content?: string;
|
||||
typeFlag?: string;
|
||||
linkName?: string;
|
||||
};
|
||||
|
||||
function makeTarGz(entries: TarEntry[]): Buffer {
|
||||
const blocks: Buffer[] = [];
|
||||
for (const entry of entries) {
|
||||
const content = Buffer.from(entry.content ?? '', 'utf-8');
|
||||
const typeFlag = entry.typeFlag ?? '0';
|
||||
const size = typeFlag === '0' ? content.byteLength : 0;
|
||||
const header = Buffer.alloc(512);
|
||||
|
||||
header.write(entry.name, 0, 100, 'utf-8');
|
||||
writeTarOctal(header, 100, 8, 0o644);
|
||||
writeTarOctal(header, 108, 8, 0);
|
||||
writeTarOctal(header, 116, 8, 0);
|
||||
writeTarOctal(header, 124, 12, size);
|
||||
writeTarOctal(header, 136, 12, 0);
|
||||
header.fill(0x20, 148, 156);
|
||||
header.write(typeFlag, 156, 1, 'ascii');
|
||||
if (entry.linkName) header.write(entry.linkName, 157, 100, 'utf-8');
|
||||
header.write('ustar', 257, 5, 'ascii');
|
||||
header.write('00', 263, 2, 'ascii');
|
||||
|
||||
const checksum = header.reduce((sum, byte) => sum + byte, 0);
|
||||
writeTarChecksum(header, checksum);
|
||||
blocks.push(header);
|
||||
|
||||
if (size > 0) {
|
||||
blocks.push(content);
|
||||
const padding = (512 - (size % 512)) % 512;
|
||||
if (padding > 0) blocks.push(Buffer.alloc(padding));
|
||||
}
|
||||
}
|
||||
blocks.push(Buffer.alloc(1024));
|
||||
return gzipSync(Buffer.concat(blocks));
|
||||
}
|
||||
|
||||
function writeTarOctal(buffer: Buffer, offset: number, length: number, value: number): void {
|
||||
const octal = value
|
||||
.toString(8)
|
||||
.padStart(length - 1, '0')
|
||||
.slice(-(length - 1));
|
||||
buffer.write(octal, offset, length - 1, 'ascii');
|
||||
buffer[offset + length - 1] = 0;
|
||||
}
|
||||
|
||||
function writeTarChecksum(buffer: Buffer, checksum: number): void {
|
||||
const octal = checksum.toString(8).padStart(6, '0').slice(-6);
|
||||
buffer.write(octal, 148, 6, 'ascii');
|
||||
buffer[154] = 0;
|
||||
buffer[155] = 0x20;
|
||||
}
|
||||
|
||||
const ARCHIVE = makeTarGz([
|
||||
{ name: 'index.txt', content: 'slack-daily-summary.ts | Daily Slack' },
|
||||
{ name: 'slack-daily-summary.ts', content: 'export default {};' },
|
||||
]);
|
||||
|
||||
it('writes the archive and runs tar on a non-local provider', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const { workspace, filesystem } = makeDaytonaWorkspace();
|
||||
|
||||
await fn(workspace, { archive: ARCHIVE, version: '"v1"' });
|
||||
|
||||
// Filesystem path: mkdir for examples/, then writeFile for the archive.
|
||||
expect(filesystem.mkdir).toHaveBeenCalledWith(expect.stringContaining('/examples'), {
|
||||
recursive: true,
|
||||
});
|
||||
expect(filesystem.writeFile).toHaveBeenCalledWith(
|
||||
expect.stringMatching(/\.templates\.tar\.gz$/),
|
||||
ARCHIVE,
|
||||
{ recursive: true },
|
||||
);
|
||||
|
||||
// tar exec runs exactly once with extract + rm in one shell expression.
|
||||
const tarCalls = fs.runInSandbox.mock.calls.filter(([, cmd]) => cmd.includes('tar -xzf'));
|
||||
expect(tarCalls).toHaveLength(1);
|
||||
expect(tarCalls[0][1]).toMatch(/tar -xzf .* -C .* rm -f .*/);
|
||||
// `status` is a read-only builtin in zsh — assigning to it would
|
||||
// silently drop tar's exit code. Use any other name.
|
||||
expect(tarCalls[0][1]).not.toMatch(/\bstatus=\$\?/);
|
||||
});
|
||||
|
||||
it('falls back to shell writes when the workspace has no filesystem', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const workspace = makeShellOnlyWorkspace();
|
||||
|
||||
await fn(workspace, { archive: ARCHIVE, version: '"v1"' });
|
||||
|
||||
// mkdir is exec'd, then archive written via writeFileViaSandbox, then tar.
|
||||
const mkdirCalls = fs.runInSandbox.mock.calls.filter(([, cmd]) => cmd.startsWith('mkdir -p'));
|
||||
expect(mkdirCalls).toHaveLength(1);
|
||||
|
||||
expect(fs.writeFileViaSandbox).toHaveBeenCalledWith(
|
||||
workspace,
|
||||
expect.stringMatching(/\.templates\.tar\.gz$/),
|
||||
ARCHIVE,
|
||||
);
|
||||
|
||||
const tarCalls = fs.runInSandbox.mock.calls.filter(([, cmd]) => cmd.includes('tar -xzf'));
|
||||
expect(tarCalls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('warns and continues when tar exits non-zero', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
fs.runInSandbox.mockImplementation(async (_, cmd) => {
|
||||
const stderr = cmd.includes('tar -xzf') ? 'tar: bad archive' : '';
|
||||
const exitCode = cmd.includes('tar -xzf') ? 1 : 0;
|
||||
return await Promise.resolve({ exitCode, stdout: '', stderr });
|
||||
});
|
||||
const { workspace } = makeDaytonaWorkspace();
|
||||
const logger = { debug: vi.fn(), warn: vi.fn() };
|
||||
|
||||
// Must not throw.
|
||||
await fn(workspace, { archive: ARCHIVE, version: '"v1"' }, logger);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('failed to extract'),
|
||||
expect.objectContaining({ stderr: 'tar: bad archive' }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each<[string, Buffer]>([
|
||||
['absolute path', makeTarGz([{ name: '/escape.ts', content: 'x' }])],
|
||||
['parent traversal', makeTarGz([{ name: '../escape.ts', content: 'x' }])],
|
||||
['nested path', makeTarGz([{ name: 'nested/template.ts', content: 'x' }])],
|
||||
['symlink entry', makeTarGz([{ name: 'link.ts', typeFlag: '2', linkName: 'target.ts' }])],
|
||||
['hardlink entry', makeTarGz([{ name: 'link.ts', typeFlag: '1', linkName: 'target.ts' }])],
|
||||
['malformed gzip', Buffer.from('not-a-gzip-archive')],
|
||||
])('rejects an archive with %s before writing it', async (_label, archive) => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const { workspace, filesystem } = makeDaytonaWorkspace();
|
||||
const logger = { debug: vi.fn(), warn: vi.fn() };
|
||||
|
||||
await fn(workspace, { archive, version: '"v1"' }, logger);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('rejected curated examples archive'),
|
||||
expect.objectContaining({ archiveVersion: '"v1"' }),
|
||||
);
|
||||
expect(filesystem.mkdir).not.toHaveBeenCalled();
|
||||
expect(filesystem.writeFile).not.toHaveBeenCalled();
|
||||
expect(fs.runInSandbox).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('no-ops when bundle.archive is null', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const { workspace, filesystem } = makeDaytonaWorkspace();
|
||||
|
||||
await fn(workspace, { archive: null, version: null });
|
||||
|
||||
expect(filesystem.writeFile).not.toHaveBeenCalled();
|
||||
expect(fs.runInSandbox).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('no-ops when bundle is null', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const { workspace, filesystem } = makeDaytonaWorkspace();
|
||||
|
||||
await fn(workspace, null);
|
||||
|
||||
expect(filesystem.writeFile).not.toHaveBeenCalled();
|
||||
expect(fs.runInSandbox).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips the local provider even with a non-empty bundle', async () => {
|
||||
const { fn, fs } = await loadWriteCuratedExamples();
|
||||
const writeFile = vi.fn<
|
||||
(...args: [string, string | Buffer, { recursive?: boolean }?]) => Promise<void>
|
||||
>(async () => {});
|
||||
const workspace = {
|
||||
filesystem: {
|
||||
provider: 'local',
|
||||
basePath: '/sandbox',
|
||||
writeFile,
|
||||
mkdir: vi.fn<(...args: [string, { recursive?: boolean }?]) => Promise<void>>(
|
||||
async () => {},
|
||||
),
|
||||
},
|
||||
} as unknown as SandboxWorkspace;
|
||||
|
||||
await fn(workspace, { archive: ARCHIVE, version: '"v1"' });
|
||||
|
||||
expect(writeFile).not.toHaveBeenCalled();
|
||||
expect(fs.runInSandbox).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatNodeCatalogLine', () => {
|
||||
it('should format a basic node with a string version', () => {
|
||||
const node: SearchableNodeDescription = {
|
||||
|
|
@ -856,3 +640,11 @@ describe('formatNodeCatalogLine', () => {
|
|||
expect(result).toBe('n8n-nodes-base.code | Code | Run custom JavaScript code | v2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPromptWorkspaceRoot', () => {
|
||||
it('returns the provider-specific workspace root used in agent prompts', () => {
|
||||
expect(getPromptWorkspaceRoot('daytona')).toBe('/home/daytona/workspace');
|
||||
expect(getPromptWorkspaceRoot('n8n-sandbox')).toBe('/home/user/workspace');
|
||||
expect(getPromptWorkspaceRoot('local')).toBe('.');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -48,6 +48,16 @@ vi.mock('../lazy-daytona', () => ({
|
|||
loadDaytona: () => ({ DaytonaError, DaytonaNotFoundError, Image }),
|
||||
}));
|
||||
|
||||
vi.mock('../builder-templates-service', () => {
|
||||
class MockBuilderTemplatesService {
|
||||
getBundle = vi.fn().mockResolvedValue({ archive: null, version: null });
|
||||
}
|
||||
return {
|
||||
BuilderTemplatesService: MockBuilderTemplatesService,
|
||||
builderTemplatesOptionsFromEnv: vi.fn().mockReturnValue({}),
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
RUNTIME_SKILL_REGISTRY_SCHEMA_VERSION,
|
||||
type RuntimeSkillLinkedFiles,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
/**
|
||||
* Builder templates service: fetches the curated workflow-template bundle from
|
||||
* the n8n-sdk-templates CDN as a single `templates.tar.gz`, caches it on disk,
|
||||
* and hands the raw bytes to the sandbox where `tar -xzf` expands them into
|
||||
* `examples/`. No host-side extraction.
|
||||
* and hands the raw bytes to `buildKnowledgeBaseWorkspaceBundle`, which extracts
|
||||
* them on the host into `knowledge-base/templates/`.
|
||||
*
|
||||
* The archive is produced by `n8n-io/n8n-sdk-templates` and is flat:
|
||||
* - `index.txt` — pipe-delimited catalog used for grep-style lookup
|
||||
* - `<slug>.ts` — one pre-rendered SDK file per publishable template
|
||||
* - `index.json` — structured catalog (preferred)
|
||||
* - `index.txt` — legacy pipe-delimited catalog (converted to `index.json` in workspace)
|
||||
* - `<slug>.ts` — one pre-rendered SDK file per publishable template
|
||||
*
|
||||
* Versioning:
|
||||
* - The companion repo emits one archive per supported SDK minor and
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ class LazyRuntimeFilesystem extends BaseFilesystem {
|
|||
|
||||
return [
|
||||
'Workspace file tools are available and create the runtime workspace on first use.',
|
||||
'Use relative workspace paths unless a loaded skill explicitly provides an absolute workspace path.',
|
||||
'Paths are relative to the workspace root unless you pass an absolute path under that root.',
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,14 +19,21 @@
|
|||
* workflow.ts # agent writes main workflow here
|
||||
* chunks/
|
||||
* *.ts # reusable node/workflow modules
|
||||
* knowledge-base/
|
||||
* index.json # combined catalog of guides and templates
|
||||
* best-practices/
|
||||
* index.json # technique guide catalog
|
||||
* *.md # guide content per technique
|
||||
* templates/
|
||||
* index.json # curated template catalog
|
||||
* *.ts # SDK workflow examples
|
||||
*/
|
||||
|
||||
import { createRequire } from 'node:module';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
|
||||
import type { Logger } from '../logger';
|
||||
import type { InstanceAiContext, SearchableNodeDescription } from '../types';
|
||||
import type { BuilderTemplatesBundle } from './builder-templates-service';
|
||||
import type { SandboxProvider } from './create-workspace';
|
||||
import {
|
||||
isLinkWorkspaceSdkEnabled,
|
||||
packWorkspaceSdk,
|
||||
|
|
@ -48,16 +55,11 @@ const NOOP_LOGGER: Logger = {
|
|||
error: () => {},
|
||||
debug: () => {},
|
||||
};
|
||||
const TAR_BLOCK_SIZE = 512;
|
||||
const TAR_TYPE_REGULAR = '0';
|
||||
const TEMPLATE_ENTRY_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*\.ts$/;
|
||||
|
||||
type SandboxWorkspaceSetupStep =
|
||||
| 'resolve-workspace-root'
|
||||
| 'read-initialization-marker'
|
||||
| 'list-node-types'
|
||||
| 'write-workspace-files'
|
||||
| 'write-curated-examples'
|
||||
| 'materialize-knowledge-base'
|
||||
| 'install-dependencies'
|
||||
| 'link-workspace-sdk'
|
||||
|
|
@ -87,12 +89,32 @@ async function setupStep<T>(step: SandboxWorkspaceSetupStep, action: () => Promi
|
|||
|
||||
export const WORKSPACE_DIR = 'workspace';
|
||||
|
||||
/** Default home directory inside the Daytona sandbox container. */
|
||||
export const DAYTONA_HOME = '/home/daytona';
|
||||
|
||||
/** Absolute workspace root inside the Daytona sandbox container. */
|
||||
export const DAYTONA_WORKSPACE_ROOT = `${DAYTONA_HOME}/${WORKSPACE_DIR}`;
|
||||
|
||||
/** Default home directory inside the n8n sandbox service container. */
|
||||
export const N8N_SANDBOX_HOME = '/home/user';
|
||||
|
||||
/** Absolute workspace root for n8n sandbox service Dockerfile steps (build-time). */
|
||||
/** Absolute workspace root inside the n8n sandbox service container. */
|
||||
export const N8N_SANDBOX_WORKSPACE_ROOT = `${N8N_SANDBOX_HOME}/${WORKSPACE_DIR}`;
|
||||
|
||||
/** Resolve the `<workspace_root>` path shown in agent system prompts for a sandbox provider. */
|
||||
export function getPromptWorkspaceRoot(provider: SandboxProvider): string {
|
||||
switch (provider) {
|
||||
case 'daytona':
|
||||
return DAYTONA_WORKSPACE_ROOT;
|
||||
case 'n8n-sandbox':
|
||||
return N8N_SANDBOX_WORKSPACE_ROOT;
|
||||
case 'local':
|
||||
// Local workspaces are already scoped to the resolved root; use `.` so
|
||||
// paths like `./knowledge-base/...` resolve under `<root>/`, not `<root>/workspace/`.
|
||||
return '.';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a dependency's installed version from the host's node_modules.
|
||||
* Falls back to `'*'` if the package is unresolvable (should only happen in
|
||||
|
|
@ -465,6 +487,22 @@ async function initializeLazyFilesystem(workspace: SandboxWorkspace): Promise<vo
|
|||
await filesystem.init?.();
|
||||
}
|
||||
|
||||
async function materializeKnowledgeBaseStep(
|
||||
workspace: SandboxWorkspace,
|
||||
root: string,
|
||||
context: InstanceAiContext,
|
||||
): Promise<void> {
|
||||
await setupStep('materialize-knowledge-base', async () => {
|
||||
const templatesBundle = (await context.templatesService?.getBundle()) ?? null;
|
||||
await materializeKnowledgeBaseIntoWorkspace({
|
||||
workspace,
|
||||
root,
|
||||
logger: context.logger,
|
||||
templatesArchive: templatesBundle?.archive ?? null,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function getWorkspaceRoot(workspace: SandboxWorkspace): Promise<string> {
|
||||
const cached = workspaceRootCache.get(workspace);
|
||||
if (cached) return cached;
|
||||
|
|
@ -483,177 +521,12 @@ export async function getWorkspaceRoot(workspace: SandboxWorkspace): Promise<str
|
|||
}
|
||||
|
||||
const result = await runInSandbox(workspace, 'echo $HOME');
|
||||
const home = result.stdout.trim() || '/home/daytona';
|
||||
const home = result.stdout.trim() || DAYTONA_HOME;
|
||||
const root = `${home}/${WORKSPACE_DIR}`;
|
||||
workspaceRootCache.set(workspace, root);
|
||||
return root;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the exact archive shape published by n8n-sdk-templates before the
|
||||
* sandbox ever sees the bytes. This is intentionally narrow: a gzip-wrapped tar
|
||||
* with only regular top-level files (`index.txt` and `<slug>.ts`). Rejecting
|
||||
* everything else prevents path traversal, symlink/hardlink writes, and nested
|
||||
* output when the sandbox later runs `tar -xzf`.
|
||||
*/
|
||||
function validateBuilderTemplatesArchive(archive: Buffer): string | null {
|
||||
let tar: Buffer;
|
||||
try {
|
||||
tar = gunzipSync(archive);
|
||||
} catch (error) {
|
||||
return `failed to gunzip archive: ${getErrorMessage(error)}`;
|
||||
}
|
||||
|
||||
let offset = 0;
|
||||
while (offset + TAR_BLOCK_SIZE <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + TAR_BLOCK_SIZE);
|
||||
// A zero header marks the end of a tar archive. We do not require the
|
||||
// optional second zero block because `tar` itself accepts archives with
|
||||
// one terminator, and this is only a preflight guard before extraction.
|
||||
if (isZeroBlock(header)) return null;
|
||||
|
||||
// USTAR stores long path components as `prefix` + `name`. Combining them
|
||||
// before validation ensures nested or absolute paths cannot hide in either
|
||||
// field independently.
|
||||
const name = readTarString(header, 0, 100);
|
||||
const prefix = readTarString(header, 345, 155);
|
||||
const entryName = prefix ? `${prefix}/${name}` : name;
|
||||
const typeFlag = readTarString(header, 156, 1);
|
||||
const size = parseTarOctal(header, 124, 12);
|
||||
|
||||
if (size === null) return `invalid size for archive entry "${entryName}"`;
|
||||
// Empty type is the old tar spelling for a regular file; `0` is the USTAR
|
||||
// spelling. All other types include directories, symlinks, hardlinks, and
|
||||
// metadata extensions, none of which belong in the curated bundle.
|
||||
if (typeFlag !== '' && typeFlag !== TAR_TYPE_REGULAR) {
|
||||
return `unsupported archive entry type "${typeFlag}" for "${entryName}"`;
|
||||
}
|
||||
if (!isAllowedTemplateEntryName(entryName)) {
|
||||
return `unsupported archive entry path "${entryName}"`;
|
||||
}
|
||||
|
||||
// Tar payloads are padded to 512-byte blocks, so jump over the file content
|
||||
// plus padding to land exactly on the next header.
|
||||
const dataBlocks = Math.ceil(size / TAR_BLOCK_SIZE);
|
||||
offset += TAR_BLOCK_SIZE + dataBlocks * TAR_BLOCK_SIZE;
|
||||
}
|
||||
|
||||
return offset === tar.length ? null : 'trailing partial tar header';
|
||||
}
|
||||
|
||||
function isAllowedTemplateEntryName(name: string): boolean {
|
||||
if (name === 'index.txt') return true;
|
||||
return TEMPLATE_ENTRY_PATTERN.test(name);
|
||||
}
|
||||
|
||||
function isZeroBlock(block: Buffer): boolean {
|
||||
return block.every((byte) => byte === 0);
|
||||
}
|
||||
|
||||
function readTarString(block: Buffer, start: number, length: number): string {
|
||||
const field = block.subarray(start, start + length);
|
||||
const nullIndex = field.indexOf(0);
|
||||
return field.subarray(0, nullIndex === -1 ? field.length : nullIndex).toString('utf-8');
|
||||
}
|
||||
|
||||
function parseTarOctal(block: Buffer, start: number, length: number): number | null {
|
||||
const raw = readTarString(block, start, length).trim();
|
||||
if (!/^[0-7]+$/.test(raw)) return null;
|
||||
const parsed = Number.parseInt(raw, 8);
|
||||
return Number.isSafeInteger(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the curated workflow examples archive into `${root}/examples/`.
|
||||
*
|
||||
* Used by the Daytona / n8n-sandbox factory paths. The local provider
|
||||
* deliberately skips this — dev iteration on the SDK doesn't need the
|
||||
* curated reference set, and the agent there operates fine without it
|
||||
* (same fallback as a cold start with the CDN unreachable).
|
||||
*
|
||||
* The CDN payload is a flat `.tar.gz` of `<slug>.ts` + `index.txt`. We
|
||||
* write the bytes into the sandbox and run `tar -xzf` in-sandbox to
|
||||
* expand them into `examples/` — far cheaper than 100+ individual
|
||||
* `writeFile` round-trips for remote providers. The archive file is
|
||||
* removed after extraction so it doesn't leak into the agent's view.
|
||||
*
|
||||
* No-op when the bundle is empty (e.g. `templatesService` was not
|
||||
* configured, or the CDN fetch failed and there was no disk cache).
|
||||
*/
|
||||
export async function writeCuratedExamples(
|
||||
workspace: SandboxWorkspace,
|
||||
bundle: BuilderTemplatesBundle | null,
|
||||
logger?: Logger,
|
||||
): Promise<void> {
|
||||
if (!bundle?.archive) return;
|
||||
|
||||
if (workspace.filesystem?.provider === 'local') {
|
||||
logger?.debug('[sandbox-setup] skipping curated examples for local provider');
|
||||
return;
|
||||
}
|
||||
|
||||
// Defense-in-depth for the curated CDN bundle. This validates the narrow
|
||||
// archive shape we publish, not arbitrary user-supplied tar files.
|
||||
const validationError = validateBuilderTemplatesArchive(bundle.archive);
|
||||
if (validationError) {
|
||||
logger?.warn('[sandbox-setup] rejected curated examples archive', {
|
||||
error: validationError,
|
||||
archiveBytes: bundle.archive.byteLength,
|
||||
archiveVersion: bundle.version,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
const root = await getWorkspaceRoot(workspace);
|
||||
const archivePath = `${root}/.templates.tar.gz`;
|
||||
const examplesDir = `${root}/examples`;
|
||||
|
||||
if (workspace.filesystem) {
|
||||
await workspace.filesystem.mkdir(examplesDir, { recursive: true });
|
||||
await workspace.filesystem.writeFile(archivePath, bundle.archive, { recursive: true });
|
||||
} else {
|
||||
const mkdirResult = await runInSandbox(
|
||||
workspace,
|
||||
`mkdir -p '${escapeSingleQuotes(examplesDir)}'`,
|
||||
);
|
||||
if (mkdirResult.exitCode !== 0) {
|
||||
logger?.warn('[sandbox-setup] failed to create examples/ dir', {
|
||||
stderr: mkdirResult.stderr,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await writeFileViaSandbox(workspace, archivePath, bundle.archive);
|
||||
}
|
||||
|
||||
// Extract and clean up in one command so a partial state isn't left
|
||||
// behind if `tar` exits non-zero. `rm -f` is always run; the exec's
|
||||
// status is `tar`'s exit code. `2>&1` folds tar's stderr into stdout so
|
||||
// the failure cause is still visible if the sandbox runtime drops stderr.
|
||||
// Avoid the variable name `status` — it's a read-only builtin in zsh.
|
||||
const extract = await runInSandbox(
|
||||
workspace,
|
||||
`tar -xzf '${escapeSingleQuotes(archivePath)}' -C '${escapeSingleQuotes(examplesDir)}' 2>&1; rc=$?; rm -f '${escapeSingleQuotes(archivePath)}'; exit $rc`,
|
||||
);
|
||||
if (extract.exitCode !== 0) {
|
||||
logger?.warn('[sandbox-setup] failed to extract curated examples', {
|
||||
exitCode: extract.exitCode,
|
||||
stderr: extract.stderr,
|
||||
stdout: extract.stdout,
|
||||
archivePath,
|
||||
archiveBytes: bundle.archive.byteLength,
|
||||
archiveVersion: bundle.version,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
logger?.debug('[sandbox-setup] prepared curated examples', {
|
||||
bytes: bundle.archive.byteLength,
|
||||
version: bundle.version,
|
||||
durationMs: Date.now() - start,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the sandbox workspace for the workflow builder agent.
|
||||
* Idempotent — skips if already initialized (checks marker file).
|
||||
|
|
@ -677,7 +550,10 @@ export async function setupSandboxWorkspace(
|
|||
'read-initialization-marker',
|
||||
async () => await readFileViaSandbox(workspace, markerFile),
|
||||
);
|
||||
if (marker !== null) return false;
|
||||
if (marker !== null) {
|
||||
await materializeKnowledgeBaseStep(workspace, root, context);
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Collect all files ──────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -723,22 +599,7 @@ export async function setupSandboxWorkspace(
|
|||
'write-workspace-files',
|
||||
async () => await writeWorkspaceFiles(workspace, root, files),
|
||||
);
|
||||
await setupStep(
|
||||
'write-curated-examples',
|
||||
async () =>
|
||||
await writeCuratedExamples(
|
||||
workspace,
|
||||
(await context.templatesService?.getBundle()) ?? null,
|
||||
context.logger,
|
||||
),
|
||||
);
|
||||
await setupStep('materialize-knowledge-base', async () => {
|
||||
await materializeKnowledgeBaseIntoWorkspace({
|
||||
workspace,
|
||||
root,
|
||||
logger: context.logger,
|
||||
});
|
||||
});
|
||||
await materializeKnowledgeBaseStep(workspace, root, context);
|
||||
|
||||
// npm install (must run after package.json is in place)
|
||||
await setupStep('install-dependencies', async () => {
|
||||
|
|
|
|||
|
|
@ -27,7 +27,11 @@ import {
|
|||
type KnowledgeBaseWorkspaceBundle,
|
||||
} from '../knowledge-base/materialize-knowledge-base';
|
||||
import type { ErrorReporter, Logger } from '../logger';
|
||||
import { PACKAGE_JSON, TSCONFIG_JSON, BUILD_MJS } from './sandbox-setup';
|
||||
import {
|
||||
BuilderTemplatesService,
|
||||
builderTemplatesOptionsFromEnv,
|
||||
} from './builder-templates-service';
|
||||
import { DAYTONA_WORKSPACE_ROOT, PACKAGE_JSON, TSCONFIG_JSON, BUILD_MJS } from './sandbox-setup';
|
||||
import { disposeSnapshotImageContext, stageWorkspaceFilesForImage } from './snapshot-image-context';
|
||||
import { buildRuntimeSkillWorkspaceBundle } from '../skills/materialize-runtime-skills';
|
||||
import { loadInstanceAiRuntimeSkillSource } from '../skills/runtime-skills';
|
||||
|
|
@ -39,7 +43,6 @@ export interface CreateSnapshotOptions {
|
|||
onLogs?: (chunk: string) => void;
|
||||
}
|
||||
|
||||
const DAYTONA_WORKSPACE_ROOT = '/home/daytona/workspace';
|
||||
const DAYTONA_WORKSPACE_BAKE_ROOT = '/tmp/n8n-workspace-bake';
|
||||
const SNAPSHOT_WORKSPACE_LAYOUT_DIRS = ['src', 'chunks', 'node-types'] as const;
|
||||
const EMPTY_RUNTIME_SKILLS_HASH = '000000000000';
|
||||
|
|
@ -62,7 +65,7 @@ export class SnapshotManager {
|
|||
private runtimeSkillBundlePromise: ReturnType<typeof buildRuntimeSkillWorkspaceBundle> | null =
|
||||
null;
|
||||
|
||||
private knowledgeBaseBundleCache: KnowledgeBaseWorkspaceBundle | null = null;
|
||||
private knowledgeBaseBundlePromise: Promise<KnowledgeBaseWorkspaceBundle> | null = null;
|
||||
|
||||
private stagingDir: string | null = null;
|
||||
|
||||
|
|
@ -72,6 +75,7 @@ export class SnapshotManager {
|
|||
private readonly n8nVersion: string | undefined,
|
||||
private readonly errorReporter?: ErrorReporter,
|
||||
private readonly runtimeSkillSource?: RuntimeSkillSource,
|
||||
private readonly templatesService?: BuilderTemplatesService,
|
||||
) {}
|
||||
|
||||
/** Get or prepare the image descriptor. */
|
||||
|
|
@ -83,7 +87,7 @@ export class SnapshotManager {
|
|||
private async prepareImage(): Promise<Image> {
|
||||
const base = this.baseImage ?? 'daytonaio/sandbox:0.5.0';
|
||||
const runtimeSkillBundle = await this.runtimeSkillBundle();
|
||||
const knowledgeBaseBundle = this.knowledgeBaseBundle();
|
||||
const knowledgeBaseBundle = await this.knowledgeBaseBundle();
|
||||
const cacheKey = await this.snapshotSuffix();
|
||||
|
||||
const workspaceFiles = new Map<string, string>([
|
||||
|
|
@ -192,7 +196,7 @@ export class SnapshotManager {
|
|||
private async snapshotSuffix(): Promise<string> {
|
||||
this.snapshotSuffixPromise ??= (async () => {
|
||||
const runtimeSkillBundle = await this.runtimeSkillBundle();
|
||||
const knowledgeBaseBundle = this.knowledgeBaseBundle();
|
||||
const knowledgeBaseBundle = await this.knowledgeBaseBundle();
|
||||
const skillsHash = runtimeSkillBundle?.skillsHash ?? EMPTY_RUNTIME_SKILLS_HASH;
|
||||
const knowledgeBaseHash = knowledgeBaseBundle.contentHash ?? EMPTY_KNOWLEDGE_BASE_HASH;
|
||||
return `${skillsHash}-${knowledgeBaseHash}`;
|
||||
|
|
@ -211,12 +215,22 @@ export class SnapshotManager {
|
|||
return await this.runtimeSkillBundlePromise;
|
||||
}
|
||||
|
||||
private knowledgeBaseBundle(): KnowledgeBaseWorkspaceBundle {
|
||||
this.knowledgeBaseBundleCache ??= buildKnowledgeBaseWorkspaceBundle({
|
||||
root: DAYTONA_WORKSPACE_ROOT,
|
||||
});
|
||||
private async knowledgeBaseBundle(): Promise<KnowledgeBaseWorkspaceBundle> {
|
||||
this.knowledgeBaseBundlePromise ??= this.buildKnowledgeBaseBundle();
|
||||
return await this.knowledgeBaseBundlePromise;
|
||||
}
|
||||
|
||||
return this.knowledgeBaseBundleCache;
|
||||
private async buildKnowledgeBaseBundle(): Promise<KnowledgeBaseWorkspaceBundle> {
|
||||
const templatesService =
|
||||
this.templatesService ??
|
||||
new BuilderTemplatesService(builderTemplatesOptionsFromEnv({ logger: this.logger }));
|
||||
const templatesBundle = await templatesService.getBundle();
|
||||
|
||||
return buildKnowledgeBaseWorkspaceBundle({
|
||||
root: DAYTONA_WORKSPACE_ROOT,
|
||||
templatesArchive: templatesBundle.archive,
|
||||
logger: this.logger,
|
||||
});
|
||||
}
|
||||
|
||||
private async snapshotName(): Promise<string> {
|
||||
|
|
@ -233,7 +247,7 @@ export class SnapshotManager {
|
|||
this.snapshotPromise = null;
|
||||
this.snapshotSuffixPromise = null;
|
||||
this.runtimeSkillBundlePromise = null;
|
||||
this.knowledgeBaseBundleCache = null;
|
||||
this.knowledgeBaseBundlePromise = null;
|
||||
this.stagingDir = null;
|
||||
if (stagingDir) {
|
||||
void disposeSnapshotImageContext(stagingDir);
|
||||
|
|
|
|||
|
|
@ -31,11 +31,11 @@ function makeOpts(): {
|
|||
}
|
||||
|
||||
describe('createTemplateTelemetrySession', () => {
|
||||
it('emits a search event when grep targets examples/index.txt', () => {
|
||||
it('emits a search event when grep targets knowledge-base/templates/index.json', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.observe(
|
||||
'grep -i "slack" /home/sandbox/workspace/examples/index.txt',
|
||||
'grep -i "slack" /home/sandbox/workspace/knowledge-base/templates/index.json',
|
||||
'slack-daily-summary.ts | Daily Slack ...\nother.ts | ...\n',
|
||||
);
|
||||
|
||||
|
|
@ -54,11 +54,11 @@ describe('createTemplateTelemetrySession', () => {
|
|||
expect(calls.find((c) => c.name === 'Builder template search')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('emits a read event when cat targets examples/<file>.ts', () => {
|
||||
it('emits a read event when cat targets knowledge-base/templates/<file>.ts', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.observe(
|
||||
'cat /home/sandbox/workspace/examples/slack-daily-summary.ts',
|
||||
'cat /home/sandbox/workspace/knowledge-base/templates/slack-daily-summary.ts',
|
||||
'/* file content here */',
|
||||
);
|
||||
|
||||
|
|
@ -71,8 +71,11 @@ describe('createTemplateTelemetrySession', () => {
|
|||
it('handles head/sed/less/more in addition to cat', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.observe('head -50 /workspace/examples/slack-daily-summary.ts', 'lines');
|
||||
session.observe('sed -n 1,30p /workspace/examples/build-your-first-ai-agent-6270.ts', 'lines');
|
||||
session.observe('head -50 /workspace/knowledge-base/templates/slack-daily-summary.ts', 'lines');
|
||||
session.observe(
|
||||
'sed -n 1,30p /workspace/knowledge-base/templates/build-your-first-ai-agent-6270.ts',
|
||||
'lines',
|
||||
);
|
||||
|
||||
const reads = calls.filter((c) => c.name === 'Builder template read');
|
||||
expect(reads.length).toBe(2);
|
||||
|
|
@ -88,9 +91,9 @@ describe('createTemplateTelemetrySession', () => {
|
|||
it('emits a session rollup on flush', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.observe('grep -i "slack" /workspace/examples/index.txt', 'a\nb\n');
|
||||
session.observe('cat /workspace/examples/slack-daily-summary.ts', 'X');
|
||||
session.observe('cat /workspace/examples/slack-daily-summary.ts', 'X');
|
||||
session.observe('grep -i "slack" /workspace/knowledge-base/templates/index.json', 'a\nb\n');
|
||||
session.observe('cat /workspace/knowledge-base/templates/slack-daily-summary.ts', 'X');
|
||||
session.observe('cat /workspace/knowledge-base/templates/slack-daily-summary.ts', 'X');
|
||||
session.flush();
|
||||
|
||||
const rollup = calls.find((c) => c.name === 'Builder template session');
|
||||
|
|
@ -115,7 +118,7 @@ describe('createTemplateTelemetrySession', () => {
|
|||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.flush();
|
||||
session.observe('cat /workspace/examples/foo.ts', 'X');
|
||||
session.observe('cat /workspace/knowledge-base/templates/foo.ts', 'X');
|
||||
session.flush();
|
||||
expect(calls.filter((c) => c.name === 'Builder template session').length).toBe(1);
|
||||
expect(calls.filter((c) => c.name === 'Builder template read').length).toBe(0);
|
||||
|
|
@ -167,21 +170,23 @@ describe('createTemplateTelemetrySession', () => {
|
|||
|
||||
describe('extractGrepQuery', () => {
|
||||
it('extracts double-quoted patterns', () => {
|
||||
expect(extractGrepQuery('grep -i "slack post" examples/index.txt')).toBe('slack post');
|
||||
expect(extractGrepQuery('grep -i "slack post" knowledge-base/templates/index.json')).toBe(
|
||||
'slack post',
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts single-quoted patterns', () => {
|
||||
expect(extractGrepQuery("grep -i 'gmail' examples/index.txt")).toBe('gmail');
|
||||
expect(extractGrepQuery("grep -i 'gmail' knowledge-base/templates/index.json")).toBe('gmail');
|
||||
});
|
||||
|
||||
it('extracts bare patterns when unquoted', () => {
|
||||
expect(extractGrepQuery('grep slack examples/index.txt')).toBe('slack');
|
||||
expect(extractGrepQuery('grep -i slack examples/index.txt')).toBe('slack');
|
||||
expect(extractGrepQuery('grep slack knowledge-base/templates/index.json')).toBe('slack');
|
||||
expect(extractGrepQuery('grep -i slack knowledge-base/templates/index.json')).toBe('slack');
|
||||
});
|
||||
|
||||
it('caps the query at 200 characters', () => {
|
||||
const long = 'x'.repeat(300);
|
||||
const result = extractGrepQuery(`grep -i "${long}" examples/index.txt`);
|
||||
const result = extractGrepQuery(`grep -i "${long}" knowledge-base/templates/index.json`);
|
||||
expect(result.length).toBe(200);
|
||||
});
|
||||
|
||||
|
|
@ -265,7 +270,7 @@ describe('observeTypedRead / observeTypedSearch', () => {
|
|||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
session.observe(
|
||||
'grep -i "sk-proj-abcdef0123456789xyz" /workspace/examples/index.txt',
|
||||
'grep -i "sk-proj-abcdef0123456789xyz" /workspace/knowledge-base/templates/index.json',
|
||||
'nothing\n',
|
||||
);
|
||||
|
||||
|
|
@ -306,19 +311,20 @@ describe('createTypedToolObserver', () => {
|
|||
};
|
||||
}
|
||||
|
||||
it('emits typed read for workspace_read_file targeting examples/<slug>.ts', () => {
|
||||
it('emits typed read for workspace_read_file targeting knowledge-base/templates/<slug>.ts', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(
|
||||
toolCall('tc-1', 'workspace_read_file', {
|
||||
path: '/workspace/examples/slack-daily-summary.ts',
|
||||
path: '/workspace/knowledge-base/templates/slack-daily-summary.ts',
|
||||
}),
|
||||
);
|
||||
observe(
|
||||
toolResult('tc-1', {
|
||||
content: '/workspace/examples/slack-daily-summary.ts (200 bytes)\nfile content here\n',
|
||||
content:
|
||||
'/workspace/knowledge-base/templates/slack-daily-summary.ts (200 bytes)\nfile content here\n',
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
@ -333,7 +339,11 @@ describe('createTypedToolObserver', () => {
|
|||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-legacy-read', 'workspace_read_file', { path: 'examples/foo.ts' }));
|
||||
observe(
|
||||
toolCall('tc-legacy-read', 'workspace_read_file', {
|
||||
path: 'knowledge-base/templates/foo.ts',
|
||||
}),
|
||||
);
|
||||
observe(toolResult('tc-legacy-read', 'file content here'));
|
||||
|
||||
const read = calls.find((c) => c.name === 'Builder template read');
|
||||
|
|
@ -342,15 +352,18 @@ describe('createTypedToolObserver', () => {
|
|||
expect(read!.props.bytes_read).toBe('file content here'.length);
|
||||
});
|
||||
|
||||
it('emits typed search for workspace_grep targeting examples/', () => {
|
||||
it('emits typed search for workspace_grep targeting knowledge-base/templates/', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-2', 'workspace_grep', { pattern: 'slack', path: 'examples/' }));
|
||||
observe(
|
||||
toolCall('tc-2', 'workspace_grep', { pattern: 'slack', path: 'knowledge-base/templates/' }),
|
||||
);
|
||||
observe(
|
||||
toolResult('tc-2', {
|
||||
content: 'examples/a.ts:1:1: slack\nexamples/b.ts:5:1: slack\n',
|
||||
content:
|
||||
'knowledge-base/templates/a.ts:1:1: slack\nknowledge-base/templates/b.ts:5:1: slack\n',
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
@ -360,19 +373,24 @@ describe('createTypedToolObserver', () => {
|
|||
expect(search!.props.result_count).toBe(2);
|
||||
});
|
||||
|
||||
it('emits typed search for grep targeting examples/index.txt', () => {
|
||||
it('emits typed search for grep targeting knowledge-base/templates/index.json', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-3', 'workspace_grep', { pattern: 'slack', path: 'examples/index.txt' }));
|
||||
observe(
|
||||
toolCall('tc-3', 'workspace_grep', {
|
||||
pattern: 'slack',
|
||||
path: 'knowledge-base/templates/index.json',
|
||||
}),
|
||||
);
|
||||
observe(toolResult('tc-3', 'slack-daily.ts | Daily Slack\nslack-onboard.ts | Onboard\n'));
|
||||
|
||||
const search = calls.find((c) => c.name === 'Builder template search');
|
||||
expect(search!.props.result_count).toBe(2);
|
||||
});
|
||||
|
||||
it('ignores read_file outside examples/', () => {
|
||||
it('ignores read_file outside knowledge-base/templates/', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
|
@ -383,7 +401,7 @@ describe('createTypedToolObserver', () => {
|
|||
expect(calls.find((c) => c.name === 'Builder template read')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores grep that does not target examples/', () => {
|
||||
it('ignores grep that does not target knowledge-base/templates/', () => {
|
||||
const { opts, calls } = makeOpts();
|
||||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
|
@ -399,7 +417,12 @@ describe('createTypedToolObserver', () => {
|
|||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-z', 'workspace_write_file', { path: 'examples/foo.ts', content: 'x' }));
|
||||
observe(
|
||||
toolCall('tc-z', 'workspace_write_file', {
|
||||
path: 'knowledge-base/templates/foo.ts',
|
||||
content: 'x',
|
||||
}),
|
||||
);
|
||||
observe(toolResult('tc-z', 'ok'));
|
||||
|
||||
expect(calls.find((c) => c.name === 'Builder template read')).toBeUndefined();
|
||||
|
|
@ -411,7 +434,7 @@ describe('createTypedToolObserver', () => {
|
|||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-e', 'workspace_read_file', { path: 'examples/foo.ts' }));
|
||||
observe(toolCall('tc-e', 'workspace_read_file', { path: 'knowledge-base/templates/foo.ts' }));
|
||||
observe(toolError('tc-e', 'permission denied'));
|
||||
|
||||
expect(calls.find((c) => c.name === 'Builder template read')).toBeUndefined();
|
||||
|
|
@ -422,7 +445,7 @@ describe('createTypedToolObserver', () => {
|
|||
const session = createTemplateTelemetrySession(opts);
|
||||
const observe = createTypedToolObserver(session);
|
||||
|
||||
observe(toolCall('tc-n', 'workspace_read_file', { path: 'examples/foo.ts' }));
|
||||
observe(toolCall('tc-n', 'workspace_read_file', { path: 'knowledge-base/templates/foo.ts' }));
|
||||
observe(toolResult('tc-n', { not: 'a string' }));
|
||||
|
||||
expect(calls.find((c) => c.name === 'Builder template read')).toBeUndefined();
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
/**
|
||||
* Template usage telemetry for the builder agent.
|
||||
*
|
||||
* Pattern-detects template-related shell commands (grep on `examples/index.txt`,
|
||||
* cat/head/sed on `examples/*.ts`) and emits three event types via the existing
|
||||
* Pattern-detects template-related shell commands (grep on
|
||||
* `knowledge-base/templates/index.json`, cat/head/sed on
|
||||
* `knowledge-base/templates/*.ts`) and emits three event types via the existing
|
||||
* `context.trackTelemetry?.(name, props)` channel:
|
||||
*
|
||||
* - `Builder template search` — fired per index grep, with the search query
|
||||
|
|
@ -21,11 +22,9 @@ import type { OrchestrationContext } from '../types';
|
|||
const MAX_QUERY_LENGTH = 200;
|
||||
const MAX_USER_REQUEST_LENGTH = 120;
|
||||
|
||||
const SEARCH_PATTERN = /\bgrep\b[^|]*\bexamples\/index\.txt\b/;
|
||||
const SEARCH_PATTERN = /\bgrep\b[^|]*\bknowledge-base\/templates\/index\.json\b/;
|
||||
const READ_COMMAND_HEADS = ['cat', 'head', 'tail', 'sed', 'less', 'more'];
|
||||
// Match `examples/<slug>.ts` anywhere in the path. `\b` boundaries reject
|
||||
// `someotherexamples/...` while accepting `/abs/path/examples/foo.ts`.
|
||||
const READ_FILE_PATTERN = /\bexamples\/([a-zA-Z0-9._-]+\.ts)\b/;
|
||||
const READ_FILE_PATTERN = /\bknowledge-base\/templates\/([a-zA-Z0-9._-]+\.ts)\b/;
|
||||
|
||||
export interface TemplateTelemetrySession {
|
||||
/** Inspect a command + its stdout; emits search/read events when patterns match. */
|
||||
|
|
@ -97,7 +96,7 @@ export function createTemplateTelemetrySession(
|
|||
function observe(command: string, stdout: string): void {
|
||||
if (!open) return;
|
||||
|
||||
// Search detection: any grep at examples/index.txt
|
||||
// Search detection: any grep at knowledge-base/templates/index.json
|
||||
if (SEARCH_PATTERN.test(command)) {
|
||||
searchCount++;
|
||||
emit('Builder template search', {
|
||||
|
|
@ -106,7 +105,7 @@ export function createTemplateTelemetrySession(
|
|||
});
|
||||
}
|
||||
|
||||
// Read detection: cat/head/sed/etc. against examples/*.ts
|
||||
// Read detection: cat/head/sed/etc. against knowledge-base/templates/*.ts
|
||||
const head = command.trim().split(/\s+/, 1)[0]?.split('/').pop() ?? '';
|
||||
if (READ_COMMAND_HEADS.includes(head)) {
|
||||
const match = command.match(READ_FILE_PATTERN);
|
||||
|
|
@ -211,9 +210,7 @@ export function getTemplateTelemetrySession(
|
|||
const TYPED_READ_TOOL = 'workspace_read_file';
|
||||
const TYPED_GREP_TOOL = 'workspace_grep';
|
||||
|
||||
// Path either is `examples` itself or sits under it: `examples`, `examples/`,
|
||||
// `examples/foo.ts`, `/abs/examples/`, etc. Rejects `someexamples`, `examples-x`.
|
||||
const EXAMPLES_PATH_PATTERN = /(?:^|\/)examples(?:$|\/)/;
|
||||
const TEMPLATES_PATH_PATTERN = /(?:^|\/)knowledge-base\/templates(?:$|\/)/;
|
||||
|
||||
type PendingTypedCall = { kind: 'read'; filename: string } | { kind: 'search'; query: string };
|
||||
|
||||
|
|
@ -271,7 +268,7 @@ function matchTypedTemplateCall(
|
|||
}
|
||||
if (toolName === TYPED_GREP_TOOL) {
|
||||
const path = typeof args.path === 'string' ? args.path : '';
|
||||
if (!EXAMPLES_PATH_PATTERN.test(path)) return undefined;
|
||||
if (!TEMPLATES_PATH_PATTERN.test(path)) return undefined;
|
||||
const pattern = typeof args.pattern === 'string' ? args.pattern : '';
|
||||
return { kind: 'search', query: pattern };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import type { z as zType } from 'zod';
|
||||
|
||||
// Manual mocks — must be declared before any imports that touch the mocked modules.
|
||||
jest.mock('@n8n/instance-ai', () => {
|
||||
const { z } = jest.requireActual<{ z: typeof zType }>('zod');
|
||||
const { z } = jest.requireActual('zod');
|
||||
const { getPromptWorkspaceRoot } = jest.requireActual('@n8n/instance-ai');
|
||||
return {
|
||||
McpClientManager: class {
|
||||
getRegularTools = jest.fn().mockResolvedValue({});
|
||||
|
|
@ -18,6 +17,9 @@ jest.mock('@n8n/instance-ai', () => {
|
|||
}),
|
||||
),
|
||||
createLazyWorkspaceRuntimeSkillSource: jest.fn(({ source }) => source),
|
||||
createScopedWorkspace: jest.fn((workspace: unknown) => workspace),
|
||||
getPromptWorkspaceRoot,
|
||||
getWorkspaceRoot: jest.fn(async () => '/home/daytona/workspace'),
|
||||
setupSandboxWorkspace: jest.fn(),
|
||||
loadInstanceAiRuntimeSkillSource: jest.fn(() => ({
|
||||
registry: {
|
||||
|
|
@ -137,8 +139,8 @@ jest.mock('@n8n/instance-ai', () => {
|
|||
};
|
||||
});
|
||||
|
||||
import type { User } from '@n8n/db';
|
||||
import type { InstanceAiAgentNode, InstanceAiEvent } from '@n8n/api-types';
|
||||
import type { User } from '@n8n/db';
|
||||
import {
|
||||
createAllTools,
|
||||
createLazyRuntimeWorkspace,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,10 @@ import {
|
|||
createWorkspace,
|
||||
createLazyRuntimeWorkspace,
|
||||
createLazyWorkspaceRuntimeSkillSource,
|
||||
createScopedWorkspace,
|
||||
setupSandboxWorkspace,
|
||||
getPromptWorkspaceRoot,
|
||||
getWorkspaceRoot,
|
||||
loadInstanceAiRuntimeSkillSource,
|
||||
createInstanceAiTraceContext,
|
||||
createInternalOperationTraceContext,
|
||||
|
|
@ -2996,35 +2999,53 @@ export class InstanceAiService {
|
|||
const baseRuntimeSkills = loadInstanceAiRuntimeSkillSource();
|
||||
let runtimeSkills = baseRuntimeSkills;
|
||||
let runtimeWorkspace: Workspace | undefined;
|
||||
let workspaceRoot: string | undefined;
|
||||
|
||||
if (adminSettings.sandboxEnabled) {
|
||||
let sandboxEntryPromise: Promise<RuntimeSandboxEntry | undefined> | undefined;
|
||||
const getSandboxEntry = async () => {
|
||||
sandboxEntryPromise ??= this.getOrCreateWorkspaceEntry(threadId, user, runId).catch(
|
||||
(error: unknown) => {
|
||||
sandboxEntryPromise = undefined;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
const sandboxConfig = await this.resolveSandboxConfig(user);
|
||||
|
||||
return await sandboxEntryPromise;
|
||||
};
|
||||
const getSetupSandboxEntry = async () => {
|
||||
return await this.getOrCreateWorkspace(threadId, user, context, runId);
|
||||
};
|
||||
if (sandboxConfig.enabled) {
|
||||
workspaceRoot = getPromptWorkspaceRoot(sandboxConfig.provider);
|
||||
|
||||
runtimeWorkspace = createLazyRuntimeWorkspace({
|
||||
ensureWorkspace: async () => (await getSetupSandboxEntry())?.workspace,
|
||||
});
|
||||
const runtimeSkillWorkspace = createLazyRuntimeWorkspace({
|
||||
id: 'instance-ai-runtime-skill-workspace',
|
||||
name: 'Instance AI runtime skill workspace',
|
||||
ensureWorkspace: async () => (await getSandboxEntry())?.workspace,
|
||||
});
|
||||
runtimeSkills = createLazyWorkspaceRuntimeSkillSource({
|
||||
source: baseRuntimeSkills,
|
||||
workspace: runtimeSkillWorkspace,
|
||||
logger: this.logger,
|
||||
});
|
||||
let sandboxEntryPromise: Promise<RuntimeSandboxEntry | undefined> | undefined;
|
||||
const getSandboxEntry = async () => {
|
||||
sandboxEntryPromise ??= this.getOrCreateWorkspaceEntry(threadId, user, runId).catch(
|
||||
(error: unknown) => {
|
||||
sandboxEntryPromise = undefined;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
|
||||
return await sandboxEntryPromise;
|
||||
};
|
||||
const getSetupSandboxEntry = async () => {
|
||||
return await this.getOrCreateWorkspace(threadId, user, context, runId);
|
||||
};
|
||||
|
||||
const scopeWorkspaceForAgent = async (
|
||||
workspace: Workspace | undefined,
|
||||
): Promise<Workspace | undefined> => {
|
||||
if (!workspace) return undefined;
|
||||
const root = await getWorkspaceRoot(workspace);
|
||||
return createScopedWorkspace(workspace, root);
|
||||
};
|
||||
|
||||
runtimeWorkspace = createLazyRuntimeWorkspace({
|
||||
ensureWorkspace: async () =>
|
||||
await scopeWorkspaceForAgent((await getSetupSandboxEntry())?.workspace),
|
||||
});
|
||||
const runtimeSkillWorkspace = createLazyRuntimeWorkspace({
|
||||
id: 'instance-ai-runtime-skill-workspace',
|
||||
name: 'Instance AI runtime skill workspace',
|
||||
ensureWorkspace: async () =>
|
||||
await scopeWorkspaceForAgent((await getSandboxEntry())?.workspace),
|
||||
});
|
||||
runtimeSkills = createLazyWorkspaceRuntimeSkillSource({
|
||||
source: baseRuntimeSkills,
|
||||
workspace: runtimeSkillWorkspace,
|
||||
logger: this.logger,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const orchestrationContext: OrchestrationContext = {
|
||||
|
|
@ -3106,6 +3127,7 @@ export class InstanceAiService {
|
|||
},
|
||||
workflowTaskService: workflowTasks,
|
||||
workspace: runtimeWorkspace,
|
||||
workspaceRoot,
|
||||
nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
|
||||
domainContext: context,
|
||||
tracingProxyConfig,
|
||||
|
|
|
|||
|
|
@ -5935,10 +5935,6 @@
|
|||
"instanceAi.tools.nodes.type-definition": "Loading node schema",
|
||||
"instanceAi.tools.nodes.suggested": "Getting suggested nodes",
|
||||
"instanceAi.tools.nodes.explore-resources": "Exploring node resources",
|
||||
"instanceAi.tools.templates": "Templates",
|
||||
"instanceAi.tools.templates.search-structures": "Searching template structures",
|
||||
"instanceAi.tools.templates.search-parameters": "Searching templates",
|
||||
"instanceAi.tools.templates.best-practices": "Loading best practices",
|
||||
"instanceAi.tools.task-control": "Task control",
|
||||
"instanceAi.tools.task-control.update-checklist": "Updating tasks",
|
||||
"instanceAi.tools.task-control.cancel-task": "Cancelling task",
|
||||
|
|
@ -5998,9 +5994,6 @@
|
|||
"instanceAi.tools.fetch-url": "Fetching page",
|
||||
"instanceAi.tools.build-workflow-with-agent": "Building workflow",
|
||||
"instanceAi.tools.build-workflow-with-agent.imperative": "edit workflow",
|
||||
"instanceAi.tools.get-best-practices": "Loading best practices",
|
||||
"instanceAi.tools.search-template-parameters": "Searching templates",
|
||||
"instanceAi.tools.search-template-structures": "Searching template structures",
|
||||
"instanceAi.tools.ask-user": "Asking user",
|
||||
"instanceAi.tools.cancel-background-task": "Cancelling task",
|
||||
"instanceAi.tools.correct-background-task": "Correcting task",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user