diff --git a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-model-recommendations.test.ts b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-model-recommendations.test.ts index aa33c34ecb1..a2235d19d65 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-model-recommendations.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-model-recommendations.test.ts @@ -1,9 +1,20 @@ import type { ProviderCatalog } from '@n8n/agents'; -import { buildModelRecommendationsSection } from '../agents-builder-model-recommendations'; -import { buildBuilderPrompt } from '../agents-builder-prompts'; +import { + buildModelRecommendationsSection, + getModelRecommendationsSection, + resetModelRecommendationsCacheForTest, +} from '../agents-builder-model-recommendations'; +import { buildBuilderPrompt, buildBuilderSessionContext } from '../agents-builder-prompts'; import { getBuilderRuntimeSkills } from '../skills'; +const { fetchProviderCatalog } = vi.hoisted(() => ({ fetchProviderCatalog: vi.fn() })); + +vi.mock('@n8n/agents/catalog', async (importOriginal) => ({ + ...(await importOriginal()), + fetchProviderCatalog, +})); + const catalog: ProviderCatalog = { anthropic: { id: 'anthropic', @@ -83,8 +94,8 @@ const catalog: ProviderCatalog = { }, }; -function buildPrompt(modelRecommendationsSection: string | null) { - return buildBuilderPrompt({ +function buildSessionContext(modelRecommendationsSection: string | null) { + return buildBuilderSessionContext({ agentPreviewPath: '/projects/project-1/agents/agent-1/preview', modelRecommendationsSection, }); @@ -107,7 +118,7 @@ describe('builder model recommendations', () => { }); it('routes distinct target-agent functions into autonomously managed skills', () => { - const prompt = buildPrompt(null); + const prompt = buildBuilderPrompt(); const skill = getBuilderRuntimeSkills().find((s) => s.id === 'agent-builder-target-skills'); expect(prompt).toContain( @@ -145,7 +156,7 @@ describe('builder model recommendations', () => { }); it('tells the builder to preserve fallback web search on model switches', () => { - const prompt = buildPrompt(null); + const prompt = buildBuilderPrompt(); expect(prompt).toContain( 'When changing models, preserve existing Brave or SearXNG\n `config.webSearch` unchanged', @@ -160,7 +171,7 @@ describe('builder model recommendations', () => { }); it('defers custom tool builder guidance to the agent-builder-custom-tools skill', () => { - const prompt = buildPrompt(null); + const prompt = buildBuilderPrompt(); const skill = getBuilderRuntimeSkills().find((s) => s.id === 'agent-builder-custom-tools'); expect(prompt).not.toContain("import { Tool } from '@n8n/agents';"); @@ -178,13 +189,14 @@ describe('builder model recommendations', () => { ); }); - it('injects the recommendation section only into the LLM selection prompt', () => { + it('injects the recommendation section into the session context, not the static prompt', () => { const section = buildModelRecommendationsSection(catalog); - expect(buildPrompt(section)).toContain('### Recommended LLM Models'); - expect(buildPrompt(section)).toContain('`openai/gpt-5` GPT-5'); - expect(buildPrompt(null)).not.toContain('### Recommended LLM Models'); - expect(buildPrompt(null)).toContain('do not recommend or name'); + expect(buildSessionContext(section)).toContain('### Recommended LLM Models'); + expect(buildSessionContext(section)).toContain('`openai/gpt-5` GPT-5'); + expect(buildBuilderPrompt()).not.toContain('### Recommended LLM Models'); + expect(buildSessionContext(null)).not.toContain('### Recommended LLM Models'); + expect(buildSessionContext(null)).toContain('do not recommend or name'); }); it('does not tell the builder to prefer Slack OAuth credentials for chat integrations', () => { @@ -196,3 +208,39 @@ describe('builder model recommendations', () => { expect(externalServicesSkill?.instructions).not.toContain('prefer the OAuth variant'); }); }); + +describe('getModelRecommendationsSection', () => { + beforeEach(() => { + resetModelRecommendationsCacheForTest(); + fetchProviderCatalog.mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('caches a successful catalog fetch', async () => { + fetchProviderCatalog.mockResolvedValue(catalog); + + const first = await getModelRecommendationsSection(); + const second = await getModelRecommendationsSection(); + + expect(first).toContain('### Recommended LLM Models'); + expect(second).toBe(first); + expect(fetchProviderCatalog).toHaveBeenCalledTimes(1); + }); + + it('waits before it retries a failed fetch, so turns do not each wait on the timeout', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + fetchProviderCatalog.mockRejectedValueOnce(new Error('offline')).mockResolvedValue(catalog); + + expect(await getModelRecommendationsSection()).toBeNull(); + expect(await getModelRecommendationsSection()).toBeNull(); + expect(fetchProviderCatalog).toHaveBeenCalledTimes(1); + + vi.setSystemTime(Date.now() + 10 * 60 * 1000 + 1); + + expect(await getModelRecommendationsSection()).toContain('### Recommended LLM Models'); + expect(fetchProviderCatalog).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-prompts.test.ts b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-prompts.test.ts index 389bac8ea56..68cd4d5348e 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/agents-builder-prompts.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/agents-builder-prompts.test.ts @@ -1,15 +1,12 @@ -import { AGENT_UI_LABELS_SECTION, buildBuilderPrompt } from '../agents-builder-prompts'; - -function buildPrompt() { - return buildBuilderPrompt({ - agentPreviewPath: '/projects/p1/agents/a1?openPreview=true', - modelRecommendationsSection: null, - }); -} +import { + AGENT_UI_LABELS_SECTION, + buildBuilderPrompt, + buildBuilderSessionContext, +} from '../agents-builder-prompts'; describe('buildBuilderPrompt', () => { it('teaches the builder the Sessions tab labels after conversation-mode guidance', () => { - const prompt = buildPrompt(); + const prompt = buildBuilderPrompt(); expect(prompt).toContain('## Agent UI labels'); expect(prompt).toContain('Sessions tab'); @@ -24,4 +21,41 @@ describe('buildBuilderPrompt', () => { expect(conversationModeIndex).toBeGreaterThan(-1); expect(uiLabelsIndex).toBeGreaterThan(conversationModeIndex); }); + + it('stays free of per-agent and per-process values so builds share one cached prompt', () => { + const prompt = buildBuilderPrompt(); + + expect(buildBuilderPrompt()).toBe(prompt); + expect(prompt).not.toContain('/projects/'); + expect(prompt).not.toContain('### Recommended LLM Models'); + expect(prompt).toContain('Preview link from the Session context'); + }); +}); + +describe('buildBuilderSessionContext', () => { + it('carries the agent Preview link', () => { + const context = buildBuilderSessionContext({ + agentPreviewPath: '/projects/p1/agents/a1?openPreview=true', + modelRecommendationsSection: null, + }); + + expect(context).toContain('## Session context'); + expect(context).toContain('[Preview](/projects/p1/agents/a1?openPreview=true)'); + }); + + it('includes the model recommendations, or the no-recommendations rule when absent', () => { + const withSection = buildBuilderSessionContext({ + agentPreviewPath: '/p', + modelRecommendationsSection: '### Recommended LLM Models\n\n- Anthropic: x', + }); + const withoutSection = buildBuilderSessionContext({ + agentPreviewPath: '/p', + modelRecommendationsSection: null, + }); + + expect(withSection).toContain('### Recommended LLM Models'); + expect(withSection).not.toContain('do not recommend or name'); + expect(withoutSection).not.toContain('### Recommended LLM Models'); + expect(withoutSection).toContain('do not recommend or name'); + }); }); diff --git a/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts b/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts index 3d49e2c6f8b..752fae4e1db 100644 --- a/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts +++ b/packages/cli/src/modules/agents/builder/__tests__/agents-builder.service.session.test.ts @@ -28,6 +28,7 @@ const agentsSdkMocks = vi.hoisted(() => { }> = []; const resumeCalls: Array<{ options: Record }> = []; const instructionsCalls: string[] = []; + const volatileInstructionsProviders: Array<() => Promise> = []; const registeredToolNames: string[] = []; const modelCalls: unknown[] = []; const configurationCalls: Array<{ maxIterations?: number }> = []; @@ -66,6 +67,10 @@ const agentsSdkMocks = vi.hoisted(() => { instructionsCalls.push(text); return this; } + volatileInstructionsProvider(provider: () => Promise) { + volatileInstructionsProviders.push(provider); + return this; + } skills(_skills: unknown) { return this; } @@ -129,6 +134,7 @@ const agentsSdkMocks = vi.hoisted(() => { streamCalls, resumeCalls, instructionsCalls, + volatileInstructionsProviders, registeredToolNames, modelCalls, configurationCalls, @@ -238,6 +244,7 @@ describe('AgentsBuilderService session isolation', () => { agentsSdkMocks.streamCalls.length = 0; agentsSdkMocks.resumeCalls.length = 0; agentsSdkMocks.instructionsCalls.length = 0; + agentsSdkMocks.volatileInstructionsProviders.length = 0; agentsSdkMocks.registeredToolNames.length = 0; agentsSdkMocks.modelCalls.length = 0; agentsSdkMocks.configurationCalls.length = 0; @@ -524,6 +531,29 @@ describe('AgentsBuilderService session isolation', () => { expect(agentsSdkMocks.configurationCalls).toEqual([{ maxIterations: 100 }]); }); + it('keeps the per-agent Preview path out of the cached instructions', async () => { + const { service, user, credentialProvider, credentialService } = setup(); + + await drain( + service.buildAgent( + 'agent-1', + 'project-1', + 'hi', + credentialProvider, + credentialService, + user, + baseSession, + ), + ); + + expect(agentsSdkMocks.instructionsCalls[0]).not.toContain('/projects/project-1/agents/agent-1'); + expect(agentsSdkMocks.volatileInstructionsProviders).toHaveLength(1); + const sessionContext = await agentsSdkMocks.volatileInstructionsProviders[0]?.(); + expect(sessionContext).toContain( + '[Preview](/projects/project-1/agents/agent-1?openPreview=true)', + ); + }); + it('enables prompt caching with a 5m Anthropic TTL for the builder agent', async () => { const { service, user, credentialProvider, credentialService } = setup(); diff --git a/packages/cli/src/modules/agents/builder/agents-builder-model-recommendations.ts b/packages/cli/src/modules/agents/builder/agents-builder-model-recommendations.ts index 05b0f7f01da..605a643a432 100644 --- a/packages/cli/src/modules/agents/builder/agents-builder-model-recommendations.ts +++ b/packages/cli/src/modules/agents/builder/agents-builder-model-recommendations.ts @@ -1,6 +1,8 @@ import { fetchProviderCatalog, type ModelInfo, type ProviderCatalog } from '@n8n/agents/catalog'; const MODEL_RECOMMENDATION_FETCH_TIMEOUT_MS = 5000; +/** After a failed fetch, skip retries for this long so each builder turn does not wait on the timeout. */ +const MODEL_RECOMMENDATION_RETRY_AFTER_MS = 10 * 60 * 1000; const MAX_RECOMMENDED_MODELS_PER_PROVIDER = 3; interface RecommendedProvider { @@ -18,6 +20,7 @@ const RECOMMENDED_PROVIDERS: RecommendedProvider[] = [ let modelRecommendationsSection: string | null | undefined; let modelRecommendationsPromise: Promise | undefined; +let modelRecommendationsRetryAt = 0; function getReleaseTime(model: ModelInfo): number | null { if (!model.releaseDate) return null; @@ -110,6 +113,7 @@ async function fetchProviderCatalogWithTimeout(): Promise { if (modelRecommendationsSection !== undefined) return modelRecommendationsSection; + if (!modelRecommendationsPromise && Date.now() < modelRecommendationsRetryAt) return null; modelRecommendationsPromise ??= fetchProviderCatalogWithTimeout() .then((catalog) => (catalog ? buildModelRecommendationsSection(catalog) : null)) @@ -119,6 +123,17 @@ export async function getModelRecommendationsSection(): Promise { }); const section = await modelRecommendationsPromise; - if (section) modelRecommendationsSection = section; + if (section) { + modelRecommendationsSection = section; + } else { + modelRecommendationsRetryAt = Date.now() + MODEL_RECOMMENDATION_RETRY_AFTER_MS; + } return section; } + +/** Test-only: clear the module-level cache. */ +export function resetModelRecommendationsCacheForTest(): void { + modelRecommendationsSection = undefined; + modelRecommendationsPromise = undefined; + modelRecommendationsRetryAt = 0; +} diff --git a/packages/cli/src/modules/agents/builder/agents-builder-prompts.ts b/packages/cli/src/modules/agents/builder/agents-builder-prompts.ts index c0b5b5d9f12..693cb753c04 100644 --- a/packages/cli/src/modules/agents/builder/agents-builder-prompts.ts +++ b/packages/cli/src/modules/agents/builder/agents-builder-prompts.ts @@ -1,6 +1,6 @@ import { getConfigMutationPrompt } from './prompts/config-mutation.prompt'; import { INITIAL_BUILD_SECTION } from './prompts/initial-build.prompt'; -import { getLlmSelectionPrompt } from './prompts/llm-selection.prompt'; +import { LLM_SELECTION_PROMPT } from './prompts/llm-selection.prompt'; import { MEMORY_PROMPT } from './prompts/memory.prompt'; import { TOOLS_PROMPT } from './prompts/tools.prompt'; @@ -48,8 +48,7 @@ When the user asks for a channel that is not supported (e.g. Microsoft Teams): When the user asks to change the target agent's channels, prefer a supported one from the list; never invent a type.`; -export function getConversationModeSection(agentPreviewPath: string): string { - return `\ +export const CONVERSATION_MODE_SECTION = `\ ## When To Build vs When To Converse Not every user message is a build request. Before changing config or creating @@ -74,17 +73,17 @@ message it from the connected platform to verify the channel. Standard tool approvals pause \`call_agent\` until the user approves or rejects them in this chat. If it returns \`approval_required\` for an unsupported interaction, explain that it cannot be -completed here and direct the user to [Preview](${agentPreviewPath}) to run it again. +completed here and direct the user to the Preview link from the Session context +section to run it again. After a successful build or config change that leaves the agent ready to try, -include the same [Preview](${agentPreviewPath}) markdown link in your wrap-up -(it can be part of a longer reply). Keep Preview links as relative app paths -and do not invent a different path. +include that same Preview markdown link in your wrap-up (it can be part of a +longer reply). Keep Preview links as relative app paths and do not invent a +different path. Never write empty or placeholder \`instructions\`. When the user gave a concrete goal, write real instructions from it and fill gaps with sensible assumptions stated in your summary. Only ask first when the overall goal itself is missing.`; -} export const AGENT_UI_LABELS_SECTION = `\ ## Agent UI labels @@ -375,23 +374,30 @@ follow-up for the credential. 4. After a successful publish, confirm the agent is live; do not send the user to the editor Publish button.`; -export interface BuilderPromptContext { +export interface BuilderSessionContext { agentPreviewPath: string; modelRecommendationsSection: string | null; } -export function buildBuilderPrompt(ctx: BuilderPromptContext): string { - const { agentPreviewPath, modelRecommendationsSection } = ctx; +const NO_MODEL_RECOMMENDATIONS = + 'No Recommended LLM models section is available; do not recommend or name current, best, latest, or fallback model IDs from memory. Ask via `ask_questions` when the user needs model guidance or choice.'; +/** + * The static builder system prompt. It must not contain any per-agent or + * per-process value: Anthropic caches the prompt by exact prefix, so a + * byte-identical prompt lets every build share one cached copy. Put dynamic + * values in `buildBuilderSessionContext` instead. + */ +export function buildBuilderPrompt(): string { const sections = [ 'You are an expert agent builder. You help users create and configure AI agents by writing raw JSON configuration and building custom tools.', TARGET_AGENT_SECTION, PREREQUISITES_SECTION, SUPPORTED_CHANNELS_SECTION, - getConversationModeSection(agentPreviewPath), + CONVERSATION_MODE_SECTION, AGENT_UI_LABELS_SECTION, getConfigMutationPrompt(), - getLlmSelectionPrompt(modelRecommendationsSection), + LLM_SELECTION_PROMPT, MEMORY_PROMPT, TOOLS_PROMPT, INTERACTIVE_TOOLS_SECTION, @@ -404,3 +410,19 @@ export function buildBuilderPrompt(ctx: BuilderPromptContext): string { return sections.join('\n\n'); } + +/** + * Per-session values for the builder. The runtime sends this as a separate + * system block after the cached static prompt, so a change here does not + * invalidate the cached prompt. + */ +export function buildBuilderSessionContext(ctx: BuilderSessionContext): string { + const { agentPreviewPath, modelRecommendationsSection } = ctx; + + return `\ +## Session context + +- Preview link for the target agent: [Preview](${agentPreviewPath}) + +${modelRecommendationsSection ?? NO_MODEL_RECOMMENDATIONS}`; +} diff --git a/packages/cli/src/modules/agents/builder/agents-builder.service.ts b/packages/cli/src/modules/agents/builder/agents-builder.service.ts index 178c186a6d6..ac8ba06cbea 100644 --- a/packages/cli/src/modules/agents/builder/agents-builder.service.ts +++ b/packages/cli/src/modules/agents/builder/agents-builder.service.ts @@ -35,7 +35,7 @@ import { AgentsService } from '../agents.service'; import { modelStreamStallOptions } from '../model-stream-stall-options'; import { buildAgentPreviewPath } from './agent-builder-preview-path'; import { getModelRecommendationsSection } from './agents-builder-model-recommendations'; -import { buildBuilderPrompt } from './agents-builder-prompts'; +import { buildBuilderPrompt, buildBuilderSessionContext } from './agents-builder-prompts'; import { AgentsBuilderToolsService, type BuilderTools } from './agents-builder-tools.service'; import { BuilderCheckpointUnavailableError } from './errors'; import { @@ -242,7 +242,8 @@ export class AgentsBuilderService { // always runs on it directly. const modelConfig = session.modelConfig; - const finalInstructions = await this.createBuilderInstructions(projectId, agentId, session); + const finalInstructions = this.createBuilderInstructions(session); + const sessionContext = await this.createBuilderSessionContext(projectId, agentId); const runtimeSkills = getBuilderRuntimeSkills(); const tools = this.agentsBuilderToolsService.getTools( @@ -264,6 +265,8 @@ export class AgentsBuilderService { const builder = new Agent('agent-builder') .model(modelConfig) .instructions(finalInstructions) + // Sent after the cached instructions so per-agent values do not break the cache. + .volatileInstructionsProvider(async () => sessionContext) .skills(runtimeSkills) .memory(builderMemory) .checkpoint(this.n8nCheckpointStorage.getStorage(agentId)) @@ -336,21 +339,21 @@ export class AgentsBuilderService { } } - private async createBuilderInstructions( - projectId: string, - agentId: string, - session: InstanceAiBuilderSessionOptions, - ): Promise { - const modelRecommendationsSection = await getModelRecommendationsSection(); - const instructions = buildBuilderPrompt({ - agentPreviewPath: buildAgentPreviewPath(projectId, agentId), - modelRecommendationsSection, - }); + /** Static per addendum, so every build with the same addendum shares one cached prompt. */ + private createBuilderInstructions(session: InstanceAiBuilderSessionOptions): string { + const instructions = buildBuilderPrompt(); return session.instructionsAddendum ? `${instructions}\n\n${session.instructionsAddendum}` : instructions; } + private async createBuilderSessionContext(projectId: string, agentId: string): Promise { + return buildBuilderSessionContext({ + agentPreviewPath: buildAgentPreviewPath(projectId, agentId), + modelRecommendationsSection: await getModelRecommendationsSection(), + }); + } + private async createBuilderMemory( agentId: string, user: User, diff --git a/packages/cli/src/modules/agents/builder/prompts/llm-selection.prompt.ts b/packages/cli/src/modules/agents/builder/prompts/llm-selection.prompt.ts index 69bc3990419..b77ee7fcd31 100644 --- a/packages/cli/src/modules/agents/builder/prompts/llm-selection.prompt.ts +++ b/packages/cli/src/modules/agents/builder/prompts/llm-selection.prompt.ts @@ -1,9 +1,8 @@ -export function getLlmSelectionPrompt(modelRecommendationsSection: string | null): string { - const recommendationGuidance = modelRecommendationsSection - ? `\n\n${modelRecommendationsSection}` - : '\n\nNo Recommended LLM models section is available; do not recommend or name current, best, latest, or fallback model IDs from memory. Ask via `ask_questions` when the user needs model guidance or choice.'; - - return `\ +/** + * Static: the live model recommendations are session context (see + * `buildBuilderSessionContext`), so this text stays byte-identical across builds. + */ +export const LLM_SELECTION_PROMPT = `\ ## LLM Selection Guidance ### Purpose @@ -43,6 +42,7 @@ Use this to resolve the target agent's main \`model\` and \`credential\`. - If the user explicitly asks for Brave or SearXNG, keep that provider even when the selected model also supports native search. - For "Anthropic via OpenRouter", pass \`provider: "openrouter"\`; if the user names a routed model, pass the routed id without adding another provider prefix. +- Model recommendations are in the "Recommended LLM Models" part of the Session context section. - Prefer a provider the user already has credentials for when choosing from recommendations. - Never copy main LLM credential IDs from \`list_credentials\`. @@ -59,5 +59,4 @@ Use this to resolve the target agent's main \`model\` and \`credential\`. - The persisted \`model\` is in \`provider/model\` form. - The persisted \`credential\` came from \`resolve_llm\`. -- Existing Brave or SearXNG \`config.webSearch\` is preserved on model changes unless the user explicitly requested a web-search method change.${recommendationGuidance}`; -} +- Existing Brave or SearXNG \`config.webSearch\` is preserved on model changes unless the user explicitly requested a web-search method change.`; diff --git a/packages/cli/src/modules/agents/instance-ai-builder-delegate.adapter.ts b/packages/cli/src/modules/agents/instance-ai-builder-delegate.adapter.ts index 911aaa1fa19..f51ec7c3526 100644 --- a/packages/cli/src/modules/agents/instance-ai-builder-delegate.adapter.ts +++ b/packages/cli/src/modules/agents/instance-ai-builder-delegate.adapter.ts @@ -31,7 +31,7 @@ export const INSTANCE_AI_BUILDER_ADDENDUM = `## Instance AI session rules You are running as a sub-agent inside n8n's instance AI chat; the user sees your questions as chat cards. -Preview links work in this chat. Include a markdown Preview link after a successful build and when \`call_agent\` reports an unsupported interaction as \`approval_required\`, using the exact relative path from "When To Build vs When To Converse" (form: \`[Preview]()\`). Do not invent absolute URLs. Do not omit the link and describe the path in plain text instead. +Preview links work in this chat. Include a markdown Preview link after a successful build and when \`call_agent\` reports an unsupported interaction as \`approval_required\`, using the exact relative path from the "Session context" section (form: \`[Preview]()\`). Do not invent absolute URLs. Do not omit the link and describe the path in plain text instead. When you mention the agent editor, say Sessions tab for history and Preview for live chat. Never say Runs, Executions, or Activity History for agents.