From 8659a73e31555450753fc8b5b47305cd30ec2000 Mon Sep 17 00:00:00 2001 From: oleg Date: Thu, 23 Oct 2025 09:59:53 +0200 Subject: [PATCH] feat(ai-builder): Properly separate system and user prompts in AI nodes (#21068) Signed-off-by: Oleg Ivaniv --- .../evaluators/agent-prompt.test.ts | 187 +++++++++++++++++- .../programmatic/evaluators/agent-prompt.ts | 40 +++- .../prompts/parameter-types/system-message.ts | 71 +++++++ .../src/chains/prompts/prompt-builder.ts | 38 +++- .../src/tools/prompts/main-agent.prompt.ts | 84 ++++++++ 5 files changed, 414 insertions(+), 6 deletions(-) create mode 100644 packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/parameter-types/system-message.ts diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts index c691f54bbf2..16ed4a84de4 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts @@ -55,6 +55,9 @@ describe('evaluateAgentPrompt', () => { position: [0, 0], parameters: { text: 'This is a static prompt without expressions', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, ], @@ -67,7 +70,7 @@ describe('evaluateAgentPrompt', () => { expect(result.violations[0]).toEqual({ type: 'minor', description: - 'Agent node "AI Agent" has no expression in its prompt field. This likely means it failed to use chatInput', + 'Agent node "AI Agent" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context', pointsDeducted: 15, }); }); @@ -83,6 +86,9 @@ describe('evaluateAgentPrompt', () => { position: [0, 0], parameters: { text: '=Process this request: {{ $json.chatInput }}', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, ], @@ -105,6 +111,9 @@ describe('evaluateAgentPrompt', () => { position: [0, 0], parameters: { text: '=Process: {{ $json.input }}', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, { @@ -115,6 +124,9 @@ describe('evaluateAgentPrompt', () => { position: [100, 0], parameters: { text: "=Process: {{$('Chat Trigger'.params.chatInput)}}", + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, { @@ -125,6 +137,9 @@ describe('evaluateAgentPrompt', () => { position: [200, 0], parameters: { text: '={{ $json.chatInput }}', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, ], @@ -171,6 +186,9 @@ describe('evaluateAgentPrompt', () => { parameters: { promptType: 'define', text: 'Static text without expressions', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, ], @@ -200,8 +218,9 @@ describe('evaluateAgentPrompt', () => { const result = evaluateAgentPrompt(workflow); - expect(result.violations).toHaveLength(1); - expect(result.violations[0].type).toBe('minor'); + // Should have violations for: no expression + no systemMessage + expect(result.violations.length).toBeGreaterThanOrEqual(1); + expect(result.violations.some((v) => v.type === 'minor')).toBe(true); }); it('should detect multiple agents with issues', () => { @@ -215,6 +234,9 @@ describe('evaluateAgentPrompt', () => { position: [0, 0], parameters: { text: 'No expression here', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, { @@ -225,6 +247,9 @@ describe('evaluateAgentPrompt', () => { position: [100, 0], parameters: { text: 'Also no expression', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, { @@ -235,6 +260,9 @@ describe('evaluateAgentPrompt', () => { position: [200, 0], parameters: { text: '=Has expression: {{ $json.input }}', + options: { + systemMessage: 'You are a helpful agent.', + }, }, }, ], @@ -247,4 +275,157 @@ describe('evaluateAgentPrompt', () => { expect(result.violations[0].description).toContain('Agent 1'); expect(result.violations[1].description).toContain('Agent 2'); }); + + describe('System Message Separation', () => { + it('should flag agent with no systemMessage as major violation', () => { + const workflow = mock({ + nodes: [ + { + id: '1', + name: 'Orchestrator Agent', + type: '@n8n/n8n-nodes-langchain.agent', + typeVersion: 3, + position: [0, 0], + parameters: { + promptType: 'define', + text: '=You are an orchestrator agent that coordinates specialized agents. Your task is to: 1) Call Research Agent 2) Call Fact-Check Agent. The research topic is: {{ $json.researchTopic }}', + }, + }, + ], + connections: {}, + }); + + const result = evaluateAgentPrompt(workflow); + + // Should have major violation for missing systemMessage + expect(result.violations.length).toBeGreaterThan(0); + const systemMessageViolation = result.violations.find((v) => + v.description.includes('no system message'), + ); + expect(systemMessageViolation).toBeDefined(); + expect(systemMessageViolation?.type).toBe('major'); + expect(systemMessageViolation?.pointsDeducted).toBe(25); + }); + + it('should not flag agent when it has proper systemMessage', () => { + const workflow = mock({ + nodes: [ + { + id: '1', + name: 'Agent', + type: '@n8n/n8n-nodes-langchain.agent', + typeVersion: 3, + position: [0, 0], + parameters: { + promptType: 'define', + text: '=You are an agent. Your task is to process: {{ $json.data }}', + options: { + systemMessage: 'You are a helpful agent.', + }, + }, + }, + ], + connections: {}, + }); + + const result = evaluateAgentPrompt(workflow); + + // Should not have any system message violations + const systemMessageViolations = result.violations.filter((v) => + v.description.includes('system message'), + ); + expect(systemMessageViolations).toHaveLength(0); + }); + + it('should pass for properly separated agent configuration', () => { + const workflow = mock({ + nodes: [ + { + id: '1', + name: 'Orchestrator Agent', + type: '@n8n/n8n-nodes-langchain.agent', + typeVersion: 3, + position: [0, 0], + parameters: { + promptType: 'define', + text: '=The research topic is: {{ $json.researchTopic }}', + options: { + systemMessage: + 'You are an orchestrator agent that coordinates specialized agents.\n\nYour task is to:\n1. Call the Research Agent Tool\n2. Call the Fact-Check Agent Tool\n3. Generate a report', + }, + }, + }, + ], + connections: {}, + }); + + const result = evaluateAgentPrompt(workflow); + + // Should have no violations + expect(result.violations).toHaveLength(0); + }); + + it('should handle agents with expressions in text and proper systemMessage', () => { + const testCases = [ + { text: "=You're analyzing: {{ $json.input }}" }, // Contains "you're" but has systemMessage + { text: '=Process this data: {{ $json.data }}' }, + { text: '=User question: {{ $json.chatInput }}' }, + { text: '=Analyze for topic: {{ $json.researchTopic }}' }, + ]; + + testCases.forEach(({ text }, index) => { + const workflow = mock({ + nodes: [ + { + id: `test-${index}`, + name: `Test Agent ${index}`, + type: '@n8n/n8n-nodes-langchain.agent', + typeVersion: 3, + position: [0, 0], + parameters: { + promptType: 'define', + text, + options: { + systemMessage: 'You are a helpful assistant.', + }, + }, + }, + ], + connections: {}, + }); + + const result = evaluateAgentPrompt(workflow); + // Should have no violations since systemMessage is present + expect(result.violations).toHaveLength(0); + }); + }); + + it('should flag major violation when agent has no systemMessage', () => { + const workflow = mock({ + nodes: [ + { + id: '1', + name: 'Agent', + type: '@n8n/n8n-nodes-langchain.agent', + typeVersion: 3, + position: [0, 0], + parameters: { + promptType: 'define', + text: '=Process: {{ $json.input }}', + }, + }, + ], + connections: {}, + }); + + const result = evaluateAgentPrompt(workflow); + + const noSystemMessageViolation = result.violations.find((v) => + v.description.includes('no system message'), + ); + expect(noSystemMessageViolation).toBeDefined(); + expect(noSystemMessageViolation?.type).toBe('major'); + expect(noSystemMessageViolation?.pointsDeducted).toBe(25); + }); + }); }); diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.ts index 422b3f30118..d75bbe9a8a9 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.ts @@ -1,3 +1,5 @@ +import type { INodeParameters } from 'n8n-workflow'; + import type { SimpleWorkflow } from '@/types'; import type { Violation } from '../../types/evaluation'; @@ -6,7 +8,23 @@ import { containsExpression } from '../../utils/expressions'; import { calcSingleEvaluatorScore } from '../../utils/score'; /** - * Evaluates Agent nodes to ensure their prompts contain expressions. + * Type guard to check if a value is a valid options object with systemMessage + */ +function isOptionsWithSystemMessage( + value: unknown, +): value is { systemMessage?: string } & INodeParameters { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + ('systemMessage' in value || Object.keys(value).length === 0) + ); +} + +/** + * Evaluates Agent nodes to ensure: + * 1. Their prompts contain expressions (for dynamic context) + * 2. They have a system message defined * Agent nodes without expressions in prompts (e.g., that failed to use chatInput * when there was a chat trigger) are most probably errors. */ @@ -25,17 +43,35 @@ export function evaluateAgentPrompt(workflow: SimpleWorkflow): SingleEvaluatorRe // Check the text parameter for expressions const textParam = node.parameters?.text; const promptType = node.parameters?.promptType; + const options = node.parameters?.options; + + // Use type guard to safely access systemMessage + let systemMessage: string | undefined; + if (isOptionsWithSystemMessage(options)) { + systemMessage = options.systemMessage; + } // Only check when promptType is 'define' or undefined (default) // 'auto' mode means it uses text from previous node if (promptType !== 'auto') { + // Check 1: Text parameter should contain expressions for dynamic context if (!textParam || !containsExpression(textParam)) { violations.push({ type: 'minor', - description: `Agent node "${node.name}" has no expression in its prompt field. This likely means it failed to use chatInput`, + description: `Agent node "${node.name}" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context`, pointsDeducted: 15, }); } + + // Check 2: Agent should have a system message + // If systemMessage is missing, it likely means all instructions are in the text field + if (!systemMessage || systemMessage.trim().length === 0) { + violations.push({ + type: 'major', + description: `Agent node "${node.name}" has no system message. System-level instructions (role, tasks, behavior) should be in the system message field, not the text field`, + pointsDeducted: 25, + }); + } } } } diff --git a/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/parameter-types/system-message.ts b/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/parameter-types/system-message.ts new file mode 100644 index 00000000000..55097f5873b --- /dev/null +++ b/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/parameter-types/system-message.ts @@ -0,0 +1,71 @@ +export const SYSTEM_MESSAGE_GUIDE = ` +## CRITICAL: System Message vs User Prompt Separation + +AI nodes (AI Agent, LLM Chain, Anthropic, OpenAI, etc.) have TWO distinct prompt fields that MUST be used correctly: + +### Field Separation +1. **System Message** - Model's persistent identity, role, and instructions +2. **User Message/Text** - Dynamic user context and data references per execution + +**Node-specific field names:** +- AI Agent: \`options.systemMessage\` and \`text\` +- LLM Chain: \`messages.messageValues[]\` with system role and \`text\` +- Anthropic: \`options.system\` and \`messages.values[]\` +- OpenAI: \`messages.values[]\` with role "system" vs "user" + +### System Message +Use for STATIC, ROLE-DEFINING content: +- Agent identity: "You are a [role] agent..." +- Task description: "Your task is to..." +- Step-by-step instructions: "1. Do X, 2. Do Y, 3. Do Z" +- Behavioral guidelines: "Always...", "Never..." +- Tool coordination: "Call the Research Tool to..., then call..." + +### Text Parameter +Use for DYNAMIC, EXECUTION-SPECIFIC content: +- User input: "={{ $json.chatInput }}" +- Workflow context: "=The research topic is: {{ $json.topic }}" +- Data from previous nodes: "=Process this data: {{ $json.data }}" +- Variable context: "=Analyze for user: {{ $json.userId }}" + +### Common Mistakes to Avoid + +❌ **WRONG - Everything in text field:** +{ + "text": "=You are an orchestrator agent that coordinates specialized agents. Your task is to: 1) Call Research Agent Tool, 2) Call Fact-Check Agent Tool, 3) Generate report. Research topic: {{ $json.researchTopic }}" +} + +✅ **CORRECT - Properly separated:** +{ + "text": "=The research topic is: {{ $json.researchTopic }}", + "options": { + "systemMessage": "You are an orchestrator agent that coordinates specialized agents.\\n\\nYour task is to:\\n1. Call the Research Agent Tool to gather information\\n2. Call the Fact-Check Agent Tool to verify findings\\n3. Generate a comprehensive report\\n\\nReturn ONLY the final report." + } +} + +### Update Pattern Examples + +Example 1 - AI Agent with orchestration: +Instructions: [ + "Set text to '=Research topic: {{ $json.researchTopic }}'", + "Set system message to 'You are an orchestrator coordinating research tasks.\\n\\nSteps:\\n1. Call Research Agent Tool\\n2. Call Fact-Check Agent Tool\\n3. Call Report Writer Agent Tool\\n4. Return final HTML report'" +] + +Example 2 - Chat-based AI node: +Instructions: [ + "Set text to '=User question: {{ $json.chatInput }}'", + "Set system message to 'You are a helpful customer support assistant. Answer questions clearly and professionally. Escalate to human if needed.'" +] + +Example 3 - Data processing with AI: +Instructions: [ + "Set text to '=Process this data: {{ $json.inputData }}'", + "Set system message to 'You are a data analysis assistant.\\n\\nTasks:\\n1. Validate data structure\\n2. Calculate statistics\\n3. Identify anomalies\\n4. Return JSON with findings'" +] + +### Why This Matters +- **Consistency**: System message stays the same across executions +- **Maintainability**: Easy to update AI behavior without touching data flow +- **Best Practice**: Follows standard AI prompt engineering patterns +- **Clarity**: Separates "what the AI model is" from "what data to process" +`; diff --git a/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/prompt-builder.ts b/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/prompt-builder.ts index 036c58f7517..665e13f1b41 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/prompt-builder.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/chains/prompts/prompt-builder.ts @@ -14,6 +14,7 @@ import { IF_NODE_GUIDE } from './node-types/if-node'; import { SET_NODE_GUIDE } from './node-types/set-node'; import { TOOL_NODES_GUIDE } from './node-types/tool-nodes'; import { RESOURCE_LOCATOR_GUIDE } from './parameter-types/resource-locator'; +import { SYSTEM_MESSAGE_GUIDE } from './parameter-types/system-message'; import { TEXT_FIELDS_GUIDE } from './parameter-types/text-fields'; import { DEFAULT_PROMPT_CONFIG, @@ -35,7 +36,9 @@ export class ParameterUpdatePromptBuilder { sections.push(EXPRESSION_RULES); // Add node-type specific guides - if (this.isSetNode(context.nodeType)) { + if (this.hasSystemMessageParameters(context.nodeDefinition)) { + sections.push(SYSTEM_MESSAGE_GUIDE); + } else if (this.isSetNode(context.nodeType)) { sections.push(SET_NODE_GUIDE); } else if (this.isIfNode(context.nodeType)) { sections.push(IF_NODE_GUIDE); @@ -78,6 +81,39 @@ export class ParameterUpdatePromptBuilder { return finalPrompt; } + /** + * Checks if node has system message parameters based on node definition + * This applies to nodes like AI Agent, LLM Chain, Anthropic, OpenAI, etc. + */ + private static hasSystemMessageParameters(nodeDefinition: INodeTypeDescription): boolean { + if (!nodeDefinition.properties) return false; + + // Check for common system message parameter patterns + const hasSystemMessageParam = nodeDefinition.properties.some((prop) => { + // Pattern 1 & 2: options.systemMessage (AI Agent) or options.system (Anthropic) + if (prop.name === 'options' && prop.type === 'collection') { + const collectionProp = prop; + if (Array.isArray(collectionProp.options)) { + return collectionProp.options.some( + (opt) => opt.name === 'systemMessage' || opt.name === 'system', + ); + } + } + + // Pattern 3: messages parameter with role support (OpenAI, LLM Chain) + if ( + prop.name === 'messages' && + (prop.type === 'fixedCollection' || prop.type === 'collection') + ) { + return true; // Messages typically support system role + } + + return false; + }); + + return hasSystemMessageParam; + } + /** * Checks if node is a Set node */ diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/prompts/main-agent.prompt.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/prompts/main-agent.prompt.ts index 69b57d85d36..f7948e6627c 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/prompts/main-agent.prompt.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/prompts/main-agent.prompt.ts @@ -231,6 +231,90 @@ Configure multiple nodes in parallel: - update_node_parameters({{ nodeId: "documentLoader1", instructions: ["Set dataType to 'binary' for processing PDF files", "Set loader to 'pdfLoader'", "Enable splitPages option"] }}) Why: Unconfigured nodes WILL fail at runtime + + +CRITICAL: For AI nodes (AI Agent, LLM Chain, Anthropic, OpenAI, etc.), you MUST separate system-level instructions from user context. + +**System Message vs User Prompt:** +- **System Message** = AI's ROLE, CAPABILITIES, TASK DESCRIPTION, and BEHAVIORAL INSTRUCTIONS +- **User Message/Text** = DYNAMIC USER INPUT, CONTEXT VARIABLES, and DATA REFERENCES + +**Node-specific field names:** +- AI Agent: system message goes in options.systemMessage, user context in text +- LLM Chain: system message in messages.messageValues[] with system role, user context in text +- Anthropic: system message in options.system, user context in messages.values[] +- OpenAI: system message in messages.values[] with role "system", user in messages.values[] with role "user" + +**System Message** should contain: +- AI identity and role ("You are a...") +- Task description ("Your task is to...") +- Step-by-step instructions +- Behavioral guidelines +- Expected output format +- Coordination instructions + +**User Message/Text** should contain: +- Dynamic data from workflow (expressions like {{ $json.field }}) +- User input references ({{ $json.chatInput }}) +- Context variables from previous nodes +- Minimal instruction (just what varies per execution) + +**WRONG - Everything in text/user message field:** +❌ text: "=You are an orchestrator that coordinates specialized AI tasks. Your task is to: 1) Call Research Tool 2) Call Fact-Check Tool 3) Return HTML. The research topic is: {{ $json.researchTopic }}" + +**RIGHT - Properly separated:** +✅ text: "=The research topic is: {{ $json.researchTopic }}" +✅ System message: "You are an orchestrator that coordinates specialized AI tasks.\n\nYour task is to:\n1. Call the Research Agent Tool to gather information\n2. Call the Fact-Check Agent Tool to verify findings\n3. Call the Report Writer Agent Tool to create a report\n4. Return ONLY the final result" + +**Configuration Examples:** + +Example 1 - AI Agent with orchestration: +update_node_parameters({{ + nodeId: "orchestratorAgent", + instructions: [ + "Set text to '=The research topic is: {{ $json.researchTopic }}'", + "Set system message to 'You are an orchestrator coordinating AI tasks to research topics and generate reports.\\n\\nYour task is to:\\n1. Call the Research Agent Tool to gather information\\n2. Call the Fact-Check Agent Tool to verify findings (require 2+ sources)\\n3. Call the Report Writer Agent Tool to create a report under 1,000 words\\n4. Call the HTML Editor Agent Tool to format as HTML\\n5. Return ONLY the final HTML content'" + ] +}}) + +Example 2 - AI Agent Tool (sub-agent): +update_node_parameters({{ + nodeId: "subAgentTool", + instructions: [ + "Set text to '=Process this input: {{ $fromAI(\\'input\\') }}'", + "Set system message to 'You are a specialized assistant. Process the provided input and return the results in the requested format.'" + ] +}}) + +CRITICAL: AI Agent Tools MUST have BOTH system message AND text field configured: +- System message: Define the tool's role and capabilities +- Text field: Pass the context/input using $fromAI() to receive parameters from the parent agent +- Never leave text field empty - the tool needs to know what to process + +Example 3 - Chat-based AI node: +update_node_parameters({{ + nodeId: "chatAssistant", + instructions: [ + "Set text to '=User question: {{ $json.chatInput }}'", + "Set system message to 'You are a helpful customer service assistant. Answer questions clearly and concisely. If you don\\'t know the answer, say so and offer to escalate to a human.'" + ] +}}) + +Example 4 - Data processing AI: +update_node_parameters({{ + nodeId: "analysisNode", + instructions: [ + "Set text to '=Analyze this data: {{ $json.data }}'", + "Set system message to 'You are a data analysis assistant. Examine the provided data and:\\n1. Identify key patterns and trends\\n2. Calculate relevant statistics\\n3. Highlight anomalies or outliers\\n4. Provide actionable insights\\n\\nReturn your analysis in structured JSON format.'" + ] +}}) + +**Why this matters:** +- Keeps AI behavior consistent (system message) while allowing dynamic context (user message) +- Makes workflows more maintainable and reusable +- Follows AI best practices for prompt engineering +- Prevents mixing static instructions with dynamic data +