diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts index f039e94861d..50e7b0f67b9 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/programmatic/evaluators/agent-prompt.test.ts @@ -69,6 +69,7 @@ describe('evaluateAgentPrompt', () => { expect(result.violations).toHaveLength(1); expect(result.violations[0]).toEqual({ type: 'major', + name: expect.any(String), description: 'Agent node "AI Agent" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context', pointsDeducted: 20, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts b/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts index 664c69b97f5..d1825ad75c0 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts @@ -1,11 +1,13 @@ import { ChatAnthropic } from '@langchain/anthropic'; +import { AIMessage, ToolMessage } from '@langchain/core/messages'; +import type { BaseMessage } from '@langchain/core/messages'; import { LangChainTracer } from '@langchain/core/tracers/tracer_langchain'; import { Logger } from '@n8n/backend-common'; import { Service } from '@n8n/di'; import { AiAssistantClient, AiAssistantSDK } from '@n8n_io/ai-assistant-sdk'; import assert from 'assert'; import { Client as TracingClient } from 'langsmith'; -import type { IUser, INodeTypeDescription } from 'n8n-workflow'; +import type { IUser, INodeTypeDescription, ITelemetryTrackProperties } from 'n8n-workflow'; import { LLMServiceError } from '@/errors'; import { anthropicClaudeSonnet45 } from '@/llm-config'; @@ -14,6 +16,8 @@ import { WorkflowBuilderAgent, type ChatPayload } from '@/workflow-builder-agent type OnCreditsUpdated = (userId: string, creditsQuota: number, creditsClaimed: number) => void; +type OnTelemetryEvent = (event: string, properties: ITelemetryTrackProperties) => void; + @Service() export class AiWorkflowBuilderService { private readonly parsedNodeTypes: INodeTypeDescription[]; @@ -23,8 +27,10 @@ export class AiWorkflowBuilderService { parsedNodeTypes: INodeTypeDescription[], private readonly client?: AiAssistantClient, private readonly logger?: Logger, + private readonly instanceId?: string, private readonly instanceUrl?: string, private readonly onCreditsUpdated?: OnCreditsUpdated, + private readonly onTelemetryEvent?: OnTelemetryEvent, ) { this.parsedNodeTypes = this.filterNodeTypes(parsedNodeTypes); this.sessionManager = new SessionManagerService(this.parsedNodeTypes, logger); @@ -44,6 +50,7 @@ export class AiWorkflowBuilderService { apiKey, headers: { ...authHeaders, + // eslint-disable-next-line @typescript-eslint/naming-convention 'anthropic-beta': 'prompt-caching-2024-07-31', }, }); @@ -54,6 +61,7 @@ export class AiWorkflowBuilderService { const authResponse = await this.client.getBuilderApiProxyToken(user); const authHeaders = { + // eslint-disable-next-line @typescript-eslint/naming-convention Authorization: `${authResponse.tokenType} ${authResponse.accessToken}`, }; @@ -63,6 +71,7 @@ export class AiWorkflowBuilderService { private async setupModels(user: IUser): Promise<{ anthropicClaude: ChatAnthropic; tracingClient?: TracingClient; + // eslint-disable-next-line @typescript-eslint/naming-convention authHeaders?: { Authorization: string }; }> { try { @@ -168,6 +177,7 @@ export class AiWorkflowBuilderService { private async onGenerationSuccess( user?: IUser, + // eslint-disable-next-line @typescript-eslint/naming-convention authHeaders?: { Authorization: string }, ): Promise { try { @@ -190,10 +200,66 @@ export class AiWorkflowBuilderService { async *chat(payload: ChatPayload, user: IUser, abortSignal?: AbortSignal) { const agent = await this.getAgent(user); + const userId = user?.id?.toString(); + const workflowId = payload.workflowContext?.currentWorkflow?.id; - for await (const output of agent.chat(payload, user?.id?.toString(), abortSignal)) { + for await (const output of agent.chat(payload, userId, abortSignal)) { yield output; } + + // After the stream completes, track telemetry + if (this.onTelemetryEvent && userId) { + try { + await this.trackBuilderReplyTelemetry(agent, workflowId, userId); + } catch (error) { + this.logger?.error('Failed to track builder reply telemetry', { error }); + } + } + } + + private async trackBuilderReplyTelemetry( + agent: WorkflowBuilderAgent, + workflowId: string | undefined, + userId: string, + ): Promise { + if (!this.onTelemetryEvent) return; + + const state = await agent.getState(workflowId, userId); + const threadId = SessionManagerService.generateThreadId(workflowId, userId); + + // extract the last message that was sent to the user for telemetry + const lastAiMessage = state.values.messages.findLast( + (m: BaseMessage): m is AIMessage => m instanceof AIMessage, + ); + const messageAi = + typeof lastAiMessage?.content === 'string' + ? lastAiMessage.content + : JSON.stringify(lastAiMessage?.content ?? ''); + + const toolMessages = state.values.messages.filter( + (m: BaseMessage): m is ToolMessage => m instanceof ToolMessage, + ); + const toolsCalled = [ + ...new Set( + toolMessages + .map((m: ToolMessage) => m.name) + .filter((name: string | undefined): name is string => name !== undefined), + ), + ]; + + // Build telemetry properties + const properties: ITelemetryTrackProperties = { + user_id: userId, + instance_id: this.instanceId, + workflow_id: workflowId, + sequence_id: threadId, + message_ai: messageAi, + tools_called: toolsCalled, + techniques_categories: state.values.techniqueCategories, + validations: state.values.validationHistory, + }; + + this.onTelemetryEvent('Builder replied to user message', properties); } async getSessions(workflowId: string | undefined, user?: IUser) { diff --git a/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts index 533168ff660..f8d00ac6952 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts @@ -188,6 +188,7 @@ describe('AiWorkflowBuilderService', () => { mockNodeTypeDescriptions, mockClient, mockLogger, + 'test-instance-id', 'https://n8n.example.com', mockOnCreditsUpdated, ); @@ -199,6 +200,7 @@ describe('AiWorkflowBuilderService', () => { mockNodeTypeDescriptions, mockClient, mockLogger, + 'test-instance-id', 'https://test.com', mockOnCreditsUpdated, ); @@ -224,6 +226,7 @@ describe('AiWorkflowBuilderService', () => { mockNodeTypeDescriptions, mockClient, mockLogger, + 'test-instance-id', 'https://test.com', mockOnCreditsUpdated, ); @@ -247,6 +250,7 @@ describe('AiWorkflowBuilderService', () => { mockNodeTypeDescriptions, mockClient, mockLogger, + 'test-instance-id', 'https://test.com', mockOnCreditsUpdated, ); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts index de94498f27f..d32c012427f 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts @@ -286,6 +286,8 @@ describe('WorkflowBuilderAgent', () => { }, }, workflowValidation: null, + validationHistory: [], + techniqueCategories: [], previousSummary: 'EMPTY', }; }; diff --git a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-state.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-state.test.ts index bed2841803f..e10c532926d 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-state.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-state.test.ts @@ -1,7 +1,8 @@ import { HumanMessage, AIMessage as AssistantMessage, ToolMessage } from '@langchain/core/messages'; import type { BaseMessage } from '@langchain/core/messages'; -import { createTrimMessagesReducer } from '../workflow-state'; +import type { TelemetryValidationStatus } from '../validation/types'; +import { createTrimMessagesReducer, WorkflowState } from '../workflow-state'; describe('createTrimMessagesReducer', () => { it('should return messages unchanged when human messages are within limit', () => { @@ -152,3 +153,139 @@ describe('createTrimMessagesReducer', () => { expect(result.length).toBe(2); }); }); + +describe('WorkflowState.validationHistory reducer', () => { + // Helper to create TelemetryValidationStatus avoiding ESLint naming-convention warnings + const createValidationStatus = ( + violations: Array<{ name: string; result: 'pass' | 'fail' }>, + ): TelemetryValidationStatus => { + const status: Record = {}; + for (const violation of violations) { + status[violation.name] = violation.result; + } + return status as TelemetryValidationStatus; + }; + + it('should append new validation history to existing history', () => { + const reducer = WorkflowState.spec.validationHistory.operator; + + const existingHistory: TelemetryValidationStatus[] = [ + createValidationStatus([ + { name: 'tool-node-has-no-parameters', result: 'pass' }, + { name: 'agent-static-prompt', result: 'fail' }, + { name: 'workflow-has-no-nodes', result: 'pass' }, + ]), + createValidationStatus([ + { name: 'tool-node-has-no-parameters', result: 'fail' }, + { name: 'agent-static-prompt', result: 'pass' }, + { name: 'workflow-has-no-nodes', result: 'pass' }, + ]), + ]; + const newHistory: TelemetryValidationStatus[] = [ + createValidationStatus([ + { name: 'tool-node-has-no-parameters', result: 'pass' }, + { name: 'agent-static-prompt', result: 'pass' }, + { name: 'workflow-has-no-nodes', result: 'pass' }, + ]), + ]; + + const result = reducer(existingHistory, newHistory); + + expect(result).toHaveLength(3); + expect(result[0]).toBe(existingHistory[0]); + expect(result[1]).toBe(existingHistory[1]); + expect(result[2]).toBe(newHistory[0]); + }); + + it('should handle empty existing history with new updates', () => { + const reducer = WorkflowState.spec.validationHistory.operator; + + const newHistory: TelemetryValidationStatus[] = [ + createValidationStatus([ + { name: 'node-missing-required-input', result: 'pass' }, + { name: 'node-merge-single-input', result: 'fail' }, + ]), + ]; + + const result = reducer([], newHistory); + + expect(result).toEqual(newHistory); + expect(result[0]).toBe(newHistory[0]); + }); + + it('should handle multiple updates sequentially', () => { + type ReducerFn = ( + x: TelemetryValidationStatus[], + y: TelemetryValidationStatus[] | undefined | null, + ) => TelemetryValidationStatus[]; + const reducer = WorkflowState.spec.validationHistory.operator as ReducerFn; + + let history: TelemetryValidationStatus[] = []; + + // First update + const update1: TelemetryValidationStatus[] = [ + createValidationStatus([{ name: 'tool-node-has-no-parameters', result: 'pass' }]), + ]; + history = reducer(history, update1); + expect(history).toHaveLength(1); + expect(history[0]).toBe(update1[0]); + + // Second update (undefined - should not change) + const prevHistory = history; + history = reducer(history, undefined); + expect(history).toBe(prevHistory); + expect(history).toHaveLength(1); + + // Third update + const update2: TelemetryValidationStatus[] = [ + createValidationStatus([{ name: 'agent-static-prompt', result: 'fail' }]), + ]; + history = reducer(history, update2); + expect(history).toHaveLength(2); + expect(history[0]).toBe(update1[0]); + expect(history[1]).toBe(update2[0]); + + // Fourth update (empty array - should not change) + const prevHistory2 = history; + history = reducer(history, []); + expect(history).toBe(prevHistory2); + expect(history).toHaveLength(2); + }); +}); + +describe('WorkflowState.techniqueCategories reducer', () => { + it('should append new technique categories to existing categories', () => { + const reducer = WorkflowState.spec.techniqueCategories.operator; + + const existingCategories = ['scraping', 'data-transformation']; + const newCategories = ['notifications', 'scheduling']; + + const result = reducer(existingCategories, newCategories); + + expect(result).toHaveLength(4); + expect(result).toEqual(['scraping', 'data-transformation', 'notifications', 'scheduling']); + }); + + it('should return existing categories when update is undefined', () => { + type ReducerFn = (x: string[], y: string[] | undefined | null) => string[]; + const reducer = WorkflowState.spec.techniqueCategories.operator as ReducerFn; + + const existingCategories = ['api-integration', 'webhook']; + + const result = reducer(existingCategories, undefined); + + expect(result).toEqual(existingCategories); + expect(result).toBe(existingCategories); + }); + + it('should handle empty existing categories with new updates', () => { + const reducer = WorkflowState.spec.techniqueCategories.operator; + + const newCategories = ['email-automation', 'file-processing']; + + const result = reducer([], newCategories); + + expect(result).toEqual(newCategories); + expect(result[0]).toBe(newCategories[0]); + }); +}); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/categorize-prompt.tool.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/categorize-prompt.tool.ts index 4eeaef7e84b..0c8db89e756 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/categorize-prompt.tool.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/categorize-prompt.tool.ts @@ -68,6 +68,7 @@ export function createCategorizePromptTool(llm: BaseChatModel, logger?: Logger): return createSuccessResponse(config, buildCategorizationMessage(categorization), { categorization, + techniqueCategories: categorization.techniques, }); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/helpers/response.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/helpers/response.ts index 0cd48fde7d6..6852f7c7da7 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/helpers/response.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/helpers/response.ts @@ -20,6 +20,7 @@ export function createSuccessResponse( new ToolMessage({ content: message, tool_call_id: toolCallId, + name: config.toolCall?.name, }), ]; @@ -42,6 +43,7 @@ export function createErrorResponse(config: ToolRunnableConfig, error: ToolError new ToolMessage({ content: `Error: ${error.message}`, tool_call_id: toolCallId, + name: config.toolCall?.name, }), ]; diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/test/validate-workflow.tool.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/test/validate-workflow.tool.test.ts index af137c23a00..1bbdc1e7644 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/test/validate-workflow.tool.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/test/validate-workflow.tool.test.ts @@ -49,6 +49,7 @@ describe('validateWorkflow tool', () => { trigger: [], agentPrompt: [ { + name: 'agent-static-prompt', type: 'minor', description: 'Agent prompt is missing required expression.', pointsDeducted: 5, @@ -118,6 +119,7 @@ describe('validateWorkflow tool', () => { ...sampleValidationResult, connections: [ { + name: 'node-missing-required-input', type: 'critical', description: 'Node HTTP Request is missing required main input.', pointsDeducted: 50, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/tools/validate-workflow.tool.ts b/packages/@n8n/ai-workflow-builder.ee/src/tools/validate-workflow.tool.ts index bba91c79b41..c646d07cc17 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/tools/validate-workflow.tool.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/tools/validate-workflow.tool.ts @@ -5,6 +5,12 @@ import { z } from 'zod'; import type { BuilderTool, BuilderToolBase } from '@/utils/stream-processor'; import { programmaticValidation } from '@/validation/programmatic'; +import type { + ProgrammaticViolation, + ProgrammaticChecksResult, + TelemetryValidationStatus, +} from '@/validation/types'; +import { PROGRAMMATIC_VIOLATION_NAMES } from '@/validation/types'; import { ToolExecutionError, ValidationError } from '../errors'; import { formatWorkflowValidation } from '../utils/workflow-validation'; @@ -19,6 +25,26 @@ export const VALIDATE_WORKFLOW_TOOL: BuilderToolBase = { displayTitle: 'Validating workflow', }; +/** + * Creates a compacted validation result for use in telemetry + * @returns `{ X: 'pass' | 'fail', Y: 'pass' | 'fail', ... }` + */ +function collectValidationResultForTelemetry( + results: ProgrammaticChecksResult, +): TelemetryValidationStatus { + const status = Object.fromEntries( + PROGRAMMATIC_VIOLATION_NAMES.map((name) => [name, 'pass' as const]), + ) as TelemetryValidationStatus; + + Object.values(results).forEach((violations: ProgrammaticViolation[]) => { + violations?.forEach((violation) => { + status[violation.name] = 'fail'; + }); + }); + + return status; +} + export function createValidateWorkflowTool( parsedNodeTypes: INodeTypeDescription[], logger?: Logger, @@ -45,12 +71,15 @@ export function createValidateWorkflowTool( parsedNodeTypes, ); + const validationResultForTelemetry = collectValidationResultForTelemetry(violations); + const message = formatWorkflowValidation(violations); reporter.complete({ message }); return createSuccessResponse(config, message, { workflowValidation: violations, + validationHistory: [validationResultForTelemetry], }); } catch (error) { if (error instanceof z.ZodError) { diff --git a/packages/@n8n/ai-workflow-builder.ee/src/utils/test/operations-processor.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/utils/test/operations-processor.test.ts index 25dc3184809..04cf8975329 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/utils/test/operations-processor.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/utils/test/operations-processor.test.ts @@ -593,6 +593,8 @@ describe('operations-processor', () => { messages: [], workflowContext: {}, workflowValidation: null, + validationHistory: [], + techniqueCategories: [], previousSummary: 'EMPTY', }); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/utils/test/tool-executor.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/utils/test/tool-executor.test.ts index fed69471b2f..6369094aad6 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/utils/test/tool-executor.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/utils/test/tool-executor.test.ts @@ -49,6 +49,8 @@ describe('tool-executor', () => { messages, workflowContext: {}, workflowValidation: null, + validationHistory: [], + techniqueCategories: [], previousSummary: 'EMPTY', }); @@ -705,5 +707,114 @@ describe('tool-executor', () => { expect(result.messages).toContain(toolResultMessage); expect(result.workflowOperations).toHaveLength(1); }); + + it('should collect validationHistory from tool state updates', async () => { + const validation1 = { + result: 'success', + checks: { total: 5, passed: 5, failed: 0 }, + }; + const validation2 = { + result: 'warning', + checks: { total: 3, passed: 2, failed: 1 }, + }; + + const command1 = new MockCommand({ + update: { + messages: [new ToolMessage({ content: 'Validation 1', tool_call_id: 'call-1' })], + validationHistory: [validation1], + }, + }); + + const command2 = new MockCommand({ + update: { + messages: [new ToolMessage({ content: 'Validation 2', tool_call_id: 'call-2' })], + validationHistory: [validation2], + }, + }); + + const mockTool1 = createMockTool(command1); + const mockTool2 = createMockTool(command2); + + const aiMessage = new AIMessage(''); + aiMessage.tool_calls = [ + { + id: 'call-1', + name: 'validate_tool_1', + args: {}, + type: 'tool_call', + }, + { + id: 'call-2', + name: 'validate_tool_2', + args: {}, + type: 'tool_call', + }, + ]; + + const state = createState([aiMessage]); + const toolMap = new Map([ + ['validate_tool_1', mockTool1], + ['validate_tool_2', mockTool2], + ]); + + const options: ToolExecutorOptions = { state, toolMap }; + const result = await executeToolsInParallel(options); + + expect(result.validationHistory).toBeDefined(); + expect(result.validationHistory).toHaveLength(2); + expect(result.validationHistory).toContain(validation1); + expect(result.validationHistory).toContain(validation2); + }); + + it('should collect techniqueCategories from tool state updates', async () => { + const categories1 = ['scraping', 'data-transformation']; + const categories2 = ['notifications', 'scheduling']; + + const command1 = new MockCommand({ + update: { + messages: [new ToolMessage({ content: 'Categorized', tool_call_id: 'call-1' })], + techniqueCategories: categories1, + }, + }); + + const command2 = new MockCommand({ + update: { + messages: [new ToolMessage({ content: 'Categorized', tool_call_id: 'call-2' })], + techniqueCategories: categories2, + }, + }); + + const mockTool1 = createMockTool(command1); + const mockTool2 = createMockTool(command2); + + const aiMessage = new AIMessage(''); + aiMessage.tool_calls = [ + { + id: 'call-1', + name: 'categorize_tool_1', + args: {}, + type: 'tool_call', + }, + { + id: 'call-2', + name: 'categorize_tool_2', + args: {}, + type: 'tool_call', + }, + ]; + + const state = createState([aiMessage]); + const toolMap = new Map([ + ['categorize_tool_1', mockTool1], + ['categorize_tool_2', mockTool2], + ]); + + const options: ToolExecutorOptions = { state, toolMap }; + const result = await executeToolsInParallel(options); + + expect(result.techniqueCategories).toBeDefined(); + expect(result.techniqueCategories).toHaveLength(4); + expect(result.techniqueCategories).toEqual([...categories1, ...categories2]); + }); }); }); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/utils/tool-executor.ts b/packages/@n8n/ai-workflow-builder.ee/src/utils/tool-executor.ts index c7c6edba52b..f1ae43283f6 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/utils/tool-executor.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/utils/tool-executor.ts @@ -88,6 +88,7 @@ export async function executeToolsInParallel( return new ToolMessage({ content: errorContent, tool_call_id: toolCall.id ?? '', + name: toolCall.name, // Include error flag so tools can handle errors appropriately additional_kwargs: { error: true }, }); @@ -128,6 +129,24 @@ export async function executeToolsInParallel( } } + // Collect all technique categories + const allTechniqueCategories: string[] = []; + + for (const update of stateUpdates) { + if (update.techniqueCategories && Array.isArray(update.techniqueCategories)) { + allTechniqueCategories.push(...update.techniqueCategories); + } + } + + // Collect all validation history + const allValidationHistory: Array<(typeof WorkflowState.State.validationHistory)[number]> = []; + + for (const update of stateUpdates) { + if (update.validationHistory && Array.isArray(update.validationHistory)) { + allValidationHistory.push(...update.validationHistory); + } + } + // Return the combined update const finalUpdate: Partial = { messages: allMessages, @@ -137,5 +156,13 @@ export async function executeToolsInParallel( finalUpdate.workflowOperations = allOperations; } + if (allTechniqueCategories.length > 0) { + finalUpdate.techniqueCategories = allTechniqueCategories; + } + + if (allValidationHistory.length > 0) { + finalUpdate.validationHistory = allValidationHistory; + } + return finalUpdate; } diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/agent-prompt.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/agent-prompt.ts index f5c88b0f5d4..416164eb131 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/agent-prompt.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/agent-prompt.ts @@ -55,6 +55,7 @@ export function validateAgentPrompt(workflow: SimpleWorkflow): ProgrammaticViola // Check 1: Text parameter should contain expressions for dynamic context if (!textParam || !containsExpression(textParam)) { violations.push({ + name: 'agent-static-prompt', type: 'major', description: `Agent node "${node.name}" has no expression in its prompt field. This likely means it failed to use chatInput or dynamic context`, pointsDeducted: 20, @@ -65,6 +66,7 @@ export function validateAgentPrompt(workflow: SimpleWorkflow): ProgrammaticViola // If systemMessage is missing, it likely means all instructions are in the text field if (!systemMessage || systemMessage.trim().length === 0) { violations.push({ + name: 'agent-no-system-prompt', type: 'major', description: `Agent node "${node.name}" has no system message. System-level instructions (role, tasks, behavior) should be in the system message field, not the text field`, pointsDeducted: 25, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/connections.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/connections.ts index 28494bf8737..6d901b2c0db 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/connections.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/connections.ts @@ -45,6 +45,7 @@ function checkMissingRequiredInputs( if (input.required && providedCount === 0) { issues.push({ + name: 'node-missing-required-input', type: 'critical', description: `Node ${nodeInfo.node.name} (${nodeInfo.node.type}) is missing required input of type ${input.type}`, pointsDeducted: 50, @@ -67,6 +68,7 @@ function checkUnsupportedConnections( for (const [type] of providedInputTypes) { if (!supportedTypes.has(type)) { issues.push({ + name: 'node-unsupported-connection-input', type: 'critical', description: `Node ${nodeInfo.node.name} (${nodeInfo.node.type}) received unsupported connection type ${type}`, pointsDeducted: 50, @@ -90,6 +92,7 @@ function checkMergeNodeConnections( if (totalInputConnections < 2) { issues.push({ + name: 'node-merge-single-input', type: 'major', description: `Merge node ${nodeInfo.node.name} has only ${totalInputConnections} input connection(s). Merge nodes require at least 2 inputs to function properly.`, pointsDeducted: 20, @@ -101,6 +104,7 @@ function checkMergeNodeConnections( if (totalInputConnections !== expectedInputs) { issues.push({ + name: 'node-merge-incorrect-num-inputs', type: 'minor', description: `Merge node ${nodeInfo.node.name} has ${totalInputConnections} input connections but is configured to accept ${expectedInputs}.`, pointsDeducted: 10, @@ -121,6 +125,7 @@ function checkMergeNodeConnections( if (missingIndexes.length > 0) { issues.push({ + name: 'node-merge-missing-input', type: 'major', description: `Merge node ${nodeInfo.node.name} is missing connections for input(s) ${missingIndexes.join(', ')}.`, pointsDeducted: 20, @@ -165,6 +170,7 @@ function checkSubNodeRootConnections( if (!hasRootConnection) { issues.push({ + name: 'sub-node-not-connected', type: 'critical', description: `Sub-node ${node.name} (${node.type}) provides ${outputType} but is not connected to a root node.`, pointsDeducted: 50, @@ -193,6 +199,7 @@ export function validateConnections( const nodeType = nodeTypeMap.get(node.type); if (!nodeType) { violations.push({ + name: 'node-type-not-found', type: 'critical', description: `Node type ${node.type} not found for node ${node.name}`, pointsDeducted: 50, @@ -207,6 +214,7 @@ export function validateConnections( nodeInfo.resolvedOutputs = resolveNodeOutputs(nodeInfo); } catch (error) { violations.push({ + name: 'failed-to-resolve-connections', type: 'critical', description: `Failed to resolve connections for node ${node.name} (${node.type}): ${ error instanceof Error ? error.message : String(error) diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/from-ai.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/from-ai.ts index 5c9f30ae00d..dae1385f106 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/from-ai.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/from-ai.ts @@ -65,6 +65,7 @@ export function validateFromAi( if (node.parameters && parametersContainFromAi(node.parameters)) { violations.push({ + name: 'non-tool-node-uses-fromai', type: 'major', description: `Non-tool node "${node.name}" (${node.type}) uses $fromAI in its parameters. $fromAI is only for tool nodes connected to AI agents.`, pointsDeducted: 20, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/tools.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/tools.ts index 2ad4ef428ab..c76e7d3c36b 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/tools.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/tools.ts @@ -34,6 +34,7 @@ export function validateTools( if (isTool(nodeType) && !toolsWithoutParameters.includes(node.type)) { if (!node.parameters || Object.keys(node.parameters).length === 0) { violations.push({ + name: 'tool-node-has-no-parameters', type: 'major', description: `Tool node "${node.name}" has no parameters set.`, pointsDeducted: 20, @@ -43,6 +44,7 @@ export function validateTools( if (!nodeParametersContainExpression(node.parameters)) { violations.push({ + name: 'tool-node-static-parameters', type: 'major', description: `Tool node "${node.name}" has no expressions in its parameters. This likely means it is not using dynamic input.`, pointsDeducted: 20, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/trigger.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/trigger.ts index e3e59788ff3..260186d2ac4 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/trigger.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/checks/trigger.ts @@ -20,6 +20,7 @@ export function validateTrigger( if (!workflow.nodes || workflow.nodes.length === 0) { violations.push({ + name: 'workflow-has-no-nodes', type: 'critical', description: 'Workflow has no nodes', pointsDeducted: 50, @@ -44,6 +45,7 @@ export function validateTrigger( if (!hasTrigger) { violations.push({ + name: 'workflow-has-no-trigger', type: 'critical', description: 'Workflow must have at least one trigger node to start execution', pointsDeducted: 50, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/validation/types.ts b/packages/@n8n/ai-workflow-builder.ee/src/validation/types.ts index f80d255921e..9512d3175a6 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/validation/types.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/validation/types.ts @@ -4,7 +4,30 @@ import type { SimpleWorkflow } from '@/types'; export type ProgrammaticViolationType = 'critical' | 'major' | 'minor'; +export const PROGRAMMATIC_VIOLATION_NAMES = [ + 'tool-node-has-no-parameters', + 'tool-node-static-parameters', + 'agent-static-prompt', + 'agent-no-system-prompt', + 'non-tool-node-uses-fromai', + 'workflow-has-no-nodes', + 'workflow-has-no-trigger', + 'node-missing-required-input', + 'node-unsupported-connection-input', + 'node-merge-single-input', + 'node-merge-incorrect-num-inputs', + 'node-merge-missing-input', + 'sub-node-not-connected', + 'node-type-not-found', + 'failed-to-resolve-connections', +] as const; + +export type ProgrammaticViolationName = (typeof PROGRAMMATIC_VIOLATION_NAMES)[number]; + +export type TelemetryValidationStatus = Record; + export interface ProgrammaticViolation { + name: ProgrammaticViolationName; type: ProgrammaticViolationType; description: string; pointsDeducted: number; diff --git a/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts b/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts index 31fd4d71396..b6cec48e0f4 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts @@ -3,7 +3,7 @@ import { AIMessage, HumanMessage, RemoveMessage } from '@langchain/core/messages import type { ToolMessage } from '@langchain/core/messages'; import type { RunnableConfig } from '@langchain/core/runnables'; import type { LangChainTracer } from '@langchain/core/tracers/tracer_langchain'; -import type { MemorySaver } from '@langchain/langgraph'; +import type { MemorySaver, StateSnapshot } from '@langchain/langgraph'; import { StateGraph, END, GraphRecursionError } from '@langchain/langgraph'; import type { Logger } from '@n8n/backend-common'; import { @@ -40,6 +40,13 @@ import { estimateTokenCountFromMessages } from './utils/token-usage'; import { executeToolsInParallel } from './utils/tool-executor'; import { WorkflowState } from './workflow-state'; +/** + * Type for the state snapshot with properly typed values + */ +export type TypedStateSnapshot = Omit & { + values: typeof WorkflowState.State; +}; + /** * Determines which node to execute next based on the current state. * This function decides if the workflow should: @@ -385,12 +392,13 @@ export class WorkflowBuilderAgent { return workflow; } - async getState(workflowId: string, userId?: string) { + async getState(workflowId?: string, userId?: string): Promise { const workflow = this.createWorkflow(); const agent = workflow.compile({ checkpointer: this.checkpointer }); - return await agent.getState({ - configurable: { thread_id: `workflow-${workflowId}-user-${userId ?? new Date().getTime()}` }, - }); + const threadId = SessionManagerService.generateThreadId(workflowId, userId); + return (await agent.getState({ + configurable: { thread_id: threadId }, + })) as TypedStateSnapshot; } private getDefaultWorkflowJSON(payload: ChatPayload): SimpleWorkflow { diff --git a/packages/@n8n/ai-workflow-builder.ee/src/workflow-state.ts b/packages/@n8n/ai-workflow-builder.ee/src/workflow-state.ts index 16f1774cb1e..02f7a9981a3 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/workflow-state.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/workflow-state.ts @@ -3,7 +3,7 @@ import { HumanMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; import type { SimpleWorkflow, WorkflowOperation } from './types'; -import type { ProgrammaticEvaluationResult } from './validation/types'; +import type { ProgrammaticEvaluationResult, TelemetryValidationStatus } from './validation/types'; import type { ChatPayload } from './workflow-builder-agent'; /** @@ -80,10 +80,21 @@ export const WorkflowState = Annotation.Root({ workflowContext: Annotation({ reducer: (x, y) => y ?? x, }), + // Results of last workflow validation workflowValidation: Annotation({ reducer: (x, y) => (y === undefined ? x : y), default: () => null, }), + // Compacted programmatic validations history for telemetry + validationHistory: Annotation({ + reducer: (x, y) => (y && y.length > 0 ? [...x, ...y] : x), + default: () => [], + }), + // Technique categories identified from categorize_prompt tool for telemetry + techniqueCategories: Annotation({ + reducer: (x, y) => (y && y.length > 0 ? [...x, ...y] : x), + default: () => [], + }), // Previous conversation summary (used for compressing long conversations) previousSummary: Annotation({ diff --git a/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts b/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts index e0dcdf30724..bd91ef4264b 100644 --- a/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts +++ b/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts @@ -3,13 +3,15 @@ import type { Logger } from '@n8n/backend-common'; import type { GlobalConfig } from '@n8n/config'; import { AiAssistantClient } from '@n8n_io/ai-assistant-sdk'; import { mock } from 'jest-mock-extended'; -import type { IUser, INodeTypeDescription } from 'n8n-workflow'; +import type { InstanceSettings } from 'n8n-core'; +import type { IUser, INodeTypeDescription, ITelemetryTrackProperties } from 'n8n-workflow'; import type { License } from '@/license'; import type { Push } from '@/push'; import { WorkflowBuilderService } from '@/services/ai-workflow-builder.service'; import type { UrlService } from '@/services/url.service'; import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; +import type { Telemetry } from '@/telemetry'; jest.mock('@n8n/ai-workflow-builder'); jest.mock('@n8n_io/ai-assistant-sdk'); @@ -28,6 +30,8 @@ describe('WorkflowBuilderService', () => { let mockLogger: Logger; let mockUrlService: UrlService; let mockPush: Push; + let mockTelemetry: Telemetry; + let mockInstanceSettings: InstanceSettings; let mockUser: IUser; beforeEach(() => { @@ -59,6 +63,8 @@ describe('WorkflowBuilderService', () => { mockLogger = mock(); mockUrlService = mock(); mockPush = mock(); + mockTelemetry = mock(); + mockInstanceSettings = mock(); mockUser = mock(); mockUser.id = 'test-user-id'; @@ -66,6 +72,7 @@ describe('WorkflowBuilderService', () => { (mockUrlService.getInstanceBaseUrl as jest.Mock).mockReturnValue('https://instance.test.com'); (mockLicense.loadCertStr as jest.Mock).mockResolvedValue('test-cert'); (mockLicense.getConsumerId as jest.Mock).mockReturnValue('test-consumer-id'); + (mockInstanceSettings.instanceId as unknown) = 'test-instance-id'; mockConfig.aiAssistant = { baseUrl: '' }; // Reset the mocked AiWorkflowBuilderService @@ -79,6 +86,8 @@ describe('WorkflowBuilderService', () => { mockLogger, mockUrlService, mockPush, + mockTelemetry, + mockInstanceSettings, ); }); @@ -110,8 +119,10 @@ describe('WorkflowBuilderService', () => { mockNodeTypeDescriptions, undefined, // No client when baseUrl is not set mockLogger, - 'https://instance.test.com', + 'test-instance-id', // instanceId + 'https://instance.test.com', // instanceUrl expect.any(Function), // onCreditsUpdated callback + expect.any(Function), // onTelemetryEvent callback ); expect(result.value).toEqual({ messages: ['response'] }); @@ -147,8 +158,10 @@ describe('WorkflowBuilderService', () => { mockNodeTypeDescriptions, expect.any(AiAssistantClient), mockLogger, - 'https://instance.test.com', - expect.any(Function), + 'test-instance-id', // instanceId + 'https://instance.test.com', // instanceUrl + expect.any(Function), // onCreditsUpdated callback + expect.any(Function), // onTelemetryEvent callback ); }); @@ -264,12 +277,13 @@ describe('WorkflowBuilderService', () => { | ((userId: string, creditsQuota: number, creditsClaimed: number) => void) | undefined; - MockedAiWorkflowBuilderService.mockImplementation( - (_parsedNodeTypes, _client, _logger, _instanceUrl, callback) => { - capturedCallback = callback; - return mockAiService; - }, - ); + MockedAiWorkflowBuilderService.mockImplementation(((...args: any[]) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const callback = args[5]; // onCreditsUpdated is the 6th parameter + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + capturedCallback = callback; + return mockAiService; + }) as any); // Trigger service creation const generator = service.chat(mockPayload, mockUser); @@ -311,12 +325,13 @@ describe('WorkflowBuilderService', () => { | ((userId: string, creditsQuota: number, creditsClaimed: number) => void) | undefined; - MockedAiWorkflowBuilderService.mockImplementation( - (_parsedNodeTypes, _client, _logger, _instanceUrl, callback) => { - capturedCallback = callback; - return mockAiService; - }, - ); + MockedAiWorkflowBuilderService.mockImplementation(((...args: any[]) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const callback = args[5]; // onCreditsUpdated is the 6th parameter + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + capturedCallback = callback; + return mockAiService; + }) as any); const generator = service.chat(mockPayload, mockUser); await generator.next(); @@ -352,6 +367,136 @@ describe('WorkflowBuilderService', () => { }); }); + describe('onTelemetryEvent callback', () => { + it('should call telemetry.track when telemetry event is triggered', async () => { + const mockPayload = { + message: 'test message', + workflowContext: {}, + }; + + const mockChatGenerator = (async function* () { + yield { messages: ['response'] }; + })(); + + const mockAiService = mock(); + (mockAiService.chat as jest.Mock).mockReturnValue(mockChatGenerator); + + let capturedTelemetryCallback: + | ((event: string, properties: ITelemetryTrackProperties) => void) + | undefined; + + MockedAiWorkflowBuilderService.mockImplementation(((...args: any[]) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const telemetryCallback = args[6]; // onTelemetryEvent is the 7th parameter + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + capturedTelemetryCallback = telemetryCallback; + return mockAiService; + }) as any); + + // Trigger service creation + const generator = service.chat(mockPayload, mockUser); + await generator.next(); + + // Verify callback was provided + expect(capturedTelemetryCallback).toBeDefined(); + + // Simulate telemetry event + const testEvent = 'ai_builder_workflow_created'; + const testProperties = { + workflow_id: 'workflow-123', + node_count: 5, + user_id: 'user-123', + }; + + capturedTelemetryCallback!(testEvent, testProperties); + + // Verify telemetry.track was called + expect(mockTelemetry.track).toHaveBeenCalledWith(testEvent, testProperties); + }); + + it('should handle multiple telemetry events', async () => { + const mockPayload = { + message: 'test message', + workflowContext: {}, + }; + + const mockChatGenerator = (async function* () { + yield { messages: ['response'] }; + })(); + + const mockAiService = mock(); + (mockAiService.chat as jest.Mock).mockReturnValue(mockChatGenerator); + + let capturedTelemetryCallback: + | ((event: string, properties: ITelemetryTrackProperties) => void) + | undefined; + + MockedAiWorkflowBuilderService.mockImplementation(((...args: any[]) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const telemetryCallback = args[6]; // onTelemetryEvent is the 7th parameter + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + capturedTelemetryCallback = telemetryCallback; + return mockAiService; + }) as any); + + const generator = service.chat(mockPayload, mockUser); + await generator.next(); + + // Simulate multiple telemetry events + const event1 = 'ai_builder_chat_started'; + const properties1 = { session_id: 'session-1' }; + + const event2 = 'ai_builder_node_added'; + const properties2 = { node_type: 'http_request', workflow_id: 'workflow-123' }; + + capturedTelemetryCallback!(event1, properties1); + capturedTelemetryCallback!(event2, properties2); + + // Verify both telemetry events were tracked + expect(mockTelemetry.track).toHaveBeenCalledTimes(2); + expect(mockTelemetry.track).toHaveBeenNthCalledWith(1, event1, properties1); + expect(mockTelemetry.track).toHaveBeenNthCalledWith(2, event2, properties2); + }); + + it('should handle telemetry events with empty properties', async () => { + const mockPayload = { + message: 'test message', + workflowContext: {}, + }; + + const mockChatGenerator = (async function* () { + yield { messages: ['response'] }; + })(); + + const mockAiService = mock(); + (mockAiService.chat as jest.Mock).mockReturnValue(mockChatGenerator); + + let capturedTelemetryCallback: + | ((event: string, properties: ITelemetryTrackProperties) => void) + | undefined; + + MockedAiWorkflowBuilderService.mockImplementation(((...args: any[]) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const telemetryCallback = args[6]; // onTelemetryEvent is the 7th parameter + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + capturedTelemetryCallback = telemetryCallback; + return mockAiService; + }) as any); + + const generator = service.chat(mockPayload, mockUser); + await generator.next(); + + // Simulate telemetry event with empty properties + const testEvent = 'ai_builder_session_ended'; + const emptyProperties = {}; + + capturedTelemetryCallback!(testEvent, emptyProperties); + + // Verify telemetry.track was called with empty properties + expect(mockTelemetry.track).toHaveBeenCalledWith(testEvent, emptyProperties); + }); + }); + describe('getBuilderInstanceCredits', () => { it('should return builder instance credits', async () => { const expectedCredits = { diff --git a/packages/cli/src/services/ai-workflow-builder.service.ts b/packages/cli/src/services/ai-workflow-builder.service.ts index 1288949da85..41e7e808a0f 100644 --- a/packages/cli/src/services/ai-workflow-builder.service.ts +++ b/packages/cli/src/services/ai-workflow-builder.service.ts @@ -4,13 +4,16 @@ import { Logger } from '@n8n/backend-common'; import { GlobalConfig } from '@n8n/config'; import { Service } from '@n8n/di'; import { AiAssistantClient } from '@n8n_io/ai-assistant-sdk'; +import { InstanceSettings } from 'n8n-core'; import type { IUser } from 'n8n-workflow'; +import { ITelemetryTrackProperties } from 'n8n-workflow'; import { N8N_VERSION } from '@/constants'; import { License } from '@/license'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { Push } from '@/push'; import { UrlService } from '@/services/url.service'; +import { Telemetry } from '@/telemetry'; /** * This service wraps the actual AiWorkflowBuilderService to avoid circular dependencies. @@ -27,6 +30,8 @@ export class WorkflowBuilderService { private readonly logger: Logger, private readonly urlService: UrlService, private readonly push: Push, + private readonly telemetry: Telemetry, + private readonly instanceSettings: InstanceSettings, ) {} private async getService(): Promise { @@ -61,14 +66,21 @@ export class WorkflowBuilderService { ); }; + // Callback for AI Builder to send telemetry events + const onTelemetryEvent = (event: string, properties: ITelemetryTrackProperties) => { + this.telemetry.track(event, properties); + }; + const { nodes: nodeTypeDescriptions } = this.loadNodesAndCredentials.types; this.service = new AiWorkflowBuilderService( nodeTypeDescriptions, client, this.logger, + this.instanceSettings.instanceId, this.urlService.getInstanceBaseUrl(), onCreditsUpdated, + onTelemetryEvent, ); }