From 39b5976e48b209ec133fc609950301b72c216a41 Mon Sep 17 00:00:00 2001 From: Luka Zivkovic Date: Thu, 15 Oct 2026 00:54:38 +0200 Subject: [PATCH 1/5] fix(editor): Clarify agent check previews and progress --- .../frontend/@n8n/i18n/src/locales/en.json | 8 + .../__tests__/AgentEvalMockNotice.test.ts | 86 ++++++++++ .../components/AgentBuilderEditorColumn.vue | 1 + .../components/AgentEvalExamplesSlider.vue | 2 +- .../agents/components/AgentEvalMockNotice.vue | 61 +++++++ .../agents/components/AgentEvalsSection.vue | 5 +- .../InstanceAiTestAgentExamplesPanel.test.ts | 63 ++++--- .../InstanceAiTestAgentPreviewPanel.test.ts | 34 +++- .../InstanceAiTestAgentExamplesPanel.vue | 161 +++++++++++++++--- .../InstanceAiTestAgentPreviewPanel.vue | 17 ++ 10 files changed, 389 insertions(+), 49 deletions(-) create mode 100644 packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalMockNotice.test.ts create mode 100644 packages/frontend/editor-ui/src/features/agents/components/AgentEvalMockNotice.vue diff --git a/packages/frontend/@n8n/i18n/src/locales/en.json b/packages/frontend/@n8n/i18n/src/locales/en.json index 704b610a44a..409762262be 100644 --- a/packages/frontend/@n8n/i18n/src/locales/en.json +++ b/packages/frontend/@n8n/i18n/src/locales/en.json @@ -9076,6 +9076,14 @@ "agents.builder.agentEvals.run.finished": "{success} of {total} case passed | {success} of {total} cases passed", "agents.builder.agentEvals.run.startError": "Couldn't start the run", "agents.builder.agentEvals.run.starting": "Starting…", + "agents.builder.agentEvals.mockNotice.none": "Your agent won't really message anyone or change anything here.", + "agents.builder.agentEvals.mockNotice.one": "Your agent won't really use {first} here.", + "agents.builder.agentEvals.mockNotice.two": "Your agent won't really use {first} or {second} here.", + "agents.builder.agentEvals.mockNotice.more": "Your agent won't really use {first}, {second} and {count} more here.", + "instanceAi.testAgentPreview.runningChecks": "Running {count} checks, {done} done", + "instanceAi.testAgentPreview.viewFirstCheck": "View", + "instanceAi.testAgentPreview.hideFirstCheck": "Hide", + "agents.builder.agentEvals.run.progress": "Running · {done} of {total}", "agents.builder.agentEvals.run.lostTrack": "Lost track of this run. Reload the page to check its progress.", "agents.builder.agentEvals.run.cancel": "Stop run", diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalMockNotice.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalMockNotice.test.ts new file mode 100644 index 00000000000..bce28a6ad11 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalMockNotice.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, vi } from 'vitest'; +import { ref } from 'vue'; +import type { AgentJsonConfig, AgentJsonToolConfig } from '@n8n/api-types'; +import { createComponentRenderer } from '@/__tests__/render'; +import AgentEvalMockNotice from '../components/AgentEvalMockNotice.vue'; + +vi.mock('@/app/stores/nodeTypes.store', () => ({ + useNodeTypesStore: () => ({ + getNodeType: (name: string) => ({ + displayName: ( + { slack: 'Slack Tool', sheets: 'Google Sheets', gmail: 'Gmail' } as Record + )[name], + }), + }), +})); +vi.mock('../composables/useAgentIntegrationsCatalog', () => ({ + useAgentIntegrationsCatalog: () => ({ + catalog: ref([ + { type: 'slack', label: 'Slack' }, + { type: 'telegram', label: 'Telegram' }, + ]), + ensureLoaded: vi.fn().mockResolvedValue([]), + }), +})); +const node = (nodeType: string): AgentJsonToolConfig => ({ + type: 'node', + name: nodeType, + node: { nodeType, nodeTypeVersion: 1, parameters: {} }, +}); +const renderComponent = createComponentRenderer(AgentEvalMockNotice); +const config = (overrides: Partial = {}): AgentJsonConfig => ({ + name: 'Agent', + instructions: '', + ...overrides, +}); + +describe('AgentEvalMockNotice', () => { + it.each([ + [[], "Your agent won't really message anyone or change anything here."], + [[node('slack')], "Your agent won't really use Slack here."], + [[node('slack'), node('sheets')], "Your agent won't really use Slack or Google Sheets here."], + [ + [node('slack'), node('sheets'), node('gmail')], + "Your agent won't really use Slack, Google Sheets and 1 more here.", + ], + ] as const)('describes the configured services', (tools, expected) => { + const { getByText } = renderComponent({ + props: { config: config({ tools: [...tools] }), projectId: 'project-1' }, + }); + expect(getByText(expected)).toBeInTheDocument(); + }); + it('deduplicates channels and node tools and excludes disabled tools', () => { + const { getByText } = renderComponent({ + props: { + projectId: 'project-1', + config: config({ + tools: [node('slack'), { ...node('sheets'), enabled: false }], + integrations: [ + { type: 'slack', credentialId: 'slack-1' }, + { type: 'telegram', credentialId: 'telegram-1' }, + ], + }), + }, + }); + expect(getByText("Your agent won't really use Slack or Telegram here.")).toBeInTheDocument(); + }); + it('uses configured workflow and MCP names without showing credential ids', () => { + const { getByText } = renderComponent({ + props: { + projectId: 'project-1', + config: config({ + tools: [{ type: 'workflow', workflow: 'Order management', name: 'Create order' }], + mcpServers: [ + { + name: 'CRM', + url: 'https://example.com/mcp', + transport: 'streamableHttp', + authentication: 'none', + }, + ], + }), + }, + }); + expect(getByText("Your agent won't really use Create order or CRM here.")).toBeInTheDocument(); + }); +}); diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue index fc26891300e..a03c1b75c7b 100644 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue @@ -392,6 +392,7 @@ defineExpose({ onSetupTaskAction }); data-testid="agent-evals-tab-content" > +import { computed, watch } from 'vue'; +import type { AgentJsonConfig } from '@n8n/api-types'; +import { N8nText } from '@n8n/design-system'; +import { useI18n } from '@n8n/i18n'; +import { useNodeTypesStore } from '@/app/stores/nodeTypes.store'; +import { useAgentIntegrationsCatalog } from '../composables/useAgentIntegrationsCatalog'; +import { formatToolNameForDisplay } from '../utils/toolDisplayName'; + +const props = defineProps<{ config?: AgentJsonConfig | null; projectId: string }>(); +const i18n = useI18n(); +const nodeTypes = useNodeTypesStore(); +const { catalog, ensureLoaded } = useAgentIntegrationsCatalog(); +watch( + () => [props.projectId, props.config?.integrations] as const, + ([projectId, integrations]) => { + if (integrations?.length) void ensureLoaded(projectId).catch(() => {}); + }, + { immediate: true }, +); + +const services = computed(() => { + const names: string[] = []; + for (const tool of props.config?.tools ?? []) { + if (tool.enabled === false) continue; + if (tool.type === 'node') { + const nodeType = nodeTypes.getNodeType(tool.node.nodeType, tool.node.nodeTypeVersion); + names.push(nodeType?.displayName?.replace(/ Tool$/, '') ?? tool.name); + } else if (tool.type === 'workflow') names.push(tool.name || tool.workflow); + else names.push(formatToolNameForDisplay(tool.id)); + } + for (const server of props.config?.mcpServers ?? []) names.push(server.name); + for (const channel of props.config?.integrations ?? []) { + if (channel.type === 'n8n_chat' || !channel.credentialId.trim()) continue; + names.push( + catalog.value?.find((entry) => entry.type === channel.type)?.label ?? + formatToolNameForDisplay(channel.type), + ); + } + return [...new Set(names.map((name) => name.trim()).filter(Boolean))]; +}); +const message = computed(() => { + const [first, second] = services.value; + if (!first) return i18n.baseText('agents.builder.agentEvals.mockNotice.none'); + if (!second) + return i18n.baseText('agents.builder.agentEvals.mockNotice.one', { interpolate: { first } }); + if (services.value.length === 2) + return i18n.baseText('agents.builder.agentEvals.mockNotice.two', { + interpolate: { first, second }, + }); + return i18n.baseText('agents.builder.agentEvals.mockNotice.more', { + interpolate: { first, second, count: String(services.value.length - 2) }, + }); +}); + + + diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue index f444ff1f9d8..d7243775d77 100644 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue @@ -7,7 +7,7 @@ * run of the newest dataset itself rather than depending on a list view. */ import { computed, onMounted, ref, watch } from 'vue'; -import type { AgentEvalDraftCase } from '@n8n/api-types'; +import type { AgentEvalDraftCase, AgentJsonConfig } from '@n8n/api-types'; import { N8nButton, N8nCallout, N8nIcon, N8nLoading, N8nText } from '@n8n/design-system'; import { useToast } from '@n8n/composables/useToast'; import { useI18n } from '@n8n/i18n'; @@ -15,6 +15,7 @@ import { useI18n } from '@n8n/i18n'; import { useTestAgentPreviewFlag } from '@/features/agents/composables/useTestAgentPreviewFlag'; import { useAgentEvalsStore } from '../agentEvals.store'; import { isDataTableDataset, resolveCaseColumns } from '../utils/agentEvalCases.utils'; +import AgentEvalMockNotice from './AgentEvalMockNotice.vue'; import AgentEvalCasesCard from './AgentEvalCasesCard.vue'; import AgentEvalChecksPanel from './AgentEvalChecksPanel.vue'; import AgentEvalResultsPanel from './AgentEvalResultsPanel.vue'; @@ -22,6 +23,7 @@ import AgentEvalsEmptyStatePreview from './AgentEvalsEmptyStatePreview.vue'; const props = defineProps<{ projectId: string; + config?: AgentJsonConfig | null; agentId: string; /** No `agent:update` — cases render but nothing can be changed. */ disabled?: boolean; @@ -220,6 +222,7 @@ watch(() => props.agentId, load);