diff --git a/packages/frontend/@n8n/i18n/src/locales/en.json b/packages/frontend/@n8n/i18n/src/locales/en.json index 528e8d64b78..15d859783c3 100644 --- a/packages/frontend/@n8n/i18n/src/locales/en.json +++ b/packages/frontend/@n8n/i18n/src/locales/en.json @@ -7733,7 +7733,7 @@ "agents.builder.header.tab.executions": "Sessions", "agents.builder.header.tab.settings": "Settings", "agents.builder.header.tab.memory": "Memory", - "agents.builder.header.tab.evaluations": "Evaluations", + "agents.builder.header.tab.agentEvals": "Evals", "agents.builder.header.tab.raw": "Raw", "agents.builder.header.workspaceCrumb": "Workspace", "agents.builder.header.projectFallback": "Project", @@ -7766,12 +7766,11 @@ "agents.builder.executions.count": "{count} session | {count} sessions", "agents.builder.settings.description": "Configure memory, sub-agent routing, and advanced runtime settings.", "agents.builder.raw.description": "Agent JSON configuration", - "agents.builder.evaluations.comingSoon": "Evaluations functionality is coming soon.", - "agents.builder.evaluations.configuredInCode": "{count} evaluation configured in code | {count} evaluations configured in code", - "agents.builder.evaluations.type.check": "Check", - "agents.builder.evaluations.type.judge": "Judge", - "agents.builder.evaluations.credentialConfigured": "Credential configured", - "agents.builder.evaluations.emptyPrefix": "No evaluations configured.", + "agents.builder.agentEvals.empty.title": "No test cases yet", + "agents.builder.agentEvals.empty.description": "Draft a few realistic requests from your agent's own instructions, run them through the agent, then tell us which answers are wrong.", + "agents.builder.agentEvals.empty.generate": "Generate test cases", + "agents.builder.agentEvals.generated": "{count} test case generated | {count} test cases generated", + "agents.builder.agentEvals.generateError": "Couldn't generate test cases", "agents.builder.quickActions.memoriesUsed.count": "{count} memory | {count} memories", "agents.builder.quickActions.memoriesUsed.keyMemory": "Memory", "agents.builder.capabilities.title": "Capabilities", diff --git a/packages/frontend/@n8n/stores/src/constants.ts b/packages/frontend/@n8n/stores/src/constants.ts index 351322c4c88..0112622629d 100644 --- a/packages/frontend/@n8n/stores/src/constants.ts +++ b/packages/frontend/@n8n/stores/src/constants.ts @@ -36,6 +36,7 @@ export const STORES = { API_KEYS: 'apiKeys', EVALUATION: 'evaluation', EVAL_COLLECTIONS: 'evalCollections', + AGENT_EVALS: 'agentEvals', FOLDERS: 'folders', MODULES: 'modules', FOCUS_PANEL: 'focusPanel', diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderEditorColumn.spec.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderEditorColumn.spec.ts index 448d2ae1998..d04012f0f2a 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderEditorColumn.spec.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderEditorColumn.spec.ts @@ -2,6 +2,8 @@ import { createTestingPinia } from '@pinia/testing'; import { describe, it, expect, vi, beforeEach } from 'vitest'; import { flushPromises, mount } from '@vue/test-utils'; +import type { AgentBuilderMainTab } from '../composables/useAgentBuilderMainTabs'; + vi.mock('@n8n/i18n', () => ({ useI18n: () => ({ baseText: (key: string) => @@ -127,12 +129,13 @@ vi.setConfig({ testTimeout: 30_000 }); async function mountColumn( overrides: Partial<{ - activeMainTab: 'agent' | 'knowledge' | 'sessions' | 'settings'; + activeMainTab: AgentBuilderMainTab; mainTabOptions: Array<{ label: string; - value: 'agent' | 'knowledge' | 'sessions' | 'settings'; + value: AgentBuilderMainTab; }>; knowledgeBaseEnabled: boolean; + canEditAgent: boolean; }> = {}, ) { const { default: AgentBuilderEditorColumn } = await import( @@ -162,7 +165,7 @@ async function mountColumn( knowledgeBaseEnabled: overrides.knowledgeBaseEnabled ?? true, appliedSkills: [], connectedTriggers: [], - canEditAgent: true, + canEditAgent: overrides.canEditAgent ?? true, executionsDescription: '', }, global: { @@ -244,6 +247,7 @@ describe('AgentBuilderEditorColumn', () => { ['knowledge', 'agent-knowledge-tab-content'], ['sessions', 'agent-sessions-tab-content'], ['settings', 'agent-settings-tab-content'], + ['evals', 'agent-evals-tab-content'], ] as const)('renders the %s tab through the shared tab panel', async (activeMainTab, testId) => { const wrapper = await mountColumn({ activeMainTab }); @@ -252,6 +256,22 @@ describe('AgentBuilderEditorColumn', () => { expect(panels[0].attributes('data-testid')).toBe(testId); }); + it('forwards the evals generate request to the host', async () => { + const wrapper = await mountColumn({ activeMainTab: 'evals' }); + + // The shell owns no generation flow yet, so the request has to reach the + // host rather than being emitted into nothing. + wrapper.getComponent({ name: 'AgentEvalsSection' }).vm.$emit('generate'); + + expect(wrapper.emitted('generate-eval-cases')).toHaveLength(1); + }); + + it('disables the evals CTA for a read-only agent', async () => { + const wrapper = await mountColumn({ activeMainTab: 'evals', canEditAgent: false }); + + expect(wrapper.getComponent({ name: 'AgentEvalsSection' }).props('disabled')).toBe(true); + }); + it('uses embedded session list spacing inside the Sessions tab panel', async () => { const wrapper = await mountColumn({ activeMainTab: 'sessions' }); diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts index 886422e486b..40c68d84f09 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts @@ -144,6 +144,12 @@ vi.mock('../composables/useAgentApi', () => ({ getAgentConfigValidation: getAgentConfigValidationMock, })); +const generateDraftCasesMock = vi.fn(); +vi.mock('../agentEvals.api', () => ({ + getDatasets: vi.fn().mockResolvedValue([]), + generateDraftCases: (...args: unknown[]) => generateDraftCasesMock(...args), +})); + const builderTelemetryMock = vi.hoisted(() => ({ fetchInitialTriggersBaseline: vi.fn().mockResolvedValue(null), trackTriggerAdded: vi.fn(), @@ -792,6 +798,37 @@ describe('AgentBuilderView — preview routing', { timeout: 60_000 }, () => { ); }); + it('generates eval cases and confirms the result with a toast', async () => { + generateDraftCasesMock.mockResolvedValue({ + datasetId: 'd1', + dataTableId: 'dt1', + cases: [ + { input: 'a', whatToCheck: 'x' }, + { input: 'b', whatToCheck: 'y' }, + ], + }); + const wrapper = await renderView(); + + wrapper.findComponent({ name: 'AgentBuilderEditorColumn' }).vm.$emit('generate-eval-cases'); + await flushPromises(); + + expect(generateDraftCasesMock).toHaveBeenCalledWith(expect.anything(), 'p1', 'a1', {}); + // Nothing renders the generated cases yet, so the toast is the only + // signal the work landed — without it the click looks like a no-op. + expect(showMessageMock).toHaveBeenCalledWith(expect.objectContaining({ type: 'success' })); + }); + + it('surfaces a failed eval-case generation', async () => { + generateDraftCasesMock.mockRejectedValue(new Error('no model configured')); + const wrapper = await renderView(); + + wrapper.findComponent({ name: 'AgentBuilderEditorColumn' }).vm.$emit('generate-eval-cases'); + await flushPromises(); + + expect(showErrorMock).toHaveBeenCalled(); + expect(showMessageMock).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'success' })); + }); + it('uploads knowledge files for an unpublished agent', async () => { // Default makeAgentResponse() has activeVersionId: null (unpublished). const wrapper = await renderView({ knowledgeBaseEnabled: true }); diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts new file mode 100644 index 00000000000..dcbe88df608 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect, vi } from 'vitest'; +import { configure } from '@testing-library/vue'; +import userEvent from '@testing-library/user-event'; + +import { createComponentRenderer } from '@/__tests__/render'; +import AgentEvalsSection from '../components/AgentEvalsSection.vue'; + +// Components use `data-testid`; the global setup configures `data-test-id`. +configure({ testIdAttribute: 'data-testid' }); + +vi.mock('@n8n/i18n', async (importOriginal) => ({ + ...(await importOriginal()), + useI18n: () => ({ baseText: (key: string) => `mocked-${key}` }), +})); + +const renderComponent = createComponentRenderer(AgentEvalsSection); + +describe('AgentEvalsSection', () => { + it('renders the first-run state with its title, description and CTA', () => { + const { getByTestId, getByText } = renderComponent(); + + expect(getByTestId('agent-evals-section')).toBeInTheDocument(); + expect(getByTestId('agent-evals-empty-state')).toBeInTheDocument(); + // Asserting on the mocked key prefix verifies the keys we intended rather + // than arbitrary copy. + expect(getByText('mocked-agents.builder.agentEvals.empty.title')).toBeInTheDocument(); + expect(getByText('mocked-agents.builder.agentEvals.empty.description')).toBeInTheDocument(); + }); + + it('emits generate when the CTA is clicked', async () => { + const { getByTestId, emitted } = renderComponent(); + + await userEvent.click(getByTestId('agent-evals-generate-button')); + + expect(emitted('generate')).toBeTruthy(); + }); + + it('disables the CTA for users who cannot edit the agent', () => { + const { getByTestId } = renderComponent({ props: { disabled: true } }); + + expect(getByTestId('agent-evals-generate-button')).toBeDisabled(); + }); +}); diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/agentEvals.store.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/agentEvals.store.test.ts new file mode 100644 index 00000000000..334960cd175 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/agentEvals.store.test.ts @@ -0,0 +1,118 @@ +import { createPinia, setActivePinia } from 'pinia'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useAgentEvalsStore } from '../agentEvals.store'; +import type { AgentEvalDatasetRecord } from '../agentEvals.types'; + +const { getDatasets, generateDraftCases } = vi.hoisted(() => ({ + getDatasets: vi.fn(), + generateDraftCases: vi.fn(), +})); + +vi.mock('../agentEvals.api', () => ({ + getDatasets, + generateDraftCases, +})); + +vi.mock('@n8n/stores/useRootStore', () => ({ + useRootStore: vi.fn(() => ({ + restApiContext: { instanceId: 'test-instance-id' }, + })), +})); + +const PROJECT_ID = 'project-1'; +const AGENT_ID = 'agent-1'; + +const dataset = (id: string, name = `dataset-${id}`): AgentEvalDatasetRecord => ({ + id, + name, + description: null, + agentId: AGENT_ID, + columnMapping: null, + createdById: null, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + datasetSource: 'data_table', + datasetRef: { dataTableId: 'dt-1' }, +}); + +describe('useAgentEvalsStore', () => { + beforeEach(() => { + setActivePinia(createPinia()); + vi.clearAllMocks(); + }); + + describe('fetchDatasets', () => { + it('caches datasets under the agent id', async () => { + getDatasets.mockResolvedValue([dataset('d1')]); + const store = useAgentEvalsStore(); + + await store.fetchDatasets(PROJECT_ID, AGENT_ID); + + expect(getDatasets).toHaveBeenCalledWith( + { instanceId: 'test-instance-id' }, + PROJECT_ID, + AGENT_ID, + ); + expect(store.getDatasets(AGENT_ID)).toHaveLength(1); + // Another agent's cache must stay empty — switching agents in the + // builder should never surface the previous agent's datasets. + expect(store.getDatasets('agent-2')).toEqual([]); + }); + + it('distinguishes "not loaded" from "loaded but empty"', async () => { + getDatasets.mockResolvedValue([]); + const store = useAgentEvalsStore(); + + expect(store.isLoaded(AGENT_ID)).toBe(false); + await store.fetchDatasets(PROJECT_ID, AGENT_ID); + + expect(store.isLoaded(AGENT_ID)).toBe(true); + expect(store.getDatasets(AGENT_ID)).toEqual([]); + }); + + it('clears the loading flag when the request rejects', async () => { + getDatasets.mockRejectedValue(new Error('boom')); + const store = useAgentEvalsStore(); + + await expect(store.fetchDatasets(PROJECT_ID, AGENT_ID)).rejects.toThrow('boom'); + + expect(store.isLoadingDatasets(AGENT_ID)).toBe(false); + }); + }); + + describe('generateDraftCases', () => { + it('re-reads the dataset list once the server has persisted the drafts', async () => { + generateDraftCases.mockResolvedValue({ + datasetId: 'd1', + dataTableId: 'dt-1', + cases: [{ input: 'hi', whatToCheck: 'is polite' }], + }); + getDatasets.mockResolvedValue([dataset('d1')]); + const store = useAgentEvalsStore(); + + const result = await store.generateDraftCases(PROJECT_ID, AGENT_ID, { count: 3 }); + + expect(generateDraftCases).toHaveBeenCalledWith( + { instanceId: 'test-instance-id' }, + PROJECT_ID, + AGENT_ID, + { count: 3 }, + ); + expect(result.cases).toHaveLength(1); + // The generate response carries drafts, not the dataset row, so the + // cache has to come from the follow-up read. + expect(getDatasets).toHaveBeenCalledTimes(1); + expect(store.getDatasets(AGENT_ID).map((d) => d.id)).toEqual(['d1']); + }); + + it('clears the generating flag when generation rejects', async () => { + generateDraftCases.mockRejectedValue(new Error('no model')); + const store = useAgentEvalsStore(); + + await expect(store.generateDraftCases(PROJECT_ID, AGENT_ID)).rejects.toThrow('no model'); + + expect(store.isGeneratingCases(AGENT_ID)).toBe(false); + }); + }); +}); diff --git a/packages/frontend/editor-ui/src/features/agents/agentEvals.api.ts b/packages/frontend/editor-ui/src/features/agents/agentEvals.api.ts new file mode 100644 index 00000000000..b27801fa3cc --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/agentEvals.api.ts @@ -0,0 +1,41 @@ +import type { + AgentEvalDatasetRecord, + GenerateDraftCasesOptions, + GenerateDraftCasesResult, +} from '@n8n/api-types'; +import type { IRestApiContext } from '@n8n/rest-api-client'; +import { makeRestApiRequest } from '@n8n/rest-api-client'; + +// REST path helper, kept inline so callers can't build a URL that drifts from +// the routes in the agent-evals controller. Every route below it is gated +// per-user on `101_agent_evals` server-side as well as by `useAgentEvalsFlag`. +// +// Reads the eval surface needs to render itself, and nothing more. Dataset +// create/update/delete belong with the dataset-management UI that uses them; +// run and result reads wait on their paginated response shapes. +const evalsPath = (projectId: string, agentId: string) => + `/projects/${projectId}/agents/v2/${agentId}/evals`; + +export const getDatasets = async (context: IRestApiContext, projectId: string, agentId: string) => { + return await makeRestApiRequest( + context, + 'GET', + `${evalsPath(projectId, agentId)}/datasets`, + ); +}; + +// Drafts cases from the agent's own config and persists them as a new dataset, +// so the response carries both the dataset it created and the drafts to review. +export const generateDraftCases = async ( + context: IRestApiContext, + projectId: string, + agentId: string, + options: GenerateDraftCasesOptions = {}, +) => { + return await makeRestApiRequest( + context, + 'POST', + `${evalsPath(projectId, agentId)}/generate`, + options, + ); +}; diff --git a/packages/frontend/editor-ui/src/features/agents/agentEvals.store.ts b/packages/frontend/editor-ui/src/features/agents/agentEvals.store.ts new file mode 100644 index 00000000000..08eef7d395c --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/agentEvals.store.ts @@ -0,0 +1,89 @@ +import { defineStore } from 'pinia'; +import { ref } from 'vue'; + +import { STORES } from '@n8n/stores'; +import { useRootStore } from '@n8n/stores/useRootStore'; + +import * as agentEvalsApi from './agentEvals.api'; +import type { AgentEvalDatasetRecord, GenerateDraftCasesOptions } from './agentEvals.types'; + +/** + * Client state for an agent's eval datasets. + * + * Runs, per-case results and ratings are deliberately not here yet: their list + * responses are being reshaped into paginated envelopes, so binding them now + * would only have to be redone. Add them alongside the views that read them. + */ +export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => { + const rootStore = useRootStore(); + + // Keyed by agentId so switching agents inside the builder can't render the + // previous agent's datasets. + const datasetsByAgentId = ref>({}); + + const loadingDatasets = ref>({}); + const generatingCases = ref>({}); + + const getDatasets = (agentId: string) => datasetsByAgentId.value[agentId] ?? []; + + // Absence of a cache entry is "not loaded yet", not "none" — callers that + // need to distinguish the two should check `isLoaded` before `getDatasets`. + const isLoaded = (agentId: string) => datasetsByAgentId.value[agentId] !== undefined; + + const isLoadingDatasets = (agentId: string) => loadingDatasets.value[agentId] === true; + + const isGeneratingCases = (agentId: string) => generatingCases.value[agentId] === true; + + const setDatasets = (agentId: string, datasets: AgentEvalDatasetRecord[]) => { + datasetsByAgentId.value = { ...datasetsByAgentId.value, [agentId]: datasets }; + }; + + const fetchDatasets = async (projectId: string, agentId: string) => { + loadingDatasets.value = { ...loadingDatasets.value, [agentId]: true }; + try { + const datasets = await agentEvalsApi.getDatasets( + rootStore.restApiContext, + projectId, + agentId, + ); + setDatasets(agentId, datasets); + return datasets; + } finally { + loadingDatasets.value = { ...loadingDatasets.value, [agentId]: false }; + } + }; + + // The response carries the drafts, not the dataset row, so the list is + // re-read rather than patched. Best-effort: generation already succeeded + // server-side and cost model credits, so a transient refresh failure must not + // surface as a generation failure (user retries → duplicate dataset). A stale + // cache self-heals on the next fetch. + const generateDraftCases = async ( + projectId: string, + agentId: string, + options: GenerateDraftCasesOptions = {}, + ) => { + generatingCases.value = { ...generatingCases.value, [agentId]: true }; + try { + const result = await agentEvalsApi.generateDraftCases( + rootStore.restApiContext, + projectId, + agentId, + options, + ); + await fetchDatasets(projectId, agentId).catch(() => null); + return result; + } finally { + generatingCases.value = { ...generatingCases.value, [agentId]: false }; + } + }; + + return { + getDatasets, + isLoaded, + isLoadingDatasets, + isGeneratingCases, + fetchDatasets, + generateDraftCases, + }; +}); diff --git a/packages/frontend/editor-ui/src/features/agents/agentEvals.types.ts b/packages/frontend/editor-ui/src/features/agents/agentEvals.types.ts new file mode 100644 index 00000000000..1f2f189dd9b --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/agentEvals.types.ts @@ -0,0 +1,13 @@ +// Re-exports from @n8n/api-types so agent-eval FE callers don't reach across +// packages. New shared type? Export it from `@n8n/api-types` first. +// +// Run/result/rating shapes are deliberately absent: their response types are +// still being reshaped into paginated envelopes, so nothing here should bind to +// them yet. + +export type { + AgentEvalDatasetRecord, + AgentEvalDraftCase, + GenerateDraftCasesOptions, + GenerateDraftCasesResult, +} from '@n8n/api-types'; diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue index 10eab8836eb..7bc29d0f97a 100644 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentBuilderEditorColumn.vue @@ -25,6 +25,7 @@ import AgentMcpPanel from './AgentMcpPanel.vue'; import AgentMemoryPanel from './AgentMemoryPanel.vue'; import AgentSubAgentsPanel from './AgentSubAgentsPanel.vue'; import AgentBuilderTabPanel from './AgentBuilderTabPanel.vue'; +import AgentEvalsSection from './AgentEvalsSection.vue'; const props = defineProps<{ activeMainTab: AgentBuilderMainTab; @@ -43,6 +44,7 @@ const props = defineProps<{ canEditAgent: boolean; agentAvailableInMcp?: boolean; executionsDescription: string; + generatingEvalCases?: boolean; tasksReloadKey?: number; artifactMode?: boolean; /** No agent row exists yet, so agent-scoped endpoints would 404. */ @@ -78,6 +80,7 @@ const emit = defineEmits<{ 'toggle-mcp-access': [enabled: boolean]; 'tasks-changed': []; 'agent-changed': []; + 'generate-eval-cases': []; }>(); const i18n = useI18n(); @@ -255,6 +258,17 @@ const i18n = useI18n(); + + + + diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsPanel.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsPanel.vue deleted file mode 100644 index e0643dd9dfb..00000000000 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsPanel.vue +++ /dev/null @@ -1,137 +0,0 @@ - - - - - diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue new file mode 100644 index 00000000000..500081d9730 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue @@ -0,0 +1,93 @@ + + + + + diff --git a/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.test.ts b/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.test.ts new file mode 100644 index 00000000000..2aaa9bd02d8 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.test.ts @@ -0,0 +1,109 @@ +import { computed, nextTick, ref } from 'vue'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useAgentBuilderMainTabs } from './useAgentBuilderMainTabs'; +import { EXECUTIONS_SECTION_KEY } from '../constants'; + +const route = { query: {} as Record }; +const replace = vi.fn(async (to: { query: Record }) => { + // Mirror the router: `replace` is what makes the query the source of truth, + // so the composable's own watcher has to see the new value. + route.query = to.query; + await nextTick(); +}); + +vi.mock('vue-router', () => ({ + useRoute: () => route, + useRouter: () => ({ replace }), +})); + +vi.mock('@n8n/i18n', async (importOriginal) => ({ + ...(await importOriginal()), + useI18n: () => ({ baseText: (key: string) => `mocked-${key}` }), +})); + +const evalsFlag = ref(false); +vi.mock('@/features/ai/evaluation.ee/composables/useAgentEvalsFlag', () => ({ + useAgentEvalsFlag: () => evalsFlag, +})); + +const setup = () => useAgentBuilderMainTabs({ executionsCount: computed(() => 0) }); + +const tabValues = () => setup().mainTabOptions.value.map((tab) => tab.value); + +describe('useAgentBuilderMainTabs', () => { + beforeEach(() => { + route.query = {}; + evalsFlag.value = false; + replace.mockClear(); + }); + + describe('flag gating', () => { + it('omits the evals tab entirely when the flag is off', () => { + expect(tabValues()).toEqual(['agent', 'knowledge', 'sessions', 'settings']); + }); + + it('appends the evals tab after settings when the flag is on', () => { + evalsFlag.value = true; + + expect(tabValues()).toEqual(['agent', 'knowledge', 'sessions', 'settings', 'evals']); + }); + + it('reacts to the flag resolving after the first frame', async () => { + const { mainTabOptions } = setup(); + expect(mainTabOptions.value.map((tab) => tab.value)).not.toContain('evals'); + + // PostHog resolves asynchronously, so the tab row has to pick it up + // rather than being computed once at setup. + evalsFlag.value = true; + await nextTick(); + + expect(mainTabOptions.value.map((tab) => tab.value)).toContain('evals'); + }); + }); + + describe('section query param', () => { + it('round-trips evals through the query param like the sibling tabs', async () => { + evalsFlag.value = true; + const { activeMainTab } = setup(); + + activeMainTab.value = 'evals'; + await nextTick(); + + expect(replace).toHaveBeenCalledWith({ query: { section: 'evals' } }); + expect(activeMainTab.value).toBe('evals'); + }); + + it('selects the evals tab from a deep link when the flag is on', () => { + evalsFlag.value = true; + route.query = { section: 'evals' }; + + expect(setup().activeMainTab.value).toBe('evals'); + }); + + it('falls back to the agent tab for an evals deep link while the flag is off', () => { + route.query = { section: 'evals' }; + + // Selecting a tab that is absent from the row would leave the row with + // nothing highlighted, so an unreachable deep link collapses to default. + expect(setup().activeMainTab.value).toBe('agent'); + }); + + it('honours an evals deep link once a late-resolving flag turns on', async () => { + route.query = { section: 'evals' }; + const { activeMainTab } = setup(); + expect(activeMainTab.value).toBe('agent'); + + evalsFlag.value = true; + await nextTick(); + + expect(activeMainTab.value).toBe('evals'); + }); + + it('leaves the sibling tabs unaffected by the flag', () => { + route.query = { section: EXECUTIONS_SECTION_KEY }; + + expect(setup().activeMainTab.value).toBe('sessions'); + }); + }); +}); diff --git a/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.ts b/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.ts index 711dd594952..192eec29618 100644 --- a/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.ts +++ b/packages/frontend/editor-ui/src/features/agents/composables/useAgentBuilderMainTabs.ts @@ -4,21 +4,31 @@ import { useRoute, useRouter } from 'vue-router'; import type { LocationQueryValue } from 'vue-router'; import { useI18n, type BaseTextKey } from '@n8n/i18n'; +import { useAgentEvalsFlag } from '@/features/ai/evaluation.ee/composables/useAgentEvalsFlag'; import { EXECUTIONS_SECTION_KEY } from '../constants'; -export type AgentBuilderMainTab = 'agent' | 'knowledge' | 'sessions' | 'settings'; +export type AgentBuilderMainTab = 'agent' | 'knowledge' | 'sessions' | 'settings' | 'evals'; -type AgentBuilderSection = 'knowledge' | typeof EXECUTIONS_SECTION_KEY | 'settings' | null; +type AgentBuilderSection = + | 'knowledge' + | typeof EXECUTIONS_SECTION_KEY + | 'settings' + | 'evals' + | null; const SECTION_QUERY_PARAM = 'section'; function getSectionFromQuery( section: LocationQueryValue | LocationQueryValue[] | undefined, + isEvalsEnabled: boolean, ): AgentBuilderSection { const value = Array.isArray(section) ? section[0] : section; if (value === 'knowledge' || value === EXECUTIONS_SECTION_KEY || value === 'settings') { return value; } + // A deep link to the evals surface falls back to the default tab while the + // flag is off, rather than selecting a tab that isn't in the row. + if (value === 'evals' && isEvalsEnabled) return 'evals'; return null; } @@ -26,6 +36,7 @@ function getSectionFromTab(tab: AgentBuilderMainTab): AgentBuilderSection { if (tab === 'knowledge') return 'knowledge'; if (tab === 'sessions') return EXECUTIONS_SECTION_KEY; if (tab === 'settings') return 'settings'; + if (tab === 'evals') return 'evals'; return null; } @@ -39,6 +50,7 @@ export function useAgentBuilderMainTabs({ const route = useRoute(); const router = useRouter(); const i18n = useI18n(); + const isEvalsEnabled = useAgentEvalsFlag(); const selectedSection = ref(null); async function setSelectedSection(section: AgentBuilderSection) { @@ -54,6 +66,7 @@ export function useAgentBuilderMainTabs({ if (selectedSection.value === 'knowledge') return 'knowledge'; if (selectedSection.value === EXECUTIONS_SECTION_KEY) return 'sessions'; if (selectedSection.value === 'settings') return 'settings'; + if (selectedSection.value === 'evals') return 'evals'; return 'agent'; }, set(tab) { @@ -72,6 +85,16 @@ export function useAgentBuilderMainTabs({ label: i18n.baseText('agents.builder.header.tab.settings' as BaseTextKey), value: 'settings', }, + // Absent rather than disabled while the flag is off — a disabled tab + // advertises a surface the user has no way to reach. + ...(isEvalsEnabled.value + ? [ + { + label: i18n.baseText('agents.builder.header.tab.agentEvals'), + value: 'evals' as const, + }, + ] + : []), ]); const executionsDescription = computed(() => @@ -81,11 +104,14 @@ export function useAgentBuilderMainTabs({ }), ); + // The flag is a watch source, not just read inside: PostHog resolves after the + // first frame, so a `?section=evals` deep link has to be re-derived once the + // flag arrives or it would stay collapsed to the default tab. watch( - () => [routeBacked.value, route.query[SECTION_QUERY_PARAM]] as const, - ([isRouteBacked, section]) => { + () => [routeBacked.value, route.query[SECTION_QUERY_PARAM], isEvalsEnabled.value] as const, + ([isRouteBacked, section, evalsEnabled]) => { if (!isRouteBacked) return; - selectedSection.value = getSectionFromQuery(section); + selectedSection.value = getSectionFromQuery(section, evalsEnabled); }, { immediate: true }, ); diff --git a/packages/frontend/editor-ui/src/features/agents/views/AgentBuilderView.vue b/packages/frontend/editor-ui/src/features/agents/views/AgentBuilderView.vue index 20cd7eb1d4f..a5d98c8eb0a 100644 --- a/packages/frontend/editor-ui/src/features/agents/views/AgentBuilderView.vue +++ b/packages/frontend/editor-ui/src/features/agents/views/AgentBuilderView.vue @@ -53,6 +53,7 @@ import { useAgentConfig } from '../composables/useAgentConfig'; import { useAgentConfigValidation } from '../composables/useAgentConfigValidation'; import { useAgentPermissions } from '../composables/useAgentPermissions'; import { useAgentSessionsStore } from '../agentSessions.store'; +import { useAgentEvalsStore } from '../agentEvals.store'; import { useAgentBuilderSession } from '../composables/useAgentBuilderSession'; import { useAgentConfigAutosave } from '../composables/useAgentConfigAutosave'; import { useAgentBuilderMainTabs } from '../composables/useAgentBuilderMainTabs'; @@ -120,6 +121,7 @@ const instanceAiAvailable = useInstanceAiAvailable(); const { canSendPreviewToInstanceAi, sendPreviewSessionToInstanceAi } = useInstanceAiAgentPreviewHandoff(); const sessionsStore = useAgentSessionsStore(); +const agentEvalsStore = useAgentEvalsStore(); const credentialsStore = useCredentialsStore(); const settingsStore = useSettingsStore(); const uiStore = useUIStore(); @@ -340,6 +342,30 @@ function isStaleAgentTarget(targetProjectId: string, targetAgentId: string): boo return projectId.value !== targetProjectId || agentId.value !== targetAgentId; } +// Drafts cases from the agent's own config. The generated dataset isn't +// rendered yet, so the toast is the only confirmation the user gets that the +// work landed. +async function onGenerateEvalCases() { + const targetProjectId = projectId.value; + const targetAgentId = agentId.value; + if (!targetProjectId || !targetAgentId) return; + + try { + const { cases } = await agentEvalsStore.generateDraftCases(targetProjectId, targetAgentId); + if (isStaleAgentTarget(targetProjectId, targetAgentId)) return; + showMessage({ + title: locale.baseText('agents.builder.agentEvals.generated', { + adjustToNumber: cases.length, + interpolate: { count: String(cases.length) }, + }), + type: 'success', + }); + } catch (error) { + if (isStaleAgentTarget(targetProjectId, targetAgentId)) return; + showError(error, locale.baseText('agents.builder.agentEvals.generateError')); + } +} + async function fetchAgent( targetProjectId: string = projectId.value, targetAgentId: string = agentId.value, @@ -1593,6 +1619,7 @@ function onPreviewBreadcrumbSelect(item: PathItem) { :agent-unsaved="isUnsaved" :ensure-agent-persisted="ensureAgentPersisted" :executions-description="executionsDescription" + :generating-eval-cases="agentEvalsStore.isGeneratingCases(agentId)" :artifact-mode="isArtifactMode" :config-validation-issues="configValidation?.issues ?? []" @update:config="onConfigFieldUpdate" @@ -1613,6 +1640,7 @@ function onPreviewBreadcrumbSelect(item: PathItem) { @toggle-mcp-access="onToggleMcpAccess" @tasks-changed="() => onConfigUpdated()" @agent-changed="refreshAgentAfterIntegrationChange" + @generate-eval-cases="onGenerateEvalCases" />