From ce20d1dea4ac77d6f6591027c1b4915c6af30d3d Mon Sep 17 00:00:00 2001 From: Luka Zivkovic <32032339+luka-zivkovic@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:42:27 +0000 Subject: [PATCH] fix: Load agent evals by default and share the rollout flag (no-changelog) (#40915) --- .../modules/__tests__/module-registry.test.ts | 13 ++- .../src/modules/module-registry.ts | 1 + .../config/src/configs/evaluation.config.ts | 1 + .../modules/agent-evals/agent-evals.module.ts | 2 +- .../src/app/constants/experiments.ts | 6 -- .../useTestAgentPreviewExperiment.test.ts | 54 ----------- .../useTestAgentPreviewExperiment.ts | 22 ----- .../agents/__tests__/AgentBuilderView.test.ts | 22 +++-- .../__tests__/AgentEvalsSection.test.ts | 6 +- .../components/AgentEvalChecksPanel.vue | 4 +- .../AgentEvalsEmptyStatePreview.vue | 2 +- .../agents/components/AgentEvalsSection.vue | 4 +- .../useTestAgentPreviewFlag.test.ts | 69 ++++++++++++++ .../composables/useTestAgentPreviewFlag.ts | 12 +++ .../ai/instanceAi/InstanceAiThreadView.vue | 8 +- .../__tests__/InstanceAiThreadView.test.ts | 90 +++++++++---------- .../InstanceAiTestAgentPreviewPanel.vue | 2 +- 17 files changed, 166 insertions(+), 152 deletions(-) delete mode 100644 packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.test.ts delete mode 100644 packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.ts create mode 100644 packages/frontend/editor-ui/src/features/agents/composables/useTestAgentPreviewFlag.test.ts create mode 100644 packages/frontend/editor-ui/src/features/agents/composables/useTestAgentPreviewFlag.ts diff --git a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts index f16efeb1b74..514c2a325e1 100644 --- a/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts +++ b/packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts @@ -57,11 +57,14 @@ describe('eligibleModules', () => { expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('policy-infrastructure'); }); - it.each(['instance-ai', 'agents'])('should include %s by default', (moduleName) => { - expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName); - }); + it.each(['instance-ai', 'agents', 'agent-evals'])( + 'should include %s by default', + (moduleName) => { + expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName); + }, + ); - it.each(['instance-ai', 'agents'])( + it.each(['instance-ai', 'agents', 'agent-evals'])( 'should allow opting out of the default %s module via env var', (moduleName) => { process.env.N8N_DISABLED_MODULES = moduleName; @@ -104,6 +107,7 @@ describe('eligibleModules', () => { 'workflow-reviews', 'instance-ai', 'agents', + 'agent-evals', 'scim', ]); }); @@ -144,6 +148,7 @@ describe('eligibleModules', () => { 'workflow-reviews', 'instance-ai', 'agents', + 'agent-evals', 'scim', 'type-availability-policies', ]); diff --git a/packages/@n8n/backend-common/src/modules/module-registry.ts b/packages/@n8n/backend-common/src/modules/module-registry.ts index d0ce242e1c3..5324c11e703 100644 --- a/packages/@n8n/backend-common/src/modules/module-registry.ts +++ b/packages/@n8n/backend-common/src/modules/module-registry.ts @@ -83,6 +83,7 @@ export class ModuleRegistry { 'workflow-reviews', 'instance-ai', 'agents', + 'agent-evals', 'scim', ]; diff --git a/packages/@n8n/config/src/configs/evaluation.config.ts b/packages/@n8n/config/src/configs/evaluation.config.ts index 4cb3aed9dbc..da4be511713 100644 --- a/packages/@n8n/config/src/configs/evaluation.config.ts +++ b/packages/@n8n/config/src/configs/evaluation.config.ts @@ -59,6 +59,7 @@ export class EvaluationConfig { * * Useful for local development and QA environments that need to exercise * the offer without building out a fully-equipped agent first. + * The agent-evals feature gate still applies. */ @Env('N8N_FORCE_AGENT_WORTH_TESTING') forceAgentWorthTesting: boolean = false; diff --git a/packages/cli/src/modules/agent-evals/agent-evals.module.ts b/packages/cli/src/modules/agent-evals/agent-evals.module.ts index 28e97cee9a5..483c2ca4103 100644 --- a/packages/cli/src/modules/agent-evals/agent-evals.module.ts +++ b/packages/cli/src/modules/agent-evals/agent-evals.module.ts @@ -7,7 +7,7 @@ import { Container } from '@n8n/di'; * per-case results, and serves the REST surface the editor consumes. * * Main-process only — the reused tool-mock substrate can't cross a queue - * boundary. Opt-in module, and dark until `101_agent_evals` is on for the user. + * boundary. Loaded by default; `101_agent_evals` gates access for each user. */ @BackendModule({ name: 'agent-evals', instanceTypes: ['main'] }) export class AgentEvalsModule implements ModuleInterface { diff --git a/packages/frontend/editor-ui/src/app/constants/experiments.ts b/packages/frontend/editor-ui/src/app/constants/experiments.ts index e33c3d297c5..79c9f537d5e 100644 --- a/packages/frontend/editor-ui/src/app/constants/experiments.ts +++ b/packages/frontend/editor-ui/src/app/constants/experiments.ts @@ -87,11 +87,6 @@ export const WORKFLOW_CARD_MCP_TOGGLE_EXPERIMENT = createExperiment('086_workflo export const INSTANCE_AI_PROACTIVE_AGENT_EXPERIMENT = createExperiment( '082_instance_ai_proactive_agent', ); -// Flag key assigned when the PostHog flag is created — follow the -// `n8n:experiments` skill to create it and fill in the real key here. -export const INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT = createExperiment( - '_test_agent_preview', -); // Experiment cleanup: remove with instanceAiPromptSuggestionsV2. export const INSTANCE_AI_PROMPT_SUGGESTIONS_V2_EXPERIMENT = createExperiment( '085_instance_ai_prompt_suggestions_v2', @@ -217,6 +212,5 @@ export const EXPERIMENTS_TO_TRACK = [ CREDENTIAL_DESCRIPTIONS_EXPERIMENT.name, SURFACE_ASSISTANT_ON_WORKFLOW_ERROR_EXPERIMENT.name, AGENTS_LIST_EMPTY_STATE_TEMPLATES_EXPERIMENT.name, - INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name, AGENTS_N8N_CHAT_EXPERIMENT.name, ]; diff --git a/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.test.ts b/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.test.ts deleted file mode 100644 index f5621c5da88..00000000000 --- a/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it, vi, beforeEach } from 'vitest'; - -import { - EXPERIMENTS_TO_TRACK, - INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT, -} from '@/app/constants/experiments'; - -import { useTestAgentPreviewExperiment } from './useTestAgentPreviewExperiment'; - -const getVariant = vi.fn(); -const settingsState = { forceAgentWorthTesting: false }; - -vi.mock('@/app/stores/posthog.store', () => ({ - usePostHog: vi.fn(() => ({ - getVariant, - })), -})); - -vi.mock('@n8n/stores/settings.store', () => ({ - useSettingsStore: () => ({ - settings: { evaluation: { forceAgentWorthTesting: settingsState.forceAgentWorthTesting } }, - }), -})); - -describe('useTestAgentPreviewExperiment', () => { - beforeEach(() => { - getVariant.mockReset(); - settingsState.forceAgentWorthTesting = false; - }); - - it.each([ - { variant: INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.variant, enabled: true }, - { variant: INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.control, enabled: false }, - { variant: undefined, enabled: false }, - ])('returns $enabled when PostHog variant is $variant', ({ variant, enabled }) => { - getVariant.mockReturnValue(variant); - - const { isFeatureEnabled } = useTestAgentPreviewExperiment(); - - expect(isFeatureEnabled.value).toBe(enabled); - expect(getVariant).toHaveBeenCalledWith(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name); - }); - - it('is enabled via the forceAgentWorthTesting override even when PostHog is off', () => { - getVariant.mockReturnValue(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.control); - settingsState.forceAgentWorthTesting = true; - - expect(useTestAgentPreviewExperiment().isFeatureEnabled.value).toBe(true); - }); - - it('registers the experiment for centralized enrollment tracking', () => { - expect(EXPERIMENTS_TO_TRACK).toContain(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name); - }); -}); diff --git a/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.ts b/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.ts deleted file mode 100644 index 5f1d2b015b5..00000000000 --- a/packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { computed } from 'vue'; - -import { INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT } from '@/app/constants/experiments'; -import { usePostHog } from '@/app/stores/posthog.store'; -import { useSettingsStore } from '@n8n/stores/settings.store'; - -export function useTestAgentPreviewExperiment() { - const posthogStore = usePostHog(); - const settingsStore = useSettingsStore(); - - // `N8N_FORCE_AGENT_WORTH_TESTING` also forces this variant on, so a local/QA - // environment gets the full preview flow from one flag instead of also - // needing a PostHog override. - const isFeatureEnabled = computed( - () => - settingsStore.settings.evaluation?.forceAgentWorthTesting === true || - posthogStore.getVariant(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name) === - INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.variant, - ); - - return { isFeatureEnabled }; -} diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts index 941b9c821fd..41af4322691 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts @@ -5720,19 +5720,29 @@ describe('AgentBuilderView — evals focus request', { timeout: 60_000 }, () => return () => wrapper.find('[data-testid="agent-evals-tab-content"]').exists(); } - it('selects the evals tab for a request raised before it mounted', async () => { + function savedGenerationCalls() { + return generateDraftCasesMock.mock.calls.filter(([, , , options]) => options?.save !== false); + } + + it('selects the evals tab and prepares unsaved examples for a request raised before it mounted', async () => { const wrapper = await seedFocusRequest('a1', false); - await settle(evalsTabShown(wrapper)); + await settle(() => evalsTabShown(wrapper)() && generateDraftCasesMock.mock.calls.length > 0); expect(evalsTabShown(wrapper)()).toBe(true); - expect(generateDraftCasesMock).not.toHaveBeenCalled(); + expect(generateDraftCasesMock).toHaveBeenCalledWith(expect.anything(), 'p1', 'a1', { + count: 10, + save: false, + }); + expect(savedGenerationCalls()).toEqual([]); }); - it('starts generation when the request asks for it', async () => { + it('starts one saved generation when the request asks for it', async () => { await seedFocusRequest('a1', true); - await settle(() => generateDraftCasesMock.mock.calls.length > 0); + await settle(() => savedGenerationCalls().length > 0); - expect(generateDraftCasesMock).toHaveBeenCalledOnce(); + // The empty-state preview can also request unsaved examples. Those must + // not be mistaken for a second request to persist a dataset. + expect(savedGenerationCalls()).toEqual([[expect.anything(), 'p1', 'a1', {}]]); }); it('ignores a request naming a different agent', async () => { diff --git a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts index 17c9b959601..2c14bd9765a 100644 --- a/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts +++ b/packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts @@ -76,8 +76,8 @@ vi.mock('../components/AgentEvalsEmptyStatePreview.vue', () => ({ // bare identifier, which only auto-unwraps for genuine refs. A plain object // would read as itself (always truthy) instead of its `.value`. const isFeatureEnabled = ref(false); -vi.mock('@/experiments/testAgentPreview/useTestAgentPreviewExperiment', () => ({ - useTestAgentPreviewExperiment: () => ({ isFeatureEnabled }), +vi.mock('@/features/agents/composables/useTestAgentPreviewFlag', () => ({ + useTestAgentPreviewFlag: () => ({ isFeatureEnabled }), })); const PROJECT_ID = 'project-1'; @@ -295,7 +295,7 @@ describe('AgentEvalsSection', () => { expect(store.startRun).toHaveBeenCalledWith(PROJECT_ID, AGENT_ID, 'd1'); }); - describe("the experiment's empty-state preview", () => { + describe('the eval rollout empty-state preview', () => { beforeEach(() => { isFeatureEnabled.value = true; }); diff --git a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalChecksPanel.vue b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalChecksPanel.vue index ff35d20f3af..b6f687c28c6 100644 --- a/packages/frontend/editor-ui/src/features/agents/components/AgentEvalChecksPanel.vue +++ b/packages/frontend/editor-ui/src/features/agents/components/AgentEvalChecksPanel.vue @@ -1,9 +1,9 @@