fix: Load agent evals by default and share the rollout flag (no-changelog) (#40915)

packages/@n8n/backend-common/src/modules/__tests__/module-registry.test.ts

@@ -57,11 +57,14 @@ describe('eligibleModules', () => {
 		expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('policy-infrastructure');
 	});
 
-	it.each(['instance-ai', 'agents'])('should include %s by default', (moduleName) => {
-		expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName);
-	});
+	it.each(['instance-ai', 'agents', 'agent-evals'])(
+		'should include %s by default',
+		(moduleName) => {
+			expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName);
+		},
+	);
 
-	it.each(['instance-ai', 'agents'])(
+	it.each(['instance-ai', 'agents', 'agent-evals'])(
 		'should allow opting out of the default %s module via env var',
 		(moduleName) => {
 			process.env.N8N_DISABLED_MODULES = moduleName;
@@ -104,6 +107,7 @@ describe('eligibleModules', () => {
 			'workflow-reviews',
 			'instance-ai',
 			'agents',
+			'agent-evals',
 			'scim',
 		]);
 	});
@@ -144,6 +148,7 @@ describe('eligibleModules', () => {
 			'workflow-reviews',
 			'instance-ai',
 			'agents',
+			'agent-evals',
 			'scim',
 			'type-availability-policies',
 		]);

packages/@n8n/backend-common/src/modules/module-registry.ts

@@ -83,6 +83,7 @@ export class ModuleRegistry {
 		'workflow-reviews',
 		'instance-ai',
 		'agents',
+		'agent-evals',
 		'scim',
 	];
 

packages/@n8n/config/src/configs/evaluation.config.ts

@@ -59,6 +59,7 @@ export class EvaluationConfig {
 	 *
 	 * Useful for local development and QA environments that need to exercise
 	 * the offer without building out a fully-equipped agent first.
+	 * The agent-evals feature gate still applies.
 	 */
 	@Env('N8N_FORCE_AGENT_WORTH_TESTING')
 	forceAgentWorthTesting: boolean = false;

packages/cli/src/modules/agent-evals/agent-evals.module.ts

@@ -7,7 +7,7 @@ import { Container } from '@n8n/di';
  * per-case results, and serves the REST surface the editor consumes.
  *
  * Main-process only — the reused tool-mock substrate can't cross a queue
- * boundary. Opt-in module, and dark until `101_agent_evals` is on for the user.
+ * boundary. Loaded by default; `101_agent_evals` gates access for each user.
  */
 @BackendModule({ name: 'agent-evals', instanceTypes: ['main'] })
 export class AgentEvalsModule implements ModuleInterface {

packages/frontend/editor-ui/src/app/constants/experiments.ts

@@ -87,11 +87,6 @@ export const WORKFLOW_CARD_MCP_TOGGLE_EXPERIMENT = createExperiment('086_workflo
 export const INSTANCE_AI_PROACTIVE_AGENT_EXPERIMENT = createExperiment(
 	'082_instance_ai_proactive_agent',
 );
-// Flag key assigned when the PostHog flag is created — follow the
-// `n8n:experiments` skill to create it and fill in the real key here.
-export const INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT = createExperiment(
-	'<flag-key>_test_agent_preview',
-);
 // Experiment cleanup: remove with instanceAiPromptSuggestionsV2.
 export const INSTANCE_AI_PROMPT_SUGGESTIONS_V2_EXPERIMENT = createExperiment(
 	'085_instance_ai_prompt_suggestions_v2',
@@ -217,6 +212,5 @@ export const EXPERIMENTS_TO_TRACK = [
 	CREDENTIAL_DESCRIPTIONS_EXPERIMENT.name,
 	SURFACE_ASSISTANT_ON_WORKFLOW_ERROR_EXPERIMENT.name,
 	AGENTS_LIST_EMPTY_STATE_TEMPLATES_EXPERIMENT.name,
-	INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name,
 	AGENTS_N8N_CHAT_EXPERIMENT.name,
 ];

packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.test.ts

@@ -1,54 +0,0 @@
-import { describe, expect, it, vi, beforeEach } from 'vitest';
-
-import {
-	EXPERIMENTS_TO_TRACK,
-	INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT,
-} from '@/app/constants/experiments';
-
-import { useTestAgentPreviewExperiment } from './useTestAgentPreviewExperiment';
-
-const getVariant = vi.fn();
-const settingsState = { forceAgentWorthTesting: false };
-
-vi.mock('@/app/stores/posthog.store', () => ({
-	usePostHog: vi.fn(() => ({
-		getVariant,
-	})),
-}));
-
-vi.mock('@n8n/stores/settings.store', () => ({
-	useSettingsStore: () => ({
-		settings: { evaluation: { forceAgentWorthTesting: settingsState.forceAgentWorthTesting } },
-	}),
-}));
-
-describe('useTestAgentPreviewExperiment', () => {
-	beforeEach(() => {
-		getVariant.mockReset();
-		settingsState.forceAgentWorthTesting = false;
-	});
-
-	it.each([
-		{ variant: INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.variant, enabled: true },
-		{ variant: INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.control, enabled: false },
-		{ variant: undefined, enabled: false },
-	])('returns $enabled when PostHog variant is $variant', ({ variant, enabled }) => {
-		getVariant.mockReturnValue(variant);
-
-		const { isFeatureEnabled } = useTestAgentPreviewExperiment();
-
-		expect(isFeatureEnabled.value).toBe(enabled);
-		expect(getVariant).toHaveBeenCalledWith(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name);
-	});
-
-	it('is enabled via the forceAgentWorthTesting override even when PostHog is off', () => {
-		getVariant.mockReturnValue(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.control);
-		settingsState.forceAgentWorthTesting = true;
-
-		expect(useTestAgentPreviewExperiment().isFeatureEnabled.value).toBe(true);
-	});
-
-	it('registers the experiment for centralized enrollment tracking', () => {
-		expect(EXPERIMENTS_TO_TRACK).toContain(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name);
-	});
-});

packages/frontend/editor-ui/src/experiments/testAgentPreview/useTestAgentPreviewExperiment.ts

@@ -1,22 +0,0 @@
-import { computed } from 'vue';
-
-import { INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT } from '@/app/constants/experiments';
-import { usePostHog } from '@/app/stores/posthog.store';
-import { useSettingsStore } from '@n8n/stores/settings.store';
-
-export function useTestAgentPreviewExperiment() {
-	const posthogStore = usePostHog();
-	const settingsStore = useSettingsStore();
-
-	// `N8N_FORCE_AGENT_WORTH_TESTING` also forces this variant on, so a local/QA
-	// environment gets the full preview flow from one flag instead of also
-	// needing a PostHog override.
-	const isFeatureEnabled = computed(
-		() =>
-			settingsStore.settings.evaluation?.forceAgentWorthTesting === true ||
-			posthogStore.getVariant(INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name) ===
-				INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.variant,
-	);
-
-	return { isFeatureEnabled };
-}

packages/frontend/editor-ui/src/features/agents/__tests__/AgentBuilderView.test.ts

@@ -5720,19 +5720,29 @@ describe('AgentBuilderView — evals focus request', { timeout: 60_000 }, () =>
 		return () => wrapper.find('[data-testid="agent-evals-tab-content"]').exists();
 	}
 
-	it('selects the evals tab for a request raised before it mounted', async () => {
+	function savedGenerationCalls() {
+		return generateDraftCasesMock.mock.calls.filter(([, , , options]) => options?.save !== false);
+	}
+
+	it('selects the evals tab and prepares unsaved examples for a request raised before it mounted', async () => {
 		const wrapper = await seedFocusRequest('a1', false);
-		await settle(evalsTabShown(wrapper));
+		await settle(() => evalsTabShown(wrapper)() && generateDraftCasesMock.mock.calls.length > 0);
 
 		expect(evalsTabShown(wrapper)()).toBe(true);
-		expect(generateDraftCasesMock).not.toHaveBeenCalled();
+		expect(generateDraftCasesMock).toHaveBeenCalledWith(expect.anything(), 'p1', 'a1', {
+			count: 10,
+			save: false,
+		});
+		expect(savedGenerationCalls()).toEqual([]);
 	});
 
-	it('starts generation when the request asks for it', async () => {
+	it('starts one saved generation when the request asks for it', async () => {
 		await seedFocusRequest('a1', true);
-		await settle(() => generateDraftCasesMock.mock.calls.length > 0);
+		await settle(() => savedGenerationCalls().length > 0);
 
-		expect(generateDraftCasesMock).toHaveBeenCalledOnce();
+		// The empty-state preview can also request unsaved examples. Those must
+		// not be mistaken for a second request to persist a dataset.
+		expect(savedGenerationCalls()).toEqual([[expect.anything(), 'p1', 'a1', {}]]);
 	});
 
 	it('ignores a request naming a different agent', async () => {

packages/frontend/editor-ui/src/features/agents/__tests__/AgentEvalsSection.test.ts

@@ -76,8 +76,8 @@ vi.mock('../components/AgentEvalsEmptyStatePreview.vue', () => ({
 // bare identifier, which only auto-unwraps for genuine refs. A plain object
 // would read as itself (always truthy) instead of its `.value`.
 const isFeatureEnabled = ref(false);
-vi.mock('@/experiments/testAgentPreview/useTestAgentPreviewExperiment', () => ({
-	useTestAgentPreviewExperiment: () => ({ isFeatureEnabled }),
+vi.mock('@/features/agents/composables/useTestAgentPreviewFlag', () => ({
+	useTestAgentPreviewFlag: () => ({ isFeatureEnabled }),
 }));
 
 const PROJECT_ID = 'project-1';
@@ -295,7 +295,7 @@ describe('AgentEvalsSection', () => {
 		expect(store.startRun).toHaveBeenCalledWith(PROJECT_ID, AGENT_ID, 'd1');
 	});
 
-	describe("the experiment's empty-state preview", () => {
+	describe('the eval rollout empty-state preview', () => {
 		beforeEach(() => {
 			isFeatureEnabled.value = true;
 		});

packages/frontend/editor-ui/src/features/agents/components/AgentEvalChecksPanel.vue

@@ -1,9 +1,9 @@
 <script setup lang="ts">
 /**
- * The checks-have-run view behind `useTestAgentPreviewExperiment`: status
+ * The checks-have-run view behind `useTestAgentPreviewFlag`: status
  * filter pills plus "Add a check" and "Run all checks" buttons up top, then each case as an
  * `AgentEvalTryRow` — replaces `AgentEvalResultsPanel`'s vote/comment review
- * model with the simpler row the rest of the experiment already uses.
+ * model with the simpler row the rest of the preview flow already uses.
  *
  * Rows have no scenario tag here — a run's result only ever carries
  * `input`/`output`/`status` (the Data Table has no column for it), so this

packages/frontend/editor-ui/src/features/agents/components/AgentEvalsEmptyStatePreview.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 /**
  * Alternate empty state for the evals tab, shown instead of the plain
- * "Generate test cases" card when `useTestAgentPreviewExperiment` is on.
+ * "Generate test cases" card when `useTestAgentPreviewFlag` is on.
  */
 import { ref } from 'vue';
 import type { AgentEvalDraftCase } from '@n8n/api-types';

packages/frontend/editor-ui/src/features/agents/components/AgentEvalsSection.vue

@@ -12,7 +12,7 @@ import { N8nButton, N8nCallout, N8nIcon, N8nLoading, N8nText } from '@n8n/design
 import { useToast } from '@n8n/composables/useToast';
 import { useI18n } from '@n8n/i18n';
 
-import { useTestAgentPreviewExperiment } from '@/experiments/testAgentPreview/useTestAgentPreviewExperiment';
+import { useTestAgentPreviewFlag } from '@/features/agents/composables/useTestAgentPreviewFlag';
 import { useAgentEvalsStore } from '../agentEvals.store';
 import { isDataTableDataset, resolveCaseColumns } from '../utils/agentEvalCases.utils';
 import AgentEvalCasesCard from './AgentEvalCasesCard.vue';
@@ -39,7 +39,7 @@ const emit = defineEmits<{
 const i18n = useI18n();
 const toast = useToast();
 const store = useAgentEvalsStore();
-const { isFeatureEnabled: showEmptyStatePreview } = useTestAgentPreviewExperiment();
+const { isFeatureEnabled: showEmptyStatePreview } = useTestAgentPreviewFlag();
 
 const datasets = computed(() => store.getDatasets(props.agentId));
 // Datasets come back newest-first, and generation makes exactly one; a picker is

packages/frontend/editor-ui/src/features/agents/composables/useTestAgentPreviewFlag.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+
+import { EXPERIMENTS_TO_TRACK } from '@/app/constants/experiments';
+import { AGENT_EVALS_FLAG } from '@n8n/api-types';
+
+import { useTestAgentPreviewFlag } from './useTestAgentPreviewFlag';
+
+const posthogState = { value: undefined as boolean | string | undefined };
+const isFeatureEnabled = vi.fn(() => posthogState.value === true);
+const settingsState = { forceAgentWorthTesting: false, agentEvalsEnabled: false };
+
+vi.mock('@/app/stores/posthog.store', () => ({
+	usePostHog: () => ({ isFeatureEnabled }),
+}));
+
+vi.mock('@n8n/stores/settings.store', () => ({
+	useSettingsStore: () => ({
+		settings: { evaluation: settingsState },
+	}),
+}));
+
+describe('useTestAgentPreviewFlag', () => {
+	beforeEach(() => {
+		vi.clearAllMocks();
+		posthogState.value = undefined;
+		settingsState.forceAgentWorthTesting = false;
+		settingsState.agentEvalsEnabled = false;
+	});
+
+	it.each([
+		{ flag: true, enabled: true },
+		{ flag: false, enabled: false },
+		{ flag: undefined, enabled: false },
+		{ flag: 'control', enabled: false },
+		{ flag: 'variant', enabled: false },
+	])('returns $enabled when the boolean rollout flag is $flag', ({ flag, enabled }) => {
+		posthogState.value = flag;
+
+		expect(useTestAgentPreviewFlag().isFeatureEnabled.value).toBe(enabled);
+		expect(isFeatureEnabled).toHaveBeenCalledWith(AGENT_EVALS_FLAG);
+	});
+
+	it.each([false, undefined, 'control', 'variant'])(
+		'does not enable the preview with the QA override when the main flag is %s',
+		(flag) => {
+			posthogState.value = flag;
+			settingsState.forceAgentWorthTesting = true;
+
+			expect(useTestAgentPreviewFlag().isFeatureEnabled.value).toBe(false);
+		},
+	);
+
+	it('keeps the preview available for an enabled eval cohort when eligibility is forced', () => {
+		posthogState.value = true;
+		settingsState.forceAgentWorthTesting = true;
+
+		expect(useTestAgentPreviewFlag().isFeatureEnabled.value).toBe(true);
+	});
+
+	it('uses the backend eval override when PostHog is unavailable', () => {
+		settingsState.agentEvalsEnabled = true;
+
+		expect(useTestAgentPreviewFlag().isFeatureEnabled.value).toBe(true);
+	});
+
+	it('does not register the boolean flag for experiment enrollment tracking', () => {
+		expect(EXPERIMENTS_TO_TRACK).not.toContain(AGENT_EVALS_FLAG);
+	});
+});

packages/frontend/editor-ui/src/features/agents/composables/useTestAgentPreviewFlag.ts

@@ -0,0 +1,12 @@
+import { computed } from 'vue';
+
+import { useAgentEvalsFlag } from '@/features/ai/evaluation.ee/composables/useAgentEvalsFlag';
+
+export function useTestAgentPreviewFlag() {
+	const isAgentEvalsEnabled = useAgentEvalsFlag();
+
+	// The force setting only bypasses agent eligibility, not this rollout gate.
+	const isFeatureEnabled = computed(() => isAgentEvalsEnabled.value);
+
+	return { isFeatureEnabled };
+}

packages/frontend/editor-ui/src/features/ai/instanceAi/InstanceAiThreadView.vue

@@ -66,7 +66,7 @@ import { useIsAgentWorking } from './composables/useIsAgentWorking';
 import { useAgentReturnContextStore } from '@/features/agents/agentReturnContext.store';
 import { useRecentWorkflowsStore } from '@/app/stores/recentWorkflows.store';
 import { useIsAssistantAtMentionsEnabled } from '@/features/ai/assistant-at-mentions/composables/useIsAssistantAtMentionsEnabled';
-import { useTestAgentPreviewExperiment } from '@/experiments/testAgentPreview/useTestAgentPreviewExperiment';
+import { useTestAgentPreviewFlag } from '@/features/agents/composables/useTestAgentPreviewFlag';
 
 const props = defineProps<{ threadId: string }>();
 
@@ -214,7 +214,7 @@ watch(
 	{ immediate: true },
 );
 
-const { isFeatureEnabled: isTestAgentPreviewVariant } = useTestAgentPreviewExperiment();
+const { isFeatureEnabled: isTestAgentPreviewEnabled } = useTestAgentPreviewFlag();
 
 // The builder's own "Testing agent" step (the `call_agent` tool) already runs
 // a representative message against the draft agent — reused here so the
@@ -987,14 +987,14 @@ function handleNewThreadClick() {
 						</Transition>
 						<Transition name="confirmation-slide">
 							<InstanceAiTestAgentPanel
-								v-if="activeTestAgentOffer && !isTestAgentPreviewVariant"
+								v-if="activeTestAgentOffer && !isTestAgentPreviewEnabled"
 								@generate="handleGenerateTestCasesFromOffer"
 								@dismiss="dismissTestAgentOffer"
 							/>
 						</Transition>
 						<Transition name="confirmation-slide">
 							<InstanceAiTestAgentPreviewPanel
-								v-if="latchedTestAgentOffer && isTestAgentPreviewVariant"
+								v-if="latchedTestAgentOffer && isTestAgentPreviewEnabled"
 								:key="latchedTestAgentOffer.agentId"
 								:target="latchedTestAgentOffer"
 								:initial-case="latestCallAgentResult"

packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/InstanceAiThreadView.test.ts

@@ -20,10 +20,7 @@ import InstanceAiThreadView from '../InstanceAiThreadView.vue';
 import { useInstanceAiStore, type ThreadRuntime } from '../instanceAi.store';
 import { usePushConnectionStore } from '@/app/stores/pushConnection.store';
 import { usePostHog } from '@/app/stores/posthog.store';
-import {
-	INSTANCE_AI_SETUP_PANEL_EXPERIMENT,
-	INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT,
-} from '@/app/constants/experiments';
+import { INSTANCE_AI_SETUP_PANEL_EXPERIMENT } from '@/app/constants/experiments';
 import { useSettingsStore } from '@n8n/stores/settings.store';
 import { INSTANCE_AI_VIEW, NEW_CONVERSATION_TITLE } from '../constants';
 import { LOCAL_STORAGE_INSTANCE_AI_CHAT_PANEL_WIDTH_RATIO } from '@/app/constants';
@@ -2487,12 +2484,7 @@ describe('InstanceAiThreadView', () => {
 			reasoning: 'It answered the question.',
 		};
 
-		function seedPreviewVariant(verdict: AgentEvalVerdict = PASS_VERDICT) {
-			mockedStore(usePostHog).getVariant.mockImplementation((flag) =>
-				flag === INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.name
-					? INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT.variant
-					: undefined,
-			);
+		function seedPreviewRun(verdict: AgentEvalVerdict = PASS_VERDICT) {
 			const evalsStore = mockedStore(useAgentEvalsStore);
 			evalsStore.previewRun.mockResolvedValueOnce({
 				status: 'completed',
@@ -2505,9 +2497,9 @@ describe('InstanceAiThreadView', () => {
 			return evalsStore;
 		}
 
-		it('renders the preview panel instead of the generic offer when the experiment is on', async () => {
+		it('renders the preview panel instead of the generic offer when the main eval flag is on', async () => {
 			seedReadyAgent();
-			seedPreviewVariant();
+			seedPreviewRun();
 
 			const { queryByTestId, findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
@@ -2519,7 +2511,7 @@ describe('InstanceAiThreadView', () => {
 		// cannot be reported as a first check. The panel runs and judges its own.
 		it("does not reuse the builder's own test call, since it has no rule or verdict", async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			thread.messages.push({
 				id: 'msg-builder',
 				role: 'assistant',
@@ -2570,7 +2562,7 @@ describe('InstanceAiThreadView', () => {
 
 		it('persists the dismissal on "Later" without requesting the evals focus', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			const user = userEvent.setup();
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
@@ -2584,7 +2576,7 @@ describe('InstanceAiThreadView', () => {
 
 		it('persists the dismissal on "Looks good" without requesting the evals focus yet', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			evalsStore.generateDraftCases.mockResolvedValueOnce({
 				datasetId: 'dataset-2',
 				dataTableId: 'table-2',
@@ -2603,7 +2595,7 @@ describe('InstanceAiThreadView', () => {
 
 		it('drops the preview panel if the dataset cache populates from elsewhere before the user commits', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 
 			// `isLoaded`/`getDatasets` are automocked vi.fn()s, so a plain
 			// `mockReturnValue` change is invisible to Vue's reactivity system —
@@ -2638,17 +2630,21 @@ describe('InstanceAiThreadView', () => {
 
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			const panel = await findByTestId('instance-ai-test-agent-panel');
-			expect(panel).toHaveTextContent('Test your agent');
+			expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 		});
 
-		it('stays hidden while the flag is off', () => {
+		it.each([false, true])('stays hidden while the main flag is off, with force=%s', (force) => {
 			seedReadyAgent();
 			testAgentOfferState.evalsFlagEnabled = false;
+			useSettingsStore().settings.evaluation = {
+				...useSettingsStore().settings.evaluation,
+				forceAgentWorthTesting: force,
+			};
 
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('stays hidden until the builder has persisted an agent', () => {
@@ -2658,6 +2654,7 @@ describe('InstanceAiThreadView', () => {
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('stays hidden while the agent is still being built', () => {
@@ -2667,6 +2664,7 @@ describe('InstanceAiThreadView', () => {
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('stays hidden once the agent already has test cases', () => {
@@ -2678,6 +2676,7 @@ describe('InstanceAiThreadView', () => {
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('still suggests when the dataset list has loaded and is empty', async () => {
@@ -2688,7 +2687,7 @@ describe('InstanceAiThreadView', () => {
 
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			expect(await findByTestId('instance-ai-test-agent-panel')).toBeInTheDocument();
+			expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 		});
 
 		it('stays hidden for an agent with no tools or skills', () => {
@@ -2702,6 +2701,7 @@ describe('InstanceAiThreadView', () => {
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('offers the suggestion when forced, even for an agent with no tools or skills', async () => {
@@ -2718,13 +2718,12 @@ describe('InstanceAiThreadView', () => {
 
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			// forceAgentWorthTesting also forces the preview experiment on — one
-			// flag gets a local/QA environment the full preview flow without a
-			// separate PostHog override.
+			// The main eval flag enables the preview. The force setting bypasses
+			// the capability check for this agent with no tools or skills.
 			expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 		});
 
-		it('offers the preview variant when forced, with no PostHog override at all', async () => {
+		it('offers the preview when forced, with no PostHog override at all', async () => {
 			seedReadyAgent();
 			const evalsStore = mockedStore(useAgentEvalsStore);
 			evalsStore.previewRun.mockResolvedValueOnce({
@@ -2756,6 +2755,7 @@ describe('InstanceAiThreadView', () => {
 			const { queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
 			expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument();
+			expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument();
 		});
 
 		it('still suggests when a different agent was dismissed', async () => {
@@ -2767,21 +2767,19 @@ describe('InstanceAiThreadView', () => {
 
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			expect(await findByTestId('instance-ai-test-agent-panel')).toBeInTheDocument();
+			expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 		});
 
-		it('requests the evals surface with generation and dismisses on the CTA', async () => {
+		it('runs the initial preview without automatically starting saved checks', async () => {
 			seedReadyAgent();
-			const user = userEvent.setup();
-			const evalsStore = mockedStore(useAgentEvalsStore);
+			const evalsStore = seedPreviewRun();
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			await user.click(await findByTestId('instance-ai-test-agent-generate'));
+			await findByTestId('instance-ai-test-agent-preview-example');
 
-			expect(evalsStore.requestEvalsFocus).toHaveBeenCalledWith('agent-1', true);
-			expect(store.updateThreadMetadata).toHaveBeenCalledWith('thread-1', {
-				dismissedContextKeys: ['test-agent:agent-1'],
-			});
+			expect(evalsStore.previewRun).toHaveBeenCalledTimes(1);
+			expect(evalsStore.startRun).not.toHaveBeenCalled();
+			expect(evalsStore.requestEvalsFocus).not.toHaveBeenCalled();
 		});
 
 		it('still suggests when a finished run left a confirmation unresolved', async () => {
@@ -2792,7 +2790,7 @@ describe('InstanceAiThreadView', () => {
 
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			expect(await findByTestId('instance-ai-test-agent-panel')).toBeInTheDocument();
+			expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 		});
 
 		it('drops an unclaimed focus request when the thread is left', () => {
@@ -2816,13 +2814,13 @@ describe('InstanceAiThreadView', () => {
 			expect(evalsStore.clearEvalsFocus).not.toHaveBeenCalled();
 		});
 
-		it('persists the dismissal without requesting generation on "Maybe later"', async () => {
+		it('persists the dismissal without requesting generation on "Later"', async () => {
 			seedReadyAgent();
 			const user = userEvent.setup();
-			const evalsStore = mockedStore(useAgentEvalsStore);
+			const evalsStore = seedPreviewRun();
 			const { findByTestId } = renderView({ props: { threadId: 'thread-1' } });
 
-			await user.click(await findByTestId('instance-ai-test-agent-dismiss'));
+			await user.click(await findByTestId('instance-ai-test-agent-preview-later'));
 
 			expect(evalsStore.requestEvalsFocus).not.toHaveBeenCalled();
 			expect(store.updateThreadMetadata).toHaveBeenCalledWith('thread-1', {
@@ -2913,7 +2911,7 @@ describe('InstanceAiThreadView', () => {
 
 		it('clears the latch and opens the evals surface from "Open evals", without requesting generation', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			const user = userEvent.setup();
 			const { findByTestId, queryByTestId } = renderView({
 				props: { threadId: 'thread-1' },
@@ -2959,7 +2957,7 @@ describe('InstanceAiThreadView', () => {
 		});
 		it('opens the evals surface on one case without clearing the latch', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			const user = userEvent.setup();
 			const { findByTestId, queryByTestId } = renderView({
 				props: { threadId: 'thread-1' },
@@ -3022,7 +3020,7 @@ describe('InstanceAiThreadView', () => {
 
 			it('drops the card when the agent starts working again before the user commits', async () => {
 				seedReadyAgent();
-				seedPreviewVariant();
+				seedPreviewRun();
 				const { findByTestId, queryByTestId } = renderWithStub();
 				await findByTestId('preview-stub');
 
@@ -3033,7 +3031,7 @@ describe('InstanceAiThreadView', () => {
 
 			it('keeps the card once the user has committed, even though the live offer is gone', async () => {
 				seedReadyAgent();
-				seedPreviewVariant();
+				seedPreviewRun();
 				const user = userEvent.setup();
 				const { findByTestId, queryByTestId } = renderWithStub();
 				await user.click(await findByTestId('preview-stub-confirm'));
@@ -3046,7 +3044,7 @@ describe('InstanceAiThreadView', () => {
 
 			it('retargets to another agent built in the same thread, instead of staying on the first', async () => {
 				seedReadyAgent();
-				seedPreviewVariant();
+				seedPreviewRun();
 				const metadata = ref<Record<string, unknown>>({
 					instanceAiAgentBuilderTarget: AGENT_TARGET,
 				});
@@ -3067,22 +3065,22 @@ describe('InstanceAiThreadView', () => {
 				);
 			});
 
-			it('shows the generic offer only while the live offer exists', async () => {
+			it('shows the unclaimed preview only while the live offer exists', async () => {
 				seedReadyAgent();
 				const { findByTestId, queryByTestId } = renderView({ props: { threadId: 'thread-1' } });
-				expect(await findByTestId('instance-ai-test-agent-panel')).toBeInTheDocument();
+				expect(await findByTestId('instance-ai-test-agent-preview-panel')).toBeInTheDocument();
 
 				thread.isStreaming = true;
 
 				await vi.waitFor(() =>
-					expect(queryByTestId('instance-ai-test-agent-panel')).not.toBeInTheDocument(),
+					expect(queryByTestId('instance-ai-test-agent-preview-panel')).not.toBeInTheDocument(),
 				);
 			});
 		});
 
 		it('opens the offered agent’s chat preview from "Try agent yourself", keeping the panel and not touching evals', async () => {
 			seedReadyAgent();
-			const evalsStore = seedPreviewVariant();
+			const evalsStore = seedPreviewRun();
 			const user = userEvent.setup();
 			const { findByTestId, queryByTestId } = renderView({
 				props: { threadId: 'thread-1' },

packages/frontend/editor-ui/src/features/ai/instanceAi/components/InstanceAiTestAgentPreviewPanel.vue

@@ -6,7 +6,7 @@
  * and what the judge found. The case comes from the builder's own test run
  * only when that run already has a rule and a verdict (`initialCase`);
  * otherwise the panel drafts, runs and judges one case of its own. Behind the
- * `INSTANCE_AI_TEST_AGENT_PREVIEW_EXPERIMENT` flag, alongside the original
+ * `101_agent_evals` flag, alongside the original
  * `InstanceAiTestAgentPanel`.
  */
 import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue';