diff --git a/packages/frontend/@n8n/i18n/src/locales/en.json b/packages/frontend/@n8n/i18n/src/locales/en.json index fc28e63a13a..0a65acb78ee 100644 --- a/packages/frontend/@n8n/i18n/src/locales/en.json +++ b/packages/frontend/@n8n/i18n/src/locales/en.json @@ -4896,12 +4896,18 @@ "scenarios.detail.expectationLabel": "Test type", "scenarios.detail.expectationHelper": "What this scenario is testing — we'll flag a mismatch when a run doesn't match.", "scenarios.detail.run": "Run scenario", + "scenarios.detail.rerun": "Run again", "scenarios.detail.notRunYet": "Not run yet", + "scenarios.result.degradedNote": "Intercepted request details are only shown for runs from the current session. Open the execution to see the full trace.", + "scenarios.result.loadingHistorical": "Loading execution…", "scenarios.expectation.pass": "Success path", "scenarios.expectation.fail": "Failure path", "scenarios.expectation.any": "Any", "scenarios.expectation.badge.pass": "Success", "scenarios.expectation.badge.fail": "Failure", + "scenarios.group.pass": "Success paths", + "scenarios.group.fail": "Failure paths", + "scenarios.group.any": "Any", "scenarios.displayStatus.unrun": "Not run yet", "scenarios.displayStatus.passed": "Passed", "scenarios.displayStatus.passedWithIssues": "Ran with issues", diff --git a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunResult.vue b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunResult.vue new file mode 100644 index 00000000000..a1735d5fdd3 --- /dev/null +++ b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunResult.vue @@ -0,0 +1,330 @@ + + + + + diff --git a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunnerDialog.vue b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunnerDialog.vue deleted file mode 100644 index 70c4769dfde..00000000000 --- a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenarioRunnerDialog.vue +++ /dev/null @@ -1,612 +0,0 @@ - - - - - - - diff --git a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenariosView.vue b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenariosView.vue index d6b3d76d460..6d086542e41 100644 --- a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenariosView.vue +++ b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/components/ScenariosView.vue @@ -6,6 +6,7 @@ import { N8nIconButton, N8nInput, N8nRadioButtons, + N8nSpinner, N8nText, N8nTooltip, type IconName, @@ -21,7 +22,7 @@ import type { ScenarioExpectation, ScenarioRecord, } from '../scenarios.store'; -import ScenarioRunnerDialog from './ScenarioRunnerDialog.vue'; +import ScenarioRunResult from './ScenarioRunResult.vue'; const props = defineProps<{ workflowId: string; @@ -40,9 +41,9 @@ const editingName = ref(false); // Draft for the inline "new scenario" form. Null when closed. const draft = ref<{ name: string; description: string } | null>(null); -// Dialog state — opens when user clicks Run on a scenario -const dialogOpen = ref(false); -const runningScenarioId = ref(null); +// Which scenario's result is currently held in runnerStore. Lets the detail +// pane render the rich trace only for the scenario that produced it. +const lastRunScenarioId = ref(null); const selectedScenario = computed(() => selectedId.value ? (scenarios.value.find((s) => s.id === selectedId.value) ?? null) : null, @@ -126,16 +127,28 @@ const expectationOptions = computed(() => [ }, ]); -function expectationBadge( - expectation: ScenarioExpectation, -): { label: string; tone: 'success' | 'danger' } | null { - if (expectation === 'pass') - return { label: i18n.baseText('scenarios.expectation.badge.pass'), tone: 'success' }; - if (expectation === 'fail') - return { label: i18n.baseText('scenarios.expectation.badge.fail'), tone: 'danger' }; - return null; +interface ScenarioGroup { + key: ScenarioExpectation; + labelKey: 'scenarios.group.pass' | 'scenarios.group.fail' | 'scenarios.group.any'; + items: ScenarioRecord[]; } +const groupedScenarios = computed(() => { + const groups: Record = { + pass: [], + fail: [], + any: [], + }; + for (const scenario of scenarios.value) { + groups[scenario.expectedOutcome].push(scenario); + } + return [ + { key: 'pass', labelKey: 'scenarios.group.pass', items: groups.pass }, + { key: 'fail', labelKey: 'scenarios.group.fail', items: groups.fail }, + { key: 'any', labelKey: 'scenarios.group.any', items: groups.any }, + ].filter((g) => g.items.length > 0) as ScenarioGroup[]; +}); + function commitExpectation(value: ScenarioExpectation) { if (!selectedScenario.value) return; if (value === selectedScenario.value.expectedOutcome) return; @@ -233,40 +246,68 @@ async function deleteSelected() { } } -function runSelected() { - if (!selectedScenario.value) return; - runnerStore.reset(); - runningScenarioId.value = selectedScenario.value.id; - // Seed the runner store's textarea with this scenario's description so the - // dialog opens with the right text. See ScenarioRunnerDialog's open watcher. - runnerStore.setScenarioText(selectedScenario.value.description); - dialogOpen.value = true; +async function runSelected() { + if (!selectedScenario.value || runnerStore.isRunning) return; + const scenario = selectedScenario.value; + lastRunScenarioId.value = scenario.id; + await runnerStore.runScenario(props.workflowId, scenario.description); + const result = runnerStore.result; + if (result) { + store.persistRunResult( + props.workflowId, + scenario.id, + result, + runnerStore.durationMs ?? undefined, + ); + } } -// When a run completes inside the dialog, persist the result on the scenario record. -watch( - () => runnerStore.status, - (status, prev) => { - if (!runningScenarioId.value) return; - if (prev === 'running' && (status === 'succeeded' || status === 'failed')) { - const result = runnerStore.result; - if (result) { - store.persistRunResult( - props.workflowId, - runningScenarioId.value, - result, - runnerStore.durationMs ?? undefined, - ); - } - } - }, +// True when runnerStore holds a live result for the currently-selected scenario. +// Drives the rich inline trace. For any other scenario, the detail pane shows +// only the summary (rich trace for historical runs is the lazy-fetch task). +const hasLiveResult = computed( + () => + runnerStore.result !== null && + selectedScenario.value !== null && + lastRunScenarioId.value === selectedScenario.value.id, ); -watch(dialogOpen, (open) => { - if (!open) { - runningScenarioId.value = null; - } +const isRunningSelected = computed( + () => + runnerStore.isRunning && + selectedScenario.value !== null && + lastRunScenarioId.value === selectedScenario.value.id, +); + +// Lazy-fetch the historical execution when selecting a scenario that hasn't +// been run this session. Rich trace comes back degraded (no intercepted +// requests, no hints) but node outputs + timings make it useful. +watch( + selectedScenario, + async (scenario) => { + if (!scenario?.lastExecutionId) return; + if (hasLiveResult.value) return; + if (store.cachedHistoricalResult(scenario.lastExecutionId)) return; + try { + await store.fetchHistoricalResult(scenario.lastExecutionId); + } catch { + // Silent — the detail pane falls back to summary-only display. + } + }, + { immediate: true }, +); + +const historicalResult = computed(() => { + if (hasLiveResult.value) return null; + if (!selectedScenario.value?.lastExecutionId) return null; + return store.cachedHistoricalResult(selectedScenario.value.lastExecutionId); }); + +const isFetchingHistorical = computed( + () => + !!selectedScenario.value?.lastExecutionId && + store.fetchingExecutionId === selectedScenario.value.lastExecutionId, +); @@ -558,11 +646,29 @@ watch(dialogOpen, (open) => { .list { display: flex; flex-direction: column; - gap: var(--spacing--3xs); + gap: var(--spacing--sm); overflow-y: auto; padding-right: var(--spacing--2xs); } +.listGroup { + display: flex; + flex-direction: column; + gap: var(--spacing--3xs); +} + +.listGroupHeader { + display: flex; + align-items: baseline; + gap: var(--spacing--2xs); + padding: var(--spacing--3xs) var(--spacing--xs); +} + +.listGroupLabel { + text-transform: uppercase; + letter-spacing: 0.05em; +} + .draftCard { display: flex; flex-direction: column; @@ -633,29 +739,6 @@ watch(dialogOpen, (open) => { min-width: 0; } -.listBadge { - flex-shrink: 0; - font-size: var(--font-size--3xs); - font-weight: var(--font-weight--bold); - line-height: 1; - padding: var(--spacing--5xs) var(--spacing--3xs); - border-radius: var(--radius--sm); - border: var(--border-width) var(--border-style) transparent; - letter-spacing: 0.02em; -} - -.listBadge_success { - color: var(--color--success--shade-1); - background-color: var(--color--success--tint-4); - border-color: var(--color--success--tint-2); -} - -.listBadge_danger { - color: var(--color--danger--shade-1); - background-color: var(--color--danger--tint-4); - border-color: var(--color--danger--tint-3); -} - .expectationOption { display: inline-flex; align-items: center; @@ -724,6 +807,16 @@ watch(dialogOpen, (open) => { flex-wrap: wrap; } +.runningBlock { + display: flex; + align-items: center; + gap: var(--spacing--2xs); + padding: var(--spacing--sm); + background-color: var(--color--background--light-2); + border-radius: var(--radius); + border: var(--border); +} + .lastRun { display: flex; align-items: baseline; diff --git a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/scenarios.store.ts b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/scenarios.store.ts index f7a838cfdfc..2fb4c4c96c7 100644 --- a/packages/frontend/editor-ui/src/features/ai/scenarioRunner/scenarios.store.ts +++ b/packages/frontend/editor-ui/src/features/ai/scenarioRunner/scenarios.store.ts @@ -1,9 +1,10 @@ -import type { InstanceAiEvalExecutionResult } from '@n8n/api-types'; +import type { InstanceAiEvalExecutionResult, InstanceAiEvalNodeResult } from '@n8n/api-types'; import { STORES } from '@n8n/stores'; import { useRootStore } from '@n8n/stores/useRootStore'; import { defineStore } from 'pinia'; import { computed, ref, watch } from 'vue'; +import { useWorkflowsStore } from '@/app/stores/workflows.store'; import * as scenarioRunnerApi from './scenarioRunner.api'; export type ScenarioRunStatus = 'passed' | 'passedWithIssues' | 'failed'; @@ -91,10 +92,18 @@ function deriveStatus(result: InstanceAiEvalExecutionResult): ScenarioRunStatus export const useScenariosStore = defineStore(STORES.SCENARIOS, () => { const rootStore = useRootStore(); + const workflowsStore = useWorkflowsStore(); const byWorkflowId = ref(loadFromStorage()); const runningScenarioId = ref(null); + // In-session cache of degraded results reconstructed from persisted + // executions. Keyed by executionId. Lost on reload — that's fine, the + // fetch is cheap and avoids duplicating data we already persisted as + // part of the normal execution record. + const historicalByExecutionId = ref>({}); + const fetchingExecutionId = ref(null); + watch( byWorkflowId, (value) => { @@ -216,9 +225,90 @@ export const useScenariosStore = defineStore(STORES.SCENARIOS, () => { } } + /** + * Build a degraded InstanceAiEvalExecutionResult from a persisted execution + * record. Node outputs and start times come from runData; intercepted + * requests and hints were never persisted so they come back empty (that's + * what the `degraded` flag in ScenarioRunResult acknowledges). + */ + function mapExecutionToResult( + executionId: string, + execution: { + data?: { + resultData?: { + runData?: Record; + error?: { message?: string }; + }; + }; + }, + ): InstanceAiEvalExecutionResult { + const runData = + (execution.data?.resultData?.runData as + | Record> + | undefined) ?? {}; + const nodeResults: Record = {}; + + for (const [nodeName, tasks] of Object.entries(runData)) { + const lastTask = tasks?.[tasks.length - 1]; + const mainOutput = lastTask?.data?.main?.[0] as Array<{ json?: unknown }> | undefined; + const items = Array.isArray(mainOutput) ? mainOutput : []; + nodeResults[nodeName] = { + output: items.slice(0, 10).map((i) => i?.json ?? null), + outputCount: items.length, + interceptedRequests: [], + executionMode: 'mocked', + startTime: lastTask?.startTime, + }; + } + + const errorMessage = execution.data?.resultData?.error?.message; + return { + executionId, + success: !errorMessage, + nodeResults, + errors: errorMessage ? [errorMessage] : [], + hints: { + globalContext: '', + triggerContent: {}, + nodeHints: {}, + warnings: [], + bypassPinData: {}, + }, + }; + } + + async function fetchHistoricalResult( + executionId: string, + ): Promise { + const cached = historicalByExecutionId.value[executionId]; + if (cached) return cached; + + fetchingExecutionId.value = executionId; + try { + const execution = await workflowsStore.getExecution(executionId); + if (!execution) return null; + const mapped = mapExecutionToResult(executionId, execution); + historicalByExecutionId.value = { + ...historicalByExecutionId.value, + [executionId]: mapped, + }; + return mapped; + } finally { + fetchingExecutionId.value = null; + } + } + + function cachedHistoricalResult( + executionId: string | undefined, + ): InstanceAiEvalExecutionResult | null { + if (!executionId) return null; + return historicalByExecutionId.value[executionId] ?? null; + } + return { byWorkflowId, runningScenarioId, + fetchingExecutionId, scenariosFor, hasScenariosFor, create, @@ -226,5 +316,7 @@ export const useScenariosStore = defineStore(STORES.SCENARIOS, () => { remove, run, persistRunResult, + fetchHistoricalResult, + cachedHistoricalResult, }; });