Merge branch 'trust-689-allow-running-checks-to-expand-in-the-ui' into trust-690-allow-adding-checks-and-applying-suggestions-during-active

This commit is contained in:
Luka Zivkovic 2026-10-16 13:15:52 +02:00
commit c50df00edd
9 changed files with 249 additions and 64 deletions

View File

@ -29,6 +29,7 @@ import {
import { generateAgentScenarioSeed } from '../agent-scenario-seed';
import { createMcpMockFetch } from '../mcp-mock-fetch';
import { createLlmMockHandler } from '../mock-handler';
import { createWebSearchMock } from '../web-search-mock';
// The service resolves the agents-module surface lazily; top-level vi.mock
// keeps those specifiers from loading the real module graph (and avoids the
@ -54,6 +55,7 @@ vi.mock('@/utils/ai-proxy-fetch', () => ({ createAiProxyFetch: vi.fn(() => vi.fn
vi.mock('../agent-scenario-seed', () => ({ generateAgentScenarioSeed: vi.fn() }));
vi.mock('../mcp-mock-fetch', () => ({ createMcpMockFetch: vi.fn(() => vi.fn()) }));
vi.mock('../mock-handler', () => ({ createLlmMockHandler: vi.fn() }));
vi.mock('../web-search-mock', () => ({ createWebSearchMock: vi.fn(() => vi.fn()) }));
const logger = mock<Logger>();
const settingsService = mock<AgentsSettingsService>();
@ -290,6 +292,113 @@ describe('EvalAgentExecutionService.executeWithLlmMock', () => {
},
);
it('records search progress only for the matching active call and ignores late endings', async () => {
const handlers = new Map<AgentEvent, AgentEventHandler>();
const onProgress = vi.fn().mockResolvedValue(undefined);
const search = { query: 'shipping' };
const searchResult = { query: 'shipping', results: [] };
reconstructFromAgentEntity.mockResolvedValue({
agent: {
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
close: vi.fn(),
generate: vi.fn(async () => {
for (const toolName of ['Slack_Tool', 'web_search']) {
handlers.get(AgentEvent.ToolExecutionStart)?.({
type: AgentEvent.ToolExecutionStart,
toolName,
toolCallId: toolName,
args: search,
});
}
const { onSearch } = vi.mocked(createWebSearchMock).mock.calls[0][0];
onSearch(search, searchResult);
handlers.get(AgentEvent.ToolExecutionEnd)?.({
type: AgentEvent.ToolExecutionEnd,
toolName: 'web_search',
toolCallId: 'web_search',
result: searchResult,
isError: false,
});
// A delegated search has no active parent call to update.
onSearch({ query: 'returns' }, { query: 'returns', results: [] });
// A runtime can report completion without a prior start event.
handlers.get(AgentEvent.ToolExecutionEnd)?.({
type: AgentEvent.ToolExecutionEnd,
toolName: 'lookup',
toolCallId: 'orphan',
result: 'found',
isError: false,
});
return makeGenerateResult({
toolCalls: [{ tool: 'lookup', input: {}, output: 'found' }],
});
}),
},
});
const result = await buildService().executeWithLlmMock('agent-1', user, request, 'Search', {
onProgress,
});
expect(onProgress.mock.calls[2][0]).toEqual([
expect.objectContaining({ tool: 'Slack_Tool', mocked: false, interceptedRequests: [] }),
expect.objectContaining({
tool: 'web_search',
status: 'running',
mocked: true,
interceptedRequests: [expect.objectContaining({ requestBody: search })],
}),
]);
expect(onProgress).toHaveBeenCalledTimes(6);
expect(
result.toolCalls.find((call) => call.tool === 'web_search')?.interceptedRequests,
).toHaveLength(2);
expect(result.toolCalls.find((call) => call.toolCallId === 'orphan')).toMatchObject({
kind: 'other',
output: 'found',
});
handlers.get(AgentEvent.ToolExecutionEnd)?.({
type: AgentEvent.ToolExecutionEnd,
toolName: 'lookup',
toolCallId: 'late',
result: 'ignored',
isError: false,
});
await Promise.resolve();
expect(onProgress).toHaveBeenCalledTimes(6);
});
it('classifies delegated MCP records from their intercepted request metadata', async () => {
reconstructFromAgentEntity.mockResolvedValue({
agent: {
close: vi.fn(),
generate: vi.fn(async () => {
vi.mocked(createMcpMockFetch).mock.calls[0][0].onToolCall({
serverName: 'child',
toolName: 'lookup',
args: {},
result: { text: 'found', isError: false },
});
const instrumentation = reconstructFromAgentEntity.mock.calls[0][5] as {
onMcpToolCallSettled: (event: {
serverName: string;
toolName: string;
success: boolean;
}) => void;
};
instrumentation.onMcpToolCallSettled({
serverName: 'child',
toolName: 'lookup',
success: true,
});
return makeGenerateResult({ toolCalls: [] });
}),
},
});
const result = await buildService().executeWithLlmMock('agent-1', user, request);
expect(result.toolCalls).toEqual([
expect.objectContaining({ tool: 'child_lookup', kind: 'mcp', mocked: true }),
]);
});
it('reports tool progress before generation finishes and flushes writes before returning', async () => {
const handlers = new Map<AgentEvent, AgentEventHandler>();
const onProgress = vi.fn().mockResolvedValue(undefined);

View File

@ -296,7 +296,9 @@ export class EvalAgentExecutionService {
});
const toolLedger: ToolLedger = new Map();
let onLedgerChange: ((toolName: string) => void) | undefined;
let onLedgerChange:
| ((toolName: string, entries: InstanceAiEvalInterceptedRequest[]) => void)
| undefined;
const credentialHelpers: EvalMockedCredentialsHelper[] = [];
const recorder = createAgentModelTurnRecorder(
createAiProxyFetch(this.outboundHttp),
@ -355,7 +357,7 @@ export class EvalAgentExecutionService {
requestBody: call.args,
mockResponse: call.result,
});
onLedgerChange?.(key);
onLedgerChange?.(key, entries);
};
// Fallback web_search mock — always created (sub-agents may enable web
@ -379,7 +381,7 @@ export class EvalAgentExecutionService {
requestBody: args,
mockResponse: result,
});
onLedgerChange?.('web_search');
onLedgerChange?.('web_search', entries);
},
});
@ -434,7 +436,7 @@ export class EvalAgentExecutionService {
mockHandler,
toolContext.toolName,
toolLedger,
() => onLedgerChange?.(toolContext.toolName),
(entries) => onLedgerChange?.(toolContext.toolName, entries),
);
},
},
@ -451,24 +453,24 @@ export class EvalAgentExecutionService {
const progressCalls = new Map<string, InstanceAiEvalAgentToolCallRecord>();
let progressWrites = Promise.resolve();
let progressClosed = false;
const publishProgress = () => {
if (progressClosed || !hooks?.onProgress) return;
const onProgress = hooks?.onProgress;
const publishProgress = (
writeProgress: (calls: InstanceAiEvalAgentToolCallRecord[]) => Promise<void>,
) => {
const calls = [...progressCalls.values()].map((call) => ({ ...call }));
progressWrites = progressWrites
.then(async () => await hooks.onProgress?.(calls))
.then(async () => await writeProgress(calls))
.catch((error: unknown) => {
this.logger.warn('[EvalAgentMock] Could not record tool progress', { error });
});
};
if (hooks?.onProgress) {
if (onProgress) {
// A send-and-wait tool records its mocked send before the tool ends.
// Notify synchronously; the existing queue owns persistence and ordering.
onLedgerChange = (toolName) => {
if (progressClosed) return;
onLedgerChange = (toolName, interceptedRequests) => {
let changed = false;
for (const [id, call] of progressCalls) {
if (call.tool !== toolName || call.status !== 'running') continue;
const interceptedRequests = toolLedger.get(toolName) ?? [];
progressCalls.set(id, {
...call,
mocked: interceptedRequests.length > 0,
@ -476,7 +478,7 @@ export class EvalAgentExecutionService {
});
changed = true;
}
if (changed) publishProgress();
if (changed) publishProgress(onProgress);
};
agent.on(AgentEvent.ToolExecutionStart, (event) => {
if (progressClosed || event.type !== AgentEvent.ToolExecutionStart) return;
@ -489,7 +491,7 @@ export class EvalAgentExecutionService {
mocked: false,
interceptedRequests: [],
});
publishProgress();
publishProgress(onProgress);
});
agent.on(AgentEvent.ToolExecutionEnd, (event) => {
if (progressClosed || event.type !== AgentEvent.ToolExecutionEnd) return;
@ -506,7 +508,7 @@ export class EvalAgentExecutionService {
mocked: interceptedRequests.length > 0,
interceptedRequests: [...interceptedRequests],
});
publishProgress();
publishProgress(onProgress);
});
}
@ -637,7 +639,6 @@ export class EvalAgentExecutionService {
skippedFeatures,
{
modelTurns: recorder.turns,
toolLedger,
credentialHelpers,
toolCalls: terminalToolCalls(),
},
@ -645,7 +646,6 @@ export class EvalAgentExecutionService {
}
return this.errorResult(`Agent run failed: ${message}`, seed, skippedFeatures, {
modelTurns: recorder.turns,
toolLedger,
credentialHelpers,
toolCalls: terminalToolCalls(),
});
@ -655,7 +655,7 @@ export class EvalAgentExecutionService {
if (call.status === 'running')
progressCalls.set(id, { ...call, status: 'error', error: 'execution_stopped' });
}
if (progressCalls.size > 0) publishProgress();
if (onProgress && progressCalls.size > 0) publishProgress(onProgress);
progressClosed = true;
await progressWrites;
await recorder.flush();
@ -706,7 +706,7 @@ export class EvalAgentExecutionService {
mockHandler: EvalLlmMockHandler,
toolName: string,
ledger: ToolLedger,
onRecorded?: () => void,
onRecorded?: (entries: InstanceAiEvalInterceptedRequest[]) => void,
): EvalLlmMockHandler {
return async (requestOptions, node) => {
const response = await mockHandler(requestOptions, node);
@ -722,7 +722,7 @@ export class EvalAgentExecutionService {
requestBody: requestOptions.body,
mockResponse: snapshotLedgerBody(response?.body),
});
onRecorded?.();
onRecorded?.(entries);
this.logger.debug(
`[EvalAgentMock] Intercepted ${requestOptions.method ?? 'GET'} ${requestOptions.url} from tool "${toolName}" (${node.type})`,
);
@ -736,9 +736,8 @@ export class EvalAgentExecutionService {
skippedFeatures: InstanceAiEvalAgentSkippedFeature[] = [],
partial?: {
modelTurns: InstanceAiEvalAgentExecutionResult['modelTurns'];
toolLedger: ToolLedger;
credentialHelpers: EvalMockedCredentialsHelper[];
toolCalls?: InstanceAiEvalAgentToolCallRecord[];
toolCalls: InstanceAiEvalAgentToolCallRecord[];
},
): InstanceAiEvalAgentExecutionResult {
this.logger.error(`[EvalAgentMock] ${message}`);
@ -747,16 +746,7 @@ export class EvalAgentExecutionService {
success: false,
errors: [message],
finalText: '',
toolCalls:
partial?.toolCalls ??
(partial
? [...partial.toolLedger.entries()].map(([tool, interceptedRequests]) => ({
tool,
kind: salvagedToolKind(interceptedRequests),
mocked: interceptedRequests.length > 0,
interceptedRequests,
}))
: []),
toolCalls: partial?.toolCalls ?? [],
modelTurns: partial?.modelTurns ?? [],
seed: seed ?? { openingMessage: '', globalContext: '', toolHints: {}, warnings: [] },
skippedFeatures,
@ -831,22 +821,6 @@ export function pruneConfigForEval(original: AgentJsonConfig): {
* tools, `sanitizeToolName` for workflow tools, the stored descriptor name
* for custom tools.
*/
/**
* Best-effort kind for a salvaged ledger entry on the failure path, where the
* summary map is gone: MCP and web-search entries are self-describing via
* nodeType; anything else with a nodeType came through the node/workflow
* engine seam.
*/
function salvagedToolKind(
interceptedRequests: InstanceAiEvalInterceptedRequest[],
): InstanceAiEvalAgentToolCallRecord['kind'] {
const nodeType = interceptedRequests.find((request) => request.nodeType)?.nodeType;
if (!nodeType) return 'other';
if (nodeType.startsWith('mcp:')) return 'mcp';
if (nodeType === 'web-search:fallback') return 'other';
return 'node';
}
export function summarizeTools(
config: AgentJsonConfig,
customTools: NonNullable<AgentEntity['tools']>,

View File

@ -30,6 +30,17 @@ describe('AgentEvalTryRow', () => {
});
describe('view="complete"', () => {
it('announces both disclosure states on the complete row', async () => {
const user = userEvent.setup();
const { getByTestId } = renderComponent({ props: { testId: 'row' } });
const toggle = getByTestId('row-toggle');
expect(toggle).toHaveAttribute('aria-expanded', 'false');
await user.click(toggle);
expect(toggle).toHaveAttribute('aria-expanded', 'true');
await user.click(toggle);
expect(toggle).toHaveAttribute('aria-expanded', 'false');
});
it.each([
['pass', 'Passes'],
['strong', 'Passes'],

View File

@ -1722,6 +1722,79 @@ describe('useAgentEvalsStore', () => {
});
describe('rerunResult', () => {
it('ignores a timer task already queued when a rerun finishes', async () => {
vi.useFakeTimers();
mockRun({ results: [result('c1')], count: 1, ratings: [] });
const store = useAgentEvalsStore();
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
let finish!: (value: AgentEvalResultRecord) => void;
rerunResult.mockImplementationOnce(
async () =>
await new Promise((resolve) => {
finish = resolve;
}),
);
const timer = vi.spyOn(globalThis, 'setTimeout');
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
const queued = timer.mock.calls.find(([, delay]) => delay === 5000)?.[0];
expect(typeof queued).toBe('function');
finish(result('c1'));
await pending;
getRunDetail.mockClear();
if (typeof queued !== 'function') throw new Error('Missing progress timer');
await queued();
expect(getRunDetail).not.toHaveBeenCalled();
expect(store.getReview(RUN_ID).results[0]).toEqual(result('c1'));
expect(vi.getTimerCount()).toBe(0);
timer.mockRestore();
});
it.each([false, true])('handles an uncached rerun (request fails: %s)', async (fails) => {
const store = useAgentEvalsStore();
if (fails) rerunResult.mockRejectedValueOnce(new Error('Unavailable'));
else rerunResult.mockResolvedValueOnce(result('uncached'));
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'uncached');
if (fails) await expect(pending).rejects.toThrow('Unavailable');
else expect(await pending).toEqual(result('uncached'));
expect(getRunDetail).not.toHaveBeenCalled();
});
it('preserves a newer attempt when a later refresh returns an older runAt', async () => {
const newer = { ...result('c1'), runAt: '2026-01-01T01:00:00.000Z' };
mockRun({ results: [newer], count: 1, ratings: [] });
const store = useAgentEvalsStore();
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
getRunDetail.mockResolvedValueOnce(runDetail([result('c1')], 1));
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
expect(store.getReview(RUN_ID).results[0]).toEqual(newer);
});
it('pauses normal progress reads while the document is hidden', async () => {
vi.useFakeTimers();
const visibility = vi.spyOn(document, 'hidden', 'get').mockReturnValue(true);
mockRun({ results: [result('c1')], count: 1, ratings: [] });
const store = useAgentEvalsStore();
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
let finish!: (value: AgentEvalResultRecord) => void;
rerunResult.mockImplementationOnce(
async () =>
await new Promise((resolve) => {
finish = resolve;
}),
);
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
getRunDetail.mockClear();
await vi.advanceTimersByTimeAsync(5000);
expect(getRunDetail).not.toHaveBeenCalled();
visibility.mockReturnValue(false);
await vi.advanceTimersByTimeAsync(5000);
expect(getRunDetail).toHaveBeenCalledOnce();
finish(result('c1'));
await pending;
expect(vi.getTimerCount()).toBe(0);
visibility.mockRestore();
});
it('patches the cached result to running before the request lands, then to the real response', async () => {
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
const store = useAgentEvalsStore();

View File

@ -1182,14 +1182,13 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
const ownsProgress = () => rerunningResults.get(result.id) === pending;
revisionByResult.set(result.id, ++resultRevision);
let timer: ReturnType<typeof setTimeout> | undefined;
let stopped = false;
let errors = 0;
const deadline = Date.now() + RUN_POLL_TIMEOUT_MS;
const confirmationDeadline = Date.now() + RESULT_CONFIRMATION_TIMEOUT_MS;
const schedule = () => {
if (stopped || !ownsProgress()) return;
if (!ownsProgress()) return;
timer = setTimeout(async () => {
if (stopped || !ownsProgress()) return;
if (!ownsProgress()) return;
const current = findCachedResult(result.id)?.result;
const confirmationExpired =
recovering && current?.runAt === result.runAt && Date.now() >= confirmationDeadline;
@ -1217,8 +1216,6 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
}, RUN_POLL_INTERVAL_MS);
};
const stop = (exhausted = false) => {
if (stopped) return;
stopped = true;
clearTimeout(timer);
if (!ownsProgress()) return;
const current = findCachedResult(result.id)?.result;

View File

@ -240,7 +240,7 @@ watch(
type="button"
:class="$style.expandToggle"
:data-test-id="testId && `${testId}-toggle`"
:aria-expanded="view === 'complete' ? expanded : undefined"
:aria-expanded="expanded"
:aria-label="i18n.baseText('instanceAi.testAgentPreview.conversation')"
>
<N8nIcon :icon="expanded ? 'chevron-up' : 'chevron-down'" size="small" />

View File

@ -130,3 +130,9 @@ it('filters malformed intercepted requests without discarding the tool call', ()
});
expect(call).toMatchObject({ mocked: false, interceptedRequests: [] });
});
it('renders a recorded error status without requiring a diagnostic string', () => {
const [call] = toDisplayToolCalls({ calls: [{ tool: 'lookup', status: 'error' }] });
expect(call.state).toBe(TOOL_CALL_STATE.ERROR);
expect(call.output).toBeUndefined();
});

View File

@ -249,6 +249,24 @@ describe('InstanceAiTestAgentPreviewPanel', () => {
expect(container.querySelector('a[href="https://example.com"]')).toBeInTheDocument();
});
it('omits the seed example when the mocked preview returns no answer', async () => {
const store = useAgentEvalsStore();
mockPreviewRun(store, { response: '' });
const generateDraftCases = vi.spyOn(store, 'generateDraftCases').mockResolvedValue({
cases: [{ input: 'Another request', whatToCheck: 'names the ticket', scenario: 'Vague' }],
});
const user = userEvent.setup();
const { findByTestId } = renderComponent();
await user.click(await findByTestId('instance-ai-test-agent-preview-check-harder'));
expect(await findByTestId('instance-ai-test-agent-examples-check-agent')).toBeInTheDocument();
expect(generateDraftCases).toHaveBeenCalledWith('project-1', 'agent-1', {
count: 10,
save: false,
});
});
it('renders the answer inside a fixed-height scrollable container regardless of length', async () => {
const store = useAgentEvalsStore();
mockPreviewRun(store, {

View File

@ -56,8 +56,8 @@ const phase = ref<Phase>('generating-preview');
// from `previewRun`'s response, which already ran the agent and returned the
// final text. No run id, no polling: unlike the suite below, this never
// touches a Data Table, dataset, or eval-run row at all.
const previewRequest = ref<string | null>(null);
const previewAnswer = ref<string | null>(null);
const previewInput = ref('');
const previewOutput = ref('');
// Generated with `save: false` — a preview, not yet persisted. Nothing backs
// these rows until "Check your agent" commits them, so dismissing or
// refreshing mid-preview leaves no orphaned dataset behind.
@ -87,9 +87,6 @@ const previewVerdict = ref<AgentEvalVerdict | null>(null);
// Whether the full example conversation is shown under "What we found".
const conversationExpanded = ref(false);
const previewInput = computed(() => previewRequest.value ?? '');
const previewOutput = computed(() => previewAnswer.value ?? '');
// Nothing failed when the judge passed the answer, or when the case had no rule to
// grade it against (`skipped`) — the card still shows, as a pass. A judge error is
// different: the answer was never graded, so it must not read as "passed".
@ -189,8 +186,8 @@ async function runGeneratedPreview() {
previewScenario.value = result.scenario || null;
previewWhatToCheck.value = result.whatToCheck;
previewVerdict.value = result.verdict;
previewRequest.value = result.input;
previewAnswer.value = result.response;
previewInput.value = result.input;
previewOutput.value = result.response;
conversationExpanded.value = false;
phase.value = 'awaiting-confirmation';
} catch (error) {
@ -443,8 +440,8 @@ async function onApplySuggestion() {
);
return;
}
previewRequest.value = preview.input;
previewAnswer.value = preview.response;
previewInput.value = preview.input;
previewOutput.value = preview.response;
previewVerdict.value = preview.verdict;
conversationExpanded.value = false;
}
@ -538,7 +535,7 @@ function onDontCreateEvals() {
<EvalInitialSample
v-if="conversationExpanded"
:preview-input="previewInput"
:preview-output="previewOutput ?? ''"
:preview-output="previewOutput"
hide-banner
/>
</div>
@ -610,7 +607,7 @@ function onDontCreateEvals() {
<template v-else-if="phase === 'suite-ready'">
<InstanceAiTestAgentExamplesPanel
:preview-input="previewInput"
:preview-output="previewOutput ?? ''"
:preview-output="previewOutput"
:preview-scenario="previewScenario"
:preview-status="previewStatus"
:preview-reasoning="previewReasoning"