mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 13:39:44 +02:00
Merge branch 'trust-689-allow-running-checks-to-expand-in-the-ui' into trust-690-allow-adding-checks-and-applying-suggestions-during-active
This commit is contained in:
commit
c50df00edd
|
|
@ -29,6 +29,7 @@ import {
|
|||
import { generateAgentScenarioSeed } from '../agent-scenario-seed';
|
||||
import { createMcpMockFetch } from '../mcp-mock-fetch';
|
||||
import { createLlmMockHandler } from '../mock-handler';
|
||||
import { createWebSearchMock } from '../web-search-mock';
|
||||
|
||||
// The service resolves the agents-module surface lazily; top-level vi.mock
|
||||
// keeps those specifiers from loading the real module graph (and avoids the
|
||||
|
|
@ -54,6 +55,7 @@ vi.mock('@/utils/ai-proxy-fetch', () => ({ createAiProxyFetch: vi.fn(() => vi.fn
|
|||
vi.mock('../agent-scenario-seed', () => ({ generateAgentScenarioSeed: vi.fn() }));
|
||||
vi.mock('../mcp-mock-fetch', () => ({ createMcpMockFetch: vi.fn(() => vi.fn()) }));
|
||||
vi.mock('../mock-handler', () => ({ createLlmMockHandler: vi.fn() }));
|
||||
vi.mock('../web-search-mock', () => ({ createWebSearchMock: vi.fn(() => vi.fn()) }));
|
||||
|
||||
const logger = mock<Logger>();
|
||||
const settingsService = mock<AgentsSettingsService>();
|
||||
|
|
@ -290,6 +292,113 @@ describe('EvalAgentExecutionService.executeWithLlmMock', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it('records search progress only for the matching active call and ignores late endings', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
const search = { query: 'shipping' };
|
||||
const searchResult = { query: 'shipping', results: [] };
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
close: vi.fn(),
|
||||
generate: vi.fn(async () => {
|
||||
for (const toolName of ['Slack_Tool', 'web_search']) {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolName,
|
||||
toolCallId: toolName,
|
||||
args: search,
|
||||
});
|
||||
}
|
||||
const { onSearch } = vi.mocked(createWebSearchMock).mock.calls[0][0];
|
||||
onSearch(search, searchResult);
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'web_search',
|
||||
toolCallId: 'web_search',
|
||||
result: searchResult,
|
||||
isError: false,
|
||||
});
|
||||
// A delegated search has no active parent call to update.
|
||||
onSearch({ query: 'returns' }, { query: 'returns', results: [] });
|
||||
// A runtime can report completion without a prior start event.
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'lookup',
|
||||
toolCallId: 'orphan',
|
||||
result: 'found',
|
||||
isError: false,
|
||||
});
|
||||
return makeGenerateResult({
|
||||
toolCalls: [{ tool: 'lookup', input: {}, output: 'found' }],
|
||||
});
|
||||
}),
|
||||
},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock('agent-1', user, request, 'Search', {
|
||||
onProgress,
|
||||
});
|
||||
expect(onProgress.mock.calls[2][0]).toEqual([
|
||||
expect.objectContaining({ tool: 'Slack_Tool', mocked: false, interceptedRequests: [] }),
|
||||
expect.objectContaining({
|
||||
tool: 'web_search',
|
||||
status: 'running',
|
||||
mocked: true,
|
||||
interceptedRequests: [expect.objectContaining({ requestBody: search })],
|
||||
}),
|
||||
]);
|
||||
expect(onProgress).toHaveBeenCalledTimes(6);
|
||||
expect(
|
||||
result.toolCalls.find((call) => call.tool === 'web_search')?.interceptedRequests,
|
||||
).toHaveLength(2);
|
||||
expect(result.toolCalls.find((call) => call.toolCallId === 'orphan')).toMatchObject({
|
||||
kind: 'other',
|
||||
output: 'found',
|
||||
});
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'lookup',
|
||||
toolCallId: 'late',
|
||||
result: 'ignored',
|
||||
isError: false,
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(onProgress).toHaveBeenCalledTimes(6);
|
||||
});
|
||||
|
||||
it('classifies delegated MCP records from their intercepted request metadata', async () => {
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
close: vi.fn(),
|
||||
generate: vi.fn(async () => {
|
||||
vi.mocked(createMcpMockFetch).mock.calls[0][0].onToolCall({
|
||||
serverName: 'child',
|
||||
toolName: 'lookup',
|
||||
args: {},
|
||||
result: { text: 'found', isError: false },
|
||||
});
|
||||
const instrumentation = reconstructFromAgentEntity.mock.calls[0][5] as {
|
||||
onMcpToolCallSettled: (event: {
|
||||
serverName: string;
|
||||
toolName: string;
|
||||
success: boolean;
|
||||
}) => void;
|
||||
};
|
||||
instrumentation.onMcpToolCallSettled({
|
||||
serverName: 'child',
|
||||
toolName: 'lookup',
|
||||
success: true,
|
||||
});
|
||||
return makeGenerateResult({ toolCalls: [] });
|
||||
}),
|
||||
},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock('agent-1', user, request);
|
||||
expect(result.toolCalls).toEqual([
|
||||
expect.objectContaining({ tool: 'child_lookup', kind: 'mcp', mocked: true }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports tool progress before generation finishes and flushes writes before returning', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
|
|
|
|||
|
|
@ -296,7 +296,9 @@ export class EvalAgentExecutionService {
|
|||
});
|
||||
|
||||
const toolLedger: ToolLedger = new Map();
|
||||
let onLedgerChange: ((toolName: string) => void) | undefined;
|
||||
let onLedgerChange:
|
||||
| ((toolName: string, entries: InstanceAiEvalInterceptedRequest[]) => void)
|
||||
| undefined;
|
||||
const credentialHelpers: EvalMockedCredentialsHelper[] = [];
|
||||
const recorder = createAgentModelTurnRecorder(
|
||||
createAiProxyFetch(this.outboundHttp),
|
||||
|
|
@ -355,7 +357,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: call.args,
|
||||
mockResponse: call.result,
|
||||
});
|
||||
onLedgerChange?.(key);
|
||||
onLedgerChange?.(key, entries);
|
||||
};
|
||||
|
||||
// Fallback web_search mock — always created (sub-agents may enable web
|
||||
|
|
@ -379,7 +381,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: args,
|
||||
mockResponse: result,
|
||||
});
|
||||
onLedgerChange?.('web_search');
|
||||
onLedgerChange?.('web_search', entries);
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -434,7 +436,7 @@ export class EvalAgentExecutionService {
|
|||
mockHandler,
|
||||
toolContext.toolName,
|
||||
toolLedger,
|
||||
() => onLedgerChange?.(toolContext.toolName),
|
||||
(entries) => onLedgerChange?.(toolContext.toolName, entries),
|
||||
);
|
||||
},
|
||||
},
|
||||
|
|
@ -451,24 +453,24 @@ export class EvalAgentExecutionService {
|
|||
const progressCalls = new Map<string, InstanceAiEvalAgentToolCallRecord>();
|
||||
let progressWrites = Promise.resolve();
|
||||
let progressClosed = false;
|
||||
const publishProgress = () => {
|
||||
if (progressClosed || !hooks?.onProgress) return;
|
||||
const onProgress = hooks?.onProgress;
|
||||
const publishProgress = (
|
||||
writeProgress: (calls: InstanceAiEvalAgentToolCallRecord[]) => Promise<void>,
|
||||
) => {
|
||||
const calls = [...progressCalls.values()].map((call) => ({ ...call }));
|
||||
progressWrites = progressWrites
|
||||
.then(async () => await hooks.onProgress?.(calls))
|
||||
.then(async () => await writeProgress(calls))
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn('[EvalAgentMock] Could not record tool progress', { error });
|
||||
});
|
||||
};
|
||||
if (hooks?.onProgress) {
|
||||
if (onProgress) {
|
||||
// A send-and-wait tool records its mocked send before the tool ends.
|
||||
// Notify synchronously; the existing queue owns persistence and ordering.
|
||||
onLedgerChange = (toolName) => {
|
||||
if (progressClosed) return;
|
||||
onLedgerChange = (toolName, interceptedRequests) => {
|
||||
let changed = false;
|
||||
for (const [id, call] of progressCalls) {
|
||||
if (call.tool !== toolName || call.status !== 'running') continue;
|
||||
const interceptedRequests = toolLedger.get(toolName) ?? [];
|
||||
progressCalls.set(id, {
|
||||
...call,
|
||||
mocked: interceptedRequests.length > 0,
|
||||
|
|
@ -476,7 +478,7 @@ export class EvalAgentExecutionService {
|
|||
});
|
||||
changed = true;
|
||||
}
|
||||
if (changed) publishProgress();
|
||||
if (changed) publishProgress(onProgress);
|
||||
};
|
||||
agent.on(AgentEvent.ToolExecutionStart, (event) => {
|
||||
if (progressClosed || event.type !== AgentEvent.ToolExecutionStart) return;
|
||||
|
|
@ -489,7 +491,7 @@ export class EvalAgentExecutionService {
|
|||
mocked: false,
|
||||
interceptedRequests: [],
|
||||
});
|
||||
publishProgress();
|
||||
publishProgress(onProgress);
|
||||
});
|
||||
agent.on(AgentEvent.ToolExecutionEnd, (event) => {
|
||||
if (progressClosed || event.type !== AgentEvent.ToolExecutionEnd) return;
|
||||
|
|
@ -506,7 +508,7 @@ export class EvalAgentExecutionService {
|
|||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests: [...interceptedRequests],
|
||||
});
|
||||
publishProgress();
|
||||
publishProgress(onProgress);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -637,7 +639,6 @@ export class EvalAgentExecutionService {
|
|||
skippedFeatures,
|
||||
{
|
||||
modelTurns: recorder.turns,
|
||||
toolLedger,
|
||||
credentialHelpers,
|
||||
toolCalls: terminalToolCalls(),
|
||||
},
|
||||
|
|
@ -645,7 +646,6 @@ export class EvalAgentExecutionService {
|
|||
}
|
||||
return this.errorResult(`Agent run failed: ${message}`, seed, skippedFeatures, {
|
||||
modelTurns: recorder.turns,
|
||||
toolLedger,
|
||||
credentialHelpers,
|
||||
toolCalls: terminalToolCalls(),
|
||||
});
|
||||
|
|
@ -655,7 +655,7 @@ export class EvalAgentExecutionService {
|
|||
if (call.status === 'running')
|
||||
progressCalls.set(id, { ...call, status: 'error', error: 'execution_stopped' });
|
||||
}
|
||||
if (progressCalls.size > 0) publishProgress();
|
||||
if (onProgress && progressCalls.size > 0) publishProgress(onProgress);
|
||||
progressClosed = true;
|
||||
await progressWrites;
|
||||
await recorder.flush();
|
||||
|
|
@ -706,7 +706,7 @@ export class EvalAgentExecutionService {
|
|||
mockHandler: EvalLlmMockHandler,
|
||||
toolName: string,
|
||||
ledger: ToolLedger,
|
||||
onRecorded?: () => void,
|
||||
onRecorded?: (entries: InstanceAiEvalInterceptedRequest[]) => void,
|
||||
): EvalLlmMockHandler {
|
||||
return async (requestOptions, node) => {
|
||||
const response = await mockHandler(requestOptions, node);
|
||||
|
|
@ -722,7 +722,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: requestOptions.body,
|
||||
mockResponse: snapshotLedgerBody(response?.body),
|
||||
});
|
||||
onRecorded?.();
|
||||
onRecorded?.(entries);
|
||||
this.logger.debug(
|
||||
`[EvalAgentMock] Intercepted ${requestOptions.method ?? 'GET'} ${requestOptions.url} from tool "${toolName}" (${node.type})`,
|
||||
);
|
||||
|
|
@ -736,9 +736,8 @@ export class EvalAgentExecutionService {
|
|||
skippedFeatures: InstanceAiEvalAgentSkippedFeature[] = [],
|
||||
partial?: {
|
||||
modelTurns: InstanceAiEvalAgentExecutionResult['modelTurns'];
|
||||
toolLedger: ToolLedger;
|
||||
credentialHelpers: EvalMockedCredentialsHelper[];
|
||||
toolCalls?: InstanceAiEvalAgentToolCallRecord[];
|
||||
toolCalls: InstanceAiEvalAgentToolCallRecord[];
|
||||
},
|
||||
): InstanceAiEvalAgentExecutionResult {
|
||||
this.logger.error(`[EvalAgentMock] ${message}`);
|
||||
|
|
@ -747,16 +746,7 @@ export class EvalAgentExecutionService {
|
|||
success: false,
|
||||
errors: [message],
|
||||
finalText: '',
|
||||
toolCalls:
|
||||
partial?.toolCalls ??
|
||||
(partial
|
||||
? [...partial.toolLedger.entries()].map(([tool, interceptedRequests]) => ({
|
||||
tool,
|
||||
kind: salvagedToolKind(interceptedRequests),
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests,
|
||||
}))
|
||||
: []),
|
||||
toolCalls: partial?.toolCalls ?? [],
|
||||
modelTurns: partial?.modelTurns ?? [],
|
||||
seed: seed ?? { openingMessage: '', globalContext: '', toolHints: {}, warnings: [] },
|
||||
skippedFeatures,
|
||||
|
|
@ -831,22 +821,6 @@ export function pruneConfigForEval(original: AgentJsonConfig): {
|
|||
* tools, `sanitizeToolName` for workflow tools, the stored descriptor name
|
||||
* for custom tools.
|
||||
*/
|
||||
/**
|
||||
* Best-effort kind for a salvaged ledger entry on the failure path, where the
|
||||
* summary map is gone: MCP and web-search entries are self-describing via
|
||||
* nodeType; anything else with a nodeType came through the node/workflow
|
||||
* engine seam.
|
||||
*/
|
||||
function salvagedToolKind(
|
||||
interceptedRequests: InstanceAiEvalInterceptedRequest[],
|
||||
): InstanceAiEvalAgentToolCallRecord['kind'] {
|
||||
const nodeType = interceptedRequests.find((request) => request.nodeType)?.nodeType;
|
||||
if (!nodeType) return 'other';
|
||||
if (nodeType.startsWith('mcp:')) return 'mcp';
|
||||
if (nodeType === 'web-search:fallback') return 'other';
|
||||
return 'node';
|
||||
}
|
||||
|
||||
export function summarizeTools(
|
||||
config: AgentJsonConfig,
|
||||
customTools: NonNullable<AgentEntity['tools']>,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,17 @@ describe('AgentEvalTryRow', () => {
|
|||
});
|
||||
|
||||
describe('view="complete"', () => {
|
||||
it('announces both disclosure states on the complete row', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId } = renderComponent({ props: { testId: 'row' } });
|
||||
const toggle = getByTestId('row-toggle');
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'false');
|
||||
await user.click(toggle);
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'true');
|
||||
await user.click(toggle);
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'false');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['pass', 'Passes'],
|
||||
['strong', 'Passes'],
|
||||
|
|
|
|||
|
|
@ -1722,6 +1722,79 @@ describe('useAgentEvalsStore', () => {
|
|||
});
|
||||
|
||||
describe('rerunResult', () => {
|
||||
it('ignores a timer task already queued when a rerun finishes', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [result('c1')], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let finish!: (value: AgentEvalResultRecord) => void;
|
||||
rerunResult.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const timer = vi.spyOn(globalThis, 'setTimeout');
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
|
||||
const queued = timer.mock.calls.find(([, delay]) => delay === 5000)?.[0];
|
||||
expect(typeof queued).toBe('function');
|
||||
finish(result('c1'));
|
||||
await pending;
|
||||
getRunDetail.mockClear();
|
||||
if (typeof queued !== 'function') throw new Error('Missing progress timer');
|
||||
await queued();
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(result('c1'));
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
timer.mockRestore();
|
||||
});
|
||||
|
||||
it.each([false, true])('handles an uncached rerun (request fails: %s)', async (fails) => {
|
||||
const store = useAgentEvalsStore();
|
||||
if (fails) rerunResult.mockRejectedValueOnce(new Error('Unavailable'));
|
||||
else rerunResult.mockResolvedValueOnce(result('uncached'));
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'uncached');
|
||||
if (fails) await expect(pending).rejects.toThrow('Unavailable');
|
||||
else expect(await pending).toEqual(result('uncached'));
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves a newer attempt when a later refresh returns an older runAt', async () => {
|
||||
const newer = { ...result('c1'), runAt: '2026-01-01T01:00:00.000Z' };
|
||||
mockRun({ results: [newer], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
getRunDetail.mockResolvedValueOnce(runDetail([result('c1')], 1));
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(newer);
|
||||
});
|
||||
|
||||
it('pauses normal progress reads while the document is hidden', async () => {
|
||||
vi.useFakeTimers();
|
||||
const visibility = vi.spyOn(document, 'hidden', 'get').mockReturnValue(true);
|
||||
mockRun({ results: [result('c1')], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let finish!: (value: AgentEvalResultRecord) => void;
|
||||
rerunResult.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
|
||||
getRunDetail.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
visibility.mockReturnValue(false);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(getRunDetail).toHaveBeenCalledOnce();
|
||||
finish(result('c1'));
|
||||
await pending;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
visibility.mockRestore();
|
||||
});
|
||||
|
||||
it('patches the cached result to running before the request lands, then to the real response', async () => {
|
||||
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
|
|
|
|||
|
|
@ -1182,14 +1182,13 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
const ownsProgress = () => rerunningResults.get(result.id) === pending;
|
||||
revisionByResult.set(result.id, ++resultRevision);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let stopped = false;
|
||||
let errors = 0;
|
||||
const deadline = Date.now() + RUN_POLL_TIMEOUT_MS;
|
||||
const confirmationDeadline = Date.now() + RESULT_CONFIRMATION_TIMEOUT_MS;
|
||||
const schedule = () => {
|
||||
if (stopped || !ownsProgress()) return;
|
||||
if (!ownsProgress()) return;
|
||||
timer = setTimeout(async () => {
|
||||
if (stopped || !ownsProgress()) return;
|
||||
if (!ownsProgress()) return;
|
||||
const current = findCachedResult(result.id)?.result;
|
||||
const confirmationExpired =
|
||||
recovering && current?.runAt === result.runAt && Date.now() >= confirmationDeadline;
|
||||
|
|
@ -1217,8 +1216,6 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
}, RUN_POLL_INTERVAL_MS);
|
||||
};
|
||||
const stop = (exhausted = false) => {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
clearTimeout(timer);
|
||||
if (!ownsProgress()) return;
|
||||
const current = findCachedResult(result.id)?.result;
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ watch(
|
|||
type="button"
|
||||
:class="$style.expandToggle"
|
||||
:data-test-id="testId && `${testId}-toggle`"
|
||||
:aria-expanded="view === 'complete' ? expanded : undefined"
|
||||
:aria-expanded="expanded"
|
||||
:aria-label="i18n.baseText('instanceAi.testAgentPreview.conversation')"
|
||||
>
|
||||
<N8nIcon :icon="expanded ? 'chevron-up' : 'chevron-down'" size="small" />
|
||||
|
|
|
|||
|
|
@ -130,3 +130,9 @@ it('filters malformed intercepted requests without discarding the tool call', ()
|
|||
});
|
||||
expect(call).toMatchObject({ mocked: false, interceptedRequests: [] });
|
||||
});
|
||||
|
||||
it('renders a recorded error status without requiring a diagnostic string', () => {
|
||||
const [call] = toDisplayToolCalls({ calls: [{ tool: 'lookup', status: 'error' }] });
|
||||
expect(call.state).toBe(TOOL_CALL_STATE.ERROR);
|
||||
expect(call.output).toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -249,6 +249,24 @@ describe('InstanceAiTestAgentPreviewPanel', () => {
|
|||
expect(container.querySelector('a[href="https://example.com"]')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('omits the seed example when the mocked preview returns no answer', async () => {
|
||||
const store = useAgentEvalsStore();
|
||||
mockPreviewRun(store, { response: '' });
|
||||
const generateDraftCases = vi.spyOn(store, 'generateDraftCases').mockResolvedValue({
|
||||
cases: [{ input: 'Another request', whatToCheck: 'names the ticket', scenario: 'Vague' }],
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
const { findByTestId } = renderComponent();
|
||||
|
||||
await user.click(await findByTestId('instance-ai-test-agent-preview-check-harder'));
|
||||
|
||||
expect(await findByTestId('instance-ai-test-agent-examples-check-agent')).toBeInTheDocument();
|
||||
expect(generateDraftCases).toHaveBeenCalledWith('project-1', 'agent-1', {
|
||||
count: 10,
|
||||
save: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('renders the answer inside a fixed-height scrollable container regardless of length', async () => {
|
||||
const store = useAgentEvalsStore();
|
||||
mockPreviewRun(store, {
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ const phase = ref<Phase>('generating-preview');
|
|||
// from `previewRun`'s response, which already ran the agent and returned the
|
||||
// final text. No run id, no polling: unlike the suite below, this never
|
||||
// touches a Data Table, dataset, or eval-run row at all.
|
||||
const previewRequest = ref<string | null>(null);
|
||||
const previewAnswer = ref<string | null>(null);
|
||||
const previewInput = ref('');
|
||||
const previewOutput = ref('');
|
||||
// Generated with `save: false` — a preview, not yet persisted. Nothing backs
|
||||
// these rows until "Check your agent" commits them, so dismissing or
|
||||
// refreshing mid-preview leaves no orphaned dataset behind.
|
||||
|
|
@ -87,9 +87,6 @@ const previewVerdict = ref<AgentEvalVerdict | null>(null);
|
|||
// Whether the full example conversation is shown under "What we found".
|
||||
const conversationExpanded = ref(false);
|
||||
|
||||
const previewInput = computed(() => previewRequest.value ?? '');
|
||||
const previewOutput = computed(() => previewAnswer.value ?? '');
|
||||
|
||||
// Nothing failed when the judge passed the answer, or when the case had no rule to
|
||||
// grade it against (`skipped`) — the card still shows, as a pass. A judge error is
|
||||
// different: the answer was never graded, so it must not read as "passed".
|
||||
|
|
@ -189,8 +186,8 @@ async function runGeneratedPreview() {
|
|||
previewScenario.value = result.scenario || null;
|
||||
previewWhatToCheck.value = result.whatToCheck;
|
||||
previewVerdict.value = result.verdict;
|
||||
previewRequest.value = result.input;
|
||||
previewAnswer.value = result.response;
|
||||
previewInput.value = result.input;
|
||||
previewOutput.value = result.response;
|
||||
conversationExpanded.value = false;
|
||||
phase.value = 'awaiting-confirmation';
|
||||
} catch (error) {
|
||||
|
|
@ -443,8 +440,8 @@ async function onApplySuggestion() {
|
|||
);
|
||||
return;
|
||||
}
|
||||
previewRequest.value = preview.input;
|
||||
previewAnswer.value = preview.response;
|
||||
previewInput.value = preview.input;
|
||||
previewOutput.value = preview.response;
|
||||
previewVerdict.value = preview.verdict;
|
||||
conversationExpanded.value = false;
|
||||
}
|
||||
|
|
@ -538,7 +535,7 @@ function onDontCreateEvals() {
|
|||
<EvalInitialSample
|
||||
v-if="conversationExpanded"
|
||||
:preview-input="previewInput"
|
||||
:preview-output="previewOutput ?? ''"
|
||||
:preview-output="previewOutput"
|
||||
hide-banner
|
||||
/>
|
||||
</div>
|
||||
|
|
@ -610,7 +607,7 @@ function onDontCreateEvals() {
|
|||
<template v-else-if="phase === 'suite-ready'">
|
||||
<InstanceAiTestAgentExamplesPanel
|
||||
:preview-input="previewInput"
|
||||
:preview-output="previewOutput ?? ''"
|
||||
:preview-output="previewOutput"
|
||||
:preview-scenario="previewScenario"
|
||||
:preview-status="previewStatus"
|
||||
:preview-reasoning="previewReasoning"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user