mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
chore: Stack agent preview design changes on active-run fixes
This commit is contained in:
Luka Zivkovic
commit
d01e521241
|
|
@ -502,6 +502,7 @@ export {
|
|||
InstanceAiSendMessageRequest,
|
||||
InstanceAiEvalExecutionRequest,
|
||||
InstanceAiEvalAgentExecutionRequest,
|
||||
INSTANCE_AI_EVAL_AGENT_MAX_TIMEOUT_MS,
|
||||
InstanceAiEvalCredentialAllowlistRequest,
|
||||
InstanceAiEvalCredentialTestTargetRequest,
|
||||
INSTANCE_AI_MEMORY_TASK_WAIT_TIMEOUT_MS,
|
||||
|
|
|
|||
|
|
@ -2933,6 +2933,9 @@ export class InstanceAiEvalExecutionRequest extends Z.class({
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface InstanceAiEvalAgentToolCallRecord {
|
||||
/** Present on live progress records. Older completed records omit these fields. */
|
||||
toolCallId?: string;
|
||||
status?: 'running' | 'completed' | 'error';
|
||||
/** Sanitized tool name — matches what the model called. */
|
||||
tool: string;
|
||||
/** Where the tool executes. 'other' covers built-ins (skills, todos, environment). */
|
||||
|
|
@ -3010,6 +3013,8 @@ export interface InstanceAiEvalAgentExecutionResult {
|
|||
llmUsage?: InstanceAiEvalLlmUsage[];
|
||||
}
|
||||
|
||||
export const INSTANCE_AI_EVAL_AGENT_MAX_TIMEOUT_MS = 3_600_000;
|
||||
|
||||
export class InstanceAiEvalAgentExecutionRequest extends Z.class({
|
||||
/** Project the agent lives in (agent routes are project-scoped). */
|
||||
projectId: z.string().min(1),
|
||||
|
|
@ -3018,7 +3023,7 @@ export class InstanceAiEvalAgentExecutionRequest extends Z.class({
|
|||
* Overall run budget. Server default applies when omitted. Shares the workflow
|
||||
* variant's ceiling — the old 900_000 cap truncated a `complex` case's budget.
|
||||
*/
|
||||
timeoutMs: z.number().int().min(30_000).max(3_600_000).optional(),
|
||||
timeoutMs: z.number().int().min(30_000).max(INSTANCE_AI_EVAL_AGENT_MAX_TIMEOUT_MS).optional(),
|
||||
}) {}
|
||||
|
||||
export class InstanceAiEvalCredentialAllowlistRequest extends Z.class({
|
||||
|
|
|
|||
|
|
@ -1,12 +1,17 @@
|
|||
import { Container } from '@n8n/di';
|
||||
import { TransactionRunner } from '../../services/transaction';
|
||||
import type { Mock } from 'vitest';
|
||||
|
||||
import { AgentEvalRun } from '../../entities/agent-eval-run.ee';
|
||||
import { AgentEvalResult } from '../../entities/agent-eval-result.ee';
|
||||
import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
|
||||
import { AgentEvalResultRepository } from '../agent-eval-result.repository.ee';
|
||||
|
||||
describe('AgentEvalResultRepository', () => {
|
||||
const entityManager = mockEntityManager(AgentEvalResult);
|
||||
Container.set(TransactionRunner, {
|
||||
run: async (ctx, fn) => await fn(ctx),
|
||||
} satisfies TransactionRunner);
|
||||
const repo = Container.get(AgentEvalResultRepository);
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
@ -57,6 +62,18 @@ describe('AgentEvalResultRepository', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('updates progress only while the attempt is running', async () => {
|
||||
const calls = { calls: [{ tool: 'Slack_Tool', status: 'running' }] };
|
||||
const runAt = new Date();
|
||||
await repo.updateRunningToolCalls('res-1', runAt, calls);
|
||||
expect(entityManager.update.mock.calls[0]?.[1]).toEqual({
|
||||
id: 'res-1',
|
||||
status: 'running',
|
||||
runAt,
|
||||
});
|
||||
expect(entityManager.update.mock.calls[0]?.[2]).toEqual({ toolCalls: calls });
|
||||
});
|
||||
|
||||
describe('markAsCompleted', () => {
|
||||
it('stores the agent output and defaults toolCalls/metrics to null', async () => {
|
||||
entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
|
||||
|
|
@ -173,21 +190,70 @@ describe('AgentEvalResultRepository', () => {
|
|||
});
|
||||
|
||||
describe('claimForRerun', () => {
|
||||
it('locks the parent run before claiming a result', async () => {
|
||||
entityManager.update.mockResolvedValue({ affected: 1, generatedMaps: [], raw: [] });
|
||||
await repo.claimForRerun('res-1', 'run-1');
|
||||
expect(entityManager.update.mock.calls[0]).toEqual([
|
||||
AgentEvalRun,
|
||||
{ id: 'run-1' },
|
||||
{ id: 'run-1' },
|
||||
]);
|
||||
expect(entityManager.update.mock.calls[1]?.[1]).toMatchObject({
|
||||
id: 'res-1',
|
||||
runId: 'run-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('claims only a settled row, clearing the previous attempt', async () => {
|
||||
entityManager.update.mockResolvedValueOnce({ affected: 1, generatedMaps: [], raw: [] });
|
||||
entityManager.update.mockResolvedValue({ affected: 1, generatedMaps: [], raw: [] });
|
||||
|
||||
await expect(repo.claimForRerun('res-1')).resolves.toBe(true);
|
||||
await expect(repo.claimForRerun('res-1', 'run-1')).resolves.toBe(true);
|
||||
|
||||
const [, where, patch] = entityManager.update.mock.calls[0] ?? [];
|
||||
const [, where, patch] = entityManager.update.mock.calls[1] ?? [];
|
||||
expect(where).toMatchObject({ id: 'res-1' });
|
||||
expect(patch).toMatchObject({ status: 'running', output: null, errorCode: null });
|
||||
});
|
||||
|
||||
it('reports false when no row matched, because it was already running or claimed', async () => {
|
||||
entityManager.update.mockResolvedValueOnce({ affected: 0, generatedMaps: [], raw: [] });
|
||||
entityManager.update.mockResolvedValue({ affected: 0, generatedMaps: [], raw: [] });
|
||||
|
||||
await expect(repo.claimForRerun('res-1')).resolves.toBe(false);
|
||||
await expect(repo.claimForRerun('res-1', 'run-1')).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('does not claim a rerun when the driver omits the affected row count', async () => {
|
||||
entityManager.update.mockResolvedValue({ generatedMaps: [], raw: [] });
|
||||
|
||||
await expect(repo.claimForRerun('res-1', 'run-1')).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('markUnfinishedAsError', () => {
|
||||
it.each([undefined, new Date('2026-10-15T00:00:00Z')])(
|
||||
'repairs only an unstarted row or the original attempt at %s',
|
||||
async (runAt) => {
|
||||
const qb = {
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockReturnThis(),
|
||||
execute: vi.fn().mockResolvedValue({ affected: 1 }),
|
||||
};
|
||||
(entityManager.createQueryBuilder as Mock).mockReturnValue(qb);
|
||||
|
||||
await repo.markUnfinishedAsError('res-1', runAt, 'Original write failed');
|
||||
|
||||
expect(qb.set).toHaveBeenCalledWith({
|
||||
status: 'error',
|
||||
completedAt: expect.any(Date),
|
||||
errorCode: 'case_dispatch_failed',
|
||||
errorDetails: { message: 'Original write failed' },
|
||||
});
|
||||
type UnfinishedRow = { id: string; status: string; runAt?: Date };
|
||||
const expectedRows: UnfinishedRow[] = [{ id: 'res-1', status: 'new' }];
|
||||
if (runAt) expectedRows.push({ id: 'res-1', status: 'running', runAt });
|
||||
expect(qb.where).toHaveBeenCalledWith(expectedRows);
|
||||
expect(qb.execute).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('markAsCancelled', () => {
|
||||
|
|
|
|||
|
|
@ -1,18 +1,78 @@
|
|||
import { Container } from '@n8n/di';
|
||||
import { TransactionRunner } from '../../services/transaction';
|
||||
import type { Mock } from 'vitest';
|
||||
|
||||
import { AgentEvalRun } from '../../entities/agent-eval-run.ee';
|
||||
import { mockEntityManager } from '../../utils/test-utils/mock-entity-manager';
|
||||
import { AgentEvalRunRepository } from '../agent-eval-run.repository.ee';
|
||||
import type { AgentEvalResultStatusCounts } from '../agent-eval-result.repository.ee';
|
||||
|
||||
describe('AgentEvalRunRepository', () => {
|
||||
const entityManager = mockEntityManager(AgentEvalRun);
|
||||
Container.set(TransactionRunner, {
|
||||
run: async (ctx, fn) => await fn(ctx),
|
||||
} satisfies TransactionRunner);
|
||||
const repo = Container.get(AgentEvalRunRepository);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
describe.each([
|
||||
[
|
||||
'completed',
|
||||
async (counts?: AgentEvalResultStatusCounts) =>
|
||||
await repo.markAsCompleted('run-1', {}, counts),
|
||||
],
|
||||
[
|
||||
'error',
|
||||
async (counts?: AgentEvalResultStatusCounts) =>
|
||||
await repo.markAsError('run-1', 'failed', null, null, counts),
|
||||
],
|
||||
[
|
||||
'cancelled',
|
||||
async (counts?: AgentEvalResultStatusCounts) =>
|
||||
await repo.markAsCancelled('run-1', null, counts),
|
||||
],
|
||||
] as const)('%s settlement', (_status, settle) => {
|
||||
it.each([
|
||||
{ affected: 1, expected: true },
|
||||
{ affected: 0, expected: false },
|
||||
{ affected: undefined, expected: false },
|
||||
])(
|
||||
'returns $expected for an update affecting $affected rows',
|
||||
async ({ affected, expected }) => {
|
||||
entityManager.update.mockResolvedValueOnce({ affected, generatedMaps: [], raw: [] });
|
||||
|
||||
expect(await settle()).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ affected: 1, expected: true },
|
||||
{ affected: 0, expected: false },
|
||||
{ affected: undefined, expected: false },
|
||||
])(
|
||||
'returns $expected for a count-guarded update affecting $affected rows',
|
||||
async ({ affected, expected }) => {
|
||||
const query = {
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
select: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockReturnThis(),
|
||||
andWhere: vi.fn().mockReturnThis(),
|
||||
getQuery: vi.fn().mockReturnValue('SELECT COUNT(*)'),
|
||||
execute: vi.fn().mockResolvedValue({ affected, generatedMaps: [], raw: [] }),
|
||||
};
|
||||
(entityManager.createQueryBuilder as Mock).mockReturnValue(query);
|
||||
|
||||
expect(await settle({ new: 0, running: 0, success: 1, error: 0, cancelled: 0 })).toBe(
|
||||
expected,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('createRun', () => {
|
||||
it('starts a run as "new" with cancellation cleared', async () => {
|
||||
(entityManager.create as Mock).mockImplementation(
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import { Service } from '@n8n/di';
|
||||
import { DataSource, In, Repository } from '@n8n/typeorm';
|
||||
import { DataSource, In } from '@n8n/typeorm';
|
||||
import type { IDataObject, JsonObject } from 'n8n-workflow';
|
||||
|
||||
import { AgentEvalResult } from '../entities';
|
||||
import { AgentEvalResult, AgentEvalRun } from '../entities';
|
||||
import { TransactionRunner } from '../services/transaction';
|
||||
import { BaseRepository } from './base-repository';
|
||||
import type { AgentEvalResultStatus } from '../entities/agent-eval-result.ee';
|
||||
|
||||
type CreateAgentEvalResultAttrs = {
|
||||
|
|
@ -18,9 +20,9 @@ export type AgentEvalResultStatusCounts = Record<AgentEvalResultStatus, number>;
|
|||
// parameter limit (SQLite in particular).
|
||||
const SEED_CHUNK_SIZE = 100;
|
||||
|
||||
const startedAttempt = () => ({
|
||||
const startedAttempt = (runAt = new Date()) => ({
|
||||
status: 'running' as const,
|
||||
runAt: new Date(),
|
||||
runAt,
|
||||
completedAt: null,
|
||||
output: null,
|
||||
toolCalls: null,
|
||||
|
|
@ -31,9 +33,9 @@ const startedAttempt = () => ({
|
|||
});
|
||||
|
||||
@Service()
|
||||
export class AgentEvalResultRepository extends Repository<AgentEvalResult> {
|
||||
constructor(dataSource: DataSource) {
|
||||
super(AgentEvalResult, dataSource.manager);
|
||||
export class AgentEvalResultRepository extends BaseRepository<AgentEvalResult> {
|
||||
constructor(dataSource: DataSource, transactionRunner: TransactionRunner) {
|
||||
super(AgentEvalResult, dataSource.manager, transactionRunner);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -83,8 +85,15 @@ export class AgentEvalResultRepository extends Repository<AgentEvalResult> {
|
|||
|
||||
/** Starts an attempt. Clears whatever a previous attempt left behind, so a
|
||||
* rerun can't settle with an old error next to a new answer (or the reverse). */
|
||||
async markAsRunning(id: string) {
|
||||
return await this.update(id, startedAttempt());
|
||||
async markAsRunning(id: string, runAt = new Date()) {
|
||||
const attempt = startedAttempt(runAt);
|
||||
await this.update(id, attempt);
|
||||
return attempt.runAt;
|
||||
}
|
||||
|
||||
/** Progress belongs to one running attempt, identified by its start timestamp. */
|
||||
async updateRunningToolCalls(id: string, runAt: Date, toolCalls: JsonObject): Promise<void> {
|
||||
await this.update({ id, status: 'running', runAt }, { toolCalls });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -92,12 +101,32 @@ export class AgentEvalResultRepository extends Repository<AgentEvalResult> {
|
|||
* not settled — already running, or claimed by a concurrent request — so only
|
||||
* one caller ever executes a given rerun.
|
||||
*/
|
||||
async claimForRerun(id: string): Promise<boolean> {
|
||||
const claimed = await this.update(
|
||||
{ id, status: In<AgentEvalResultStatus>(['success', 'error', 'cancelled']) },
|
||||
startedAttempt(),
|
||||
);
|
||||
return (claimed.affected ?? 0) > 0;
|
||||
async claimForRerun(id: string, runId: string): Promise<boolean> {
|
||||
return await this.runInTransaction({}, async (manager) => {
|
||||
// Serialize with terminal aggregation before changing any result. A write
|
||||
// lock works on both Postgres and SQLite and needs no schema change.
|
||||
await manager.update(AgentEvalRun, { id: runId }, { id: runId });
|
||||
const claimed = await manager.update(
|
||||
AgentEvalResult,
|
||||
{ id, runId, status: In<AgentEvalResultStatus>(['success', 'error', 'cancelled']) },
|
||||
startedAttempt(),
|
||||
);
|
||||
return (claimed.affected ?? 0) > 0;
|
||||
});
|
||||
}
|
||||
|
||||
/** Repair only an unfinished original attempt. Never overwrite a newer rerun. */
|
||||
async markUnfinishedAsError(id: string, runAt: Date | undefined, message: string): Promise<void> {
|
||||
await this.createQueryBuilder()
|
||||
.update()
|
||||
.set({
|
||||
status: 'error',
|
||||
completedAt: new Date(),
|
||||
errorCode: 'case_dispatch_failed',
|
||||
errorDetails: { message },
|
||||
})
|
||||
.where([{ id, status: 'new' }, ...(runAt ? [{ id, status: 'running', runAt }] : [])])
|
||||
.execute();
|
||||
}
|
||||
|
||||
async markAsCancelled(id: string) {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { Service } from '@n8n/di';
|
||||
import { DataSource, In, Repository } from '@n8n/typeorm';
|
||||
import { DataSource, In } from '@n8n/typeorm';
|
||||
import type { QueryDeepPartialEntity } from '@n8n/typeorm/query-builder/QueryPartialEntity';
|
||||
import type { IDataObject } from 'n8n-workflow';
|
||||
|
||||
import { AgentEvalRun } from '../entities';
|
||||
import { AgentEvalRun, AgentEvalResult } from '../entities';
|
||||
import { TransactionRunner } from '../services/transaction';
|
||||
import { BaseRepository } from './base-repository';
|
||||
import type { AgentEvalResultStatusCounts } from './agent-eval-result.repository.ee';
|
||||
|
||||
type CreateAgentEvalRunAttrs = {
|
||||
datasetId: string;
|
||||
|
|
@ -11,9 +15,9 @@ type CreateAgentEvalRunAttrs = {
|
|||
};
|
||||
|
||||
@Service()
|
||||
export class AgentEvalRunRepository extends Repository<AgentEvalRun> {
|
||||
constructor(dataSource: DataSource) {
|
||||
super(AgentEvalRun, dataSource.manager);
|
||||
export class AgentEvalRunRepository extends BaseRepository<AgentEvalRun> {
|
||||
constructor(dataSource: DataSource, transactionRunner: TransactionRunner) {
|
||||
super(AgentEvalRun, dataSource.manager, transactionRunner);
|
||||
}
|
||||
|
||||
async createRun(attrs: CreateAgentEvalRunAttrs): Promise<AgentEvalRun> {
|
||||
|
|
@ -36,13 +40,21 @@ export class AgentEvalRunRepository extends Repository<AgentEvalRun> {
|
|||
});
|
||||
}
|
||||
|
||||
async markAsCompleted(id: string, metrics: IDataObject | null) {
|
||||
return await this.update(id, {
|
||||
status: 'completed',
|
||||
completedAt: new Date(),
|
||||
metrics,
|
||||
runningInstanceId: null,
|
||||
});
|
||||
async markAsCompleted(
|
||||
id: string,
|
||||
metrics: IDataObject | null,
|
||||
counts?: AgentEvalResultStatusCounts,
|
||||
): Promise<boolean> {
|
||||
return await this.settleWithResultCounts(
|
||||
id,
|
||||
{
|
||||
status: 'completed',
|
||||
completedAt: new Date(),
|
||||
metrics,
|
||||
runningInstanceId: null,
|
||||
},
|
||||
counts,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -54,15 +66,20 @@ export class AgentEvalRunRepository extends Repository<AgentEvalRun> {
|
|||
errorCode: string,
|
||||
errorDetails?: IDataObject | null,
|
||||
metrics?: IDataObject | null,
|
||||
) {
|
||||
return await this.update(id, {
|
||||
status: 'error',
|
||||
completedAt: new Date(),
|
||||
errorCode,
|
||||
errorDetails: errorDetails ?? null,
|
||||
metrics: metrics ?? null,
|
||||
runningInstanceId: null,
|
||||
});
|
||||
counts?: AgentEvalResultStatusCounts,
|
||||
): Promise<boolean> {
|
||||
return await this.settleWithResultCounts(
|
||||
id,
|
||||
{
|
||||
status: 'error',
|
||||
completedAt: new Date(),
|
||||
errorCode,
|
||||
errorDetails: errorDetails ?? null,
|
||||
metrics: metrics ?? null,
|
||||
runningInstanceId: null,
|
||||
},
|
||||
counts,
|
||||
);
|
||||
}
|
||||
|
||||
/** Rewrites only the recorded tally — used when a result is removed from a settled run. */
|
||||
|
|
@ -70,12 +87,59 @@ export class AgentEvalRunRepository extends Repository<AgentEvalRun> {
|
|||
return await this.update(id, { metrics });
|
||||
}
|
||||
|
||||
async markAsCancelled(id: string, metrics: IDataObject | null = null) {
|
||||
return await this.update(id, {
|
||||
status: 'cancelled',
|
||||
completedAt: new Date(),
|
||||
metrics,
|
||||
runningInstanceId: null,
|
||||
async markAsCancelled(
|
||||
id: string,
|
||||
metrics: IDataObject | null = null,
|
||||
counts?: AgentEvalResultStatusCounts,
|
||||
): Promise<boolean> {
|
||||
return await this.settleWithResultCounts(
|
||||
id,
|
||||
{
|
||||
status: 'cancelled',
|
||||
completedAt: new Date(),
|
||||
metrics,
|
||||
runningInstanceId: null,
|
||||
},
|
||||
counts,
|
||||
);
|
||||
}
|
||||
|
||||
/** Reject stale aggregation if a result was claimed or settled after the read. */
|
||||
private async settleWithResultCounts(
|
||||
id: string,
|
||||
attributes: QueryDeepPartialEntity<AgentEvalRun>,
|
||||
counts?: AgentEvalResultStatusCounts,
|
||||
): Promise<boolean> {
|
||||
if (!counts) {
|
||||
const result = await this.update(id, attributes);
|
||||
return (result.affected ?? 0) > 0;
|
||||
}
|
||||
return await this.runInTransaction({}, async (manager) => {
|
||||
// Acquire the claim lock in a separate statement. Postgres must take a new
|
||||
// snapshot for the count predicates after a concurrent claim commits.
|
||||
await manager.update(AgentEvalRun, { id }, { id });
|
||||
const query = manager
|
||||
.createQueryBuilder(AgentEvalRun, 'run')
|
||||
.update()
|
||||
.set(attributes)
|
||||
.where({ id, status: 'running' });
|
||||
if (attributes.status !== 'cancelled') query.andWhere({ cancelRequested: false });
|
||||
for (const status of ['new', 'running', 'success', 'error', 'cancelled'] as const) {
|
||||
const countQuery = manager
|
||||
.createQueryBuilder(AgentEvalResult, 'result')
|
||||
.select('COUNT(*)')
|
||||
.where('result.runId = :runId')
|
||||
.andWhere(`result.status = :status_${status}`)
|
||||
.getQuery();
|
||||
const expected = status === 'new' || status === 'running' ? 0 : counts[status];
|
||||
query.andWhere(`(${countQuery}) = :count_${status}`, {
|
||||
runId: id,
|
||||
[`status_${status}`]: status,
|
||||
[`count_${status}`]: expected,
|
||||
});
|
||||
}
|
||||
const result = await query.execute();
|
||||
return (result.affected ?? 0) > 0;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
AgentEvalDatasetRepository,
|
||||
AgentEvalResultRepository,
|
||||
AgentEvalRunRepository,
|
||||
AgentEvalRun,
|
||||
GLOBAL_OWNER_ROLE,
|
||||
type User,
|
||||
} from '@n8n/db';
|
||||
|
|
@ -84,6 +85,9 @@ async function createAgent(projectId: string): Promise<Agent> {
|
|||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// The contention test needs two concurrent repository transactions. CI
|
||||
// otherwise uses one connection to detect accidental nested transactions.
|
||||
Container.get(GlobalConfig).database.postgresdb.poolSize = 2;
|
||||
await testModules.loadModules(['data-table', 'agents']);
|
||||
await testDb.init();
|
||||
owner = await createUserShell(GLOBAL_OWNER_ROLE);
|
||||
|
|
@ -111,6 +115,195 @@ afterAll(async () => {
|
|||
});
|
||||
|
||||
describe('AgentEvalRunnerService (integration)', () => {
|
||||
it('rejects aggregation when a result starts or finishes a new attempt after the count read', async () => {
|
||||
const project = await createTeamProject();
|
||||
const agent = await createAgent(project.id);
|
||||
const dataset = await Container.get(AgentEvalDatasetRepository).createDataset({
|
||||
name: 'ds',
|
||||
agentId: agent.id,
|
||||
datasetSource: 'data_table',
|
||||
datasetRef: { dataTableId: 'table' },
|
||||
columnMapping: { input: 'question' },
|
||||
});
|
||||
const runs = Container.get(AgentEvalRunRepository);
|
||||
const results = Container.get(AgentEvalResultRepository);
|
||||
const run = await runs.createRun({ datasetId: dataset.id });
|
||||
await runs.markAsRunning(run.id);
|
||||
const [result] = await results.seedResults([{ runId: run.id }]);
|
||||
await results.markAsError(result.id, 'execution_failed');
|
||||
const oldCounts = await results.countByStatus(run.id);
|
||||
expect(await results.claimForRerun(result.id, result.runId)).toBe(true);
|
||||
const oldMetrics = { total: 1, pending: 0, success: 0, error: 1, cancelled: 0 };
|
||||
expect(await runs.markAsCompleted(run.id, oldMetrics, oldCounts)).toBe(false);
|
||||
await results.markAsCompleted(result.id, { output: { finalText: 'Updated answer' } });
|
||||
expect(await runs.markAsCompleted(run.id, oldMetrics, oldCounts)).toBe(false);
|
||||
const currentCounts = await results.countByStatus(run.id);
|
||||
const currentMetrics = { ...oldMetrics, success: 1, error: 0 };
|
||||
await runs.requestCancellation(run.id);
|
||||
expect(await runs.markAsCompleted(run.id, currentMetrics, currentCounts)).toBe(false);
|
||||
expect(await runs.markAsCancelled(run.id, currentMetrics, currentCounts)).toBe(true);
|
||||
expect(await runs.findById(run.id)).toMatchObject({
|
||||
status: 'cancelled',
|
||||
metrics: currentMetrics,
|
||||
});
|
||||
});
|
||||
|
||||
it('repairs a failed original attempt without overwriting a newer rerun', async () => {
|
||||
const project = await createTeamProject();
|
||||
const agent = await createAgent(project.id);
|
||||
const dataset = await Container.get(AgentEvalDatasetRepository).createDataset({
|
||||
name: 'ds',
|
||||
agentId: agent.id,
|
||||
datasetSource: 'data_table',
|
||||
datasetRef: { dataTableId: 'table' },
|
||||
columnMapping: { input: 'question' },
|
||||
});
|
||||
const run = await Container.get(AgentEvalRunRepository).createRun({ datasetId: dataset.id });
|
||||
const results = Container.get(AgentEvalResultRepository);
|
||||
const [result, queued] = await results.seedResults([{ runId: run.id }, { runId: run.id }]);
|
||||
const originalAt = new Date('2025-01-01T00:00:00.000Z');
|
||||
await results.markAsRunning(result.id, originalAt);
|
||||
await results.markUnfinishedAsError(result.id, originalAt, 'Original write failed');
|
||||
expect((await results.findById(result.id))?.status).toBe('error');
|
||||
expect(await results.claimForRerun(result.id, run.id)).toBe(true);
|
||||
await results.markUnfinishedAsError(result.id, originalAt, 'Late repair');
|
||||
expect((await results.findById(result.id))?.status).toBe('running');
|
||||
await results.markUnfinishedAsError(queued.id, undefined, 'Dispatch failed');
|
||||
expect((await results.findById(queued.id))?.status).toBe('error');
|
||||
});
|
||||
|
||||
it.runIf(process.env.DB_TYPE === 'postgresdb')(
|
||||
'serializes a rerun claim with a terminal statement already waiting on the run lock',
|
||||
async () => {
|
||||
const project = await createTeamProject();
|
||||
const agent = await createAgent(project.id);
|
||||
const dataset = await Container.get(AgentEvalDatasetRepository).createDataset({
|
||||
name: 'ds',
|
||||
agentId: agent.id,
|
||||
datasetSource: 'data_table',
|
||||
datasetRef: { dataTableId: 'table' },
|
||||
columnMapping: { input: 'question' },
|
||||
});
|
||||
const runs = Container.get(AgentEvalRunRepository);
|
||||
const results = Container.get(AgentEvalResultRepository);
|
||||
const run = await runs.createRun({ datasetId: dataset.id });
|
||||
await runs.markAsRunning(run.id);
|
||||
const [result] = await results.seedResults([{ runId: run.id }]);
|
||||
await results.markAsCompleted(result.id, { output: {} });
|
||||
const oldCounts = await results.countByStatus(run.id);
|
||||
const connection = Container.get(DataSource);
|
||||
if (connection.options.type !== 'postgres') throw new Error('Postgres is required');
|
||||
// Keep the blocker and observer outside the two-connection application
|
||||
// pool so both repository transactions can acquire a connection.
|
||||
const control = await new DataSource({
|
||||
...connection.options,
|
||||
poolSize: 2,
|
||||
}).initialize();
|
||||
const blocker = control.createQueryRunner();
|
||||
const pending: Promise<unknown>[] = [];
|
||||
try {
|
||||
await blocker.startTransaction();
|
||||
await blocker.manager.update(AgentEvalRun, { id: run.id }, { id: run.id });
|
||||
const claim = results.claimForRerun(result.id, run.id);
|
||||
pending.push(claim);
|
||||
const waitForLocks = async (count: number) => {
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
const [row] = await control.query(`SELECT count(*)::int AS count
|
||||
FROM pg_stat_activity WHERE datname = current_database()
|
||||
AND wait_event_type = 'Lock' AND query LIKE 'UPDATE %agent_eval_run%'`);
|
||||
expect(row.count).toBe(count);
|
||||
},
|
||||
{ timeout: 2_000 },
|
||||
);
|
||||
};
|
||||
// The claim must hold this same lock before it can change the result.
|
||||
await waitForLocks(1);
|
||||
const settling = runs.markAsCompleted(run.id, { success: 1 }, oldCounts);
|
||||
pending.push(settling);
|
||||
await waitForLocks(2);
|
||||
await blocker.commitTransaction();
|
||||
expect(await claim).toBe(true);
|
||||
// Its first statement started before the claim committed. The count
|
||||
// statement must use a fresh snapshot after it acquires the lock.
|
||||
expect(await settling).toBe(false);
|
||||
expect((await runs.findById(run.id))?.status).toBe('running');
|
||||
expect((await results.findById(result.id))?.status).toBe('running');
|
||||
} finally {
|
||||
if (blocker.isTransactionActive) await blocker.rollbackTransaction();
|
||||
await blocker.release();
|
||||
await Promise.allSettled(pending);
|
||||
await control.destroy();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps an original batch running until a targeted rerun has settled', async () => {
|
||||
const project = await createTeamProject();
|
||||
const agent = await createAgent(project.id);
|
||||
const dataTables = Container.get(DataTableService);
|
||||
const table = await dataTables.createDataTable(project.id, {
|
||||
name: 'cases',
|
||||
columns: [{ name: 'question', type: 'string' }],
|
||||
});
|
||||
await dataTables.insertRows(table.id, project.id, [
|
||||
{ question: 'First' },
|
||||
{ question: 'Second' },
|
||||
]);
|
||||
const dataset = await Container.get(AgentEvalDatasetRepository).createDataset({
|
||||
name: 'ds',
|
||||
agentId: agent.id,
|
||||
datasetSource: 'data_table',
|
||||
datasetRef: { dataTableId: table.id },
|
||||
columnMapping: { input: 'question' },
|
||||
});
|
||||
agentRepository.findByIdAndProjectId.mockResolvedValue(agent);
|
||||
let finishOriginal!: () => void;
|
||||
let finishRerun!: () => void;
|
||||
const original = new Promise<void>((resolve) => {
|
||||
finishOriginal = resolve;
|
||||
});
|
||||
const rerun = new Promise<void>((resolve) => {
|
||||
finishRerun = resolve;
|
||||
});
|
||||
let firstAttempts = 0;
|
||||
evalAgentExecutionService.executeWithLlmMock.mockImplementation(
|
||||
async (_id, _user, _options, input) => {
|
||||
if (input === 'First' && ++firstAttempts > 1) await rerun;
|
||||
if (input === 'Second') await original;
|
||||
return { success: true, finalText: 'Answer', toolCalls: [], skippedFeatures: [] } as never;
|
||||
},
|
||||
);
|
||||
const runner = buildRunner();
|
||||
const { runId, finished } = await runner.startRun(dataset.id, project.id, owner);
|
||||
const results = Container.get(AgentEvalResultRepository);
|
||||
let firstResult;
|
||||
await vi.waitFor(async () => {
|
||||
const [rows] = await results.findAndCountByRunId(runId);
|
||||
firstResult = rows.find((row) => row.input?.input === 'First');
|
||||
expect(firstResult?.status).toBe('success');
|
||||
});
|
||||
if (!firstResult) throw new Error('Missing first result');
|
||||
const rerunFinished = runner.rerunResult(firstResult, agent.id, project.id, owner);
|
||||
await vi.waitFor(() => expect(firstAttempts).toBe(2));
|
||||
finishOriginal();
|
||||
await vi.waitFor(async () => {
|
||||
expect(await results.countByStatus(runId)).toMatchObject({ success: 1, running: 1 });
|
||||
});
|
||||
expect((await Container.get(AgentEvalRunRepository).findById(runId))?.status).toBe('running');
|
||||
finishRerun();
|
||||
await Promise.all([rerunFinished, finished]);
|
||||
expect(await runner.getRunSummary(runId, agent.id)).toMatchObject({
|
||||
status: 'completed',
|
||||
counts: { total: 2, success: 2, pending: 0 },
|
||||
});
|
||||
expect((await Container.get(AgentEvalRunRepository).findById(runId))?.metrics).toMatchObject({
|
||||
total: 2,
|
||||
success: 2,
|
||||
pending: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('runs each dataset row against the agent and persists results + run status', async () => {
|
||||
const project = await createTeamProject();
|
||||
const agent = await createAgent(project.id);
|
||||
|
|
|
|||
|
|
@ -164,6 +164,9 @@ describe('AgentEvalRunnerService', () => {
|
|||
moduleRegistry.isActive.mockReturnValue(true);
|
||||
datasetRepository = mock<AgentEvalDatasetRepository>();
|
||||
runRepository = mock<AgentEvalRunRepository>();
|
||||
runRepository.markAsCompleted.mockResolvedValue(true);
|
||||
runRepository.markAsCancelled.mockResolvedValue(true);
|
||||
runRepository.markAsError.mockResolvedValue(true);
|
||||
resultRepository = mock<AgentEvalResultRepository>();
|
||||
agentRepository = mock<AgentRepository>();
|
||||
dataTableService = mock<DataTableService>();
|
||||
|
|
@ -341,6 +344,246 @@ describe('AgentEvalRunnerService', () => {
|
|||
});
|
||||
|
||||
describe('running cases', () => {
|
||||
it('retries stale settlement until the repository confirms completion', async () => {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { success: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
runRepository.markAsCompleted.mockResolvedValueOnce(false).mockResolvedValue(true);
|
||||
runRepository.findById
|
||||
.mockResolvedValueOnce(mock({ id: 'run-1', status: 'running' }))
|
||||
.mockResolvedValue(mock({ id: 'run-1', status: 'completed' }));
|
||||
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
|
||||
expect(runRepository.markAsCompleted).toHaveBeenCalledTimes(2);
|
||||
expect(runRepository.findById).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('keeps one config snapshot when instructions change with a queued case', async () => {
|
||||
seedFor(
|
||||
[
|
||||
{ id: 'row-1', question: 'First' },
|
||||
{ id: 'row-2', question: 'Second' },
|
||||
],
|
||||
{ success: 2 },
|
||||
);
|
||||
let releaseFirst!: () => void;
|
||||
const first = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
const source = { agent: mock<Agent>({ id: 'agent-1' }), skillRefs: [], skills: {} };
|
||||
evalAgentExecutionService.captureRunSource.mockResolvedValue(source);
|
||||
evalAgentExecutionService.executeWithLlmMock
|
||||
.mockImplementationOnce(async () => {
|
||||
await first;
|
||||
return successExec() as never;
|
||||
})
|
||||
.mockResolvedValue(successExec() as never);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await vi.waitFor(() =>
|
||||
expect(evalAgentExecutionService.executeWithLlmMock).toHaveBeenCalledOnce(),
|
||||
);
|
||||
agentRepository.findByIdAndProjectId.mockResolvedValue(
|
||||
mock<Agent>({ id: 'agent-1', schema: { instructions: 'Changed' } as never }),
|
||||
);
|
||||
releaseFirst();
|
||||
await finished;
|
||||
expect(evalAgentExecutionService.captureRunSource).toHaveBeenCalledOnce();
|
||||
for (const call of evalAgentExecutionService.executeWithLlmMock.mock.calls) {
|
||||
expect(call[4]?.source).toBe(source);
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for an active targeted rerun after the last original case settles', async () => {
|
||||
seedFor(
|
||||
[
|
||||
{ id: 'row-1', question: 'First' },
|
||||
{ id: 'row-2', question: 'Second' },
|
||||
],
|
||||
{ success: 2 },
|
||||
);
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ running: 1, success: 1 }));
|
||||
let finished = false;
|
||||
const run = await service.startRun('ds-1', 'proj-1', user);
|
||||
void run.finished.then(() => {
|
||||
finished = true;
|
||||
});
|
||||
await vi.waitFor(() => expect(resultRepository.countByStatus).toHaveBeenCalled());
|
||||
expect(finished).toBe(false);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ success: 2 }));
|
||||
await run.finished;
|
||||
expect(runRepository.markAsCompleted).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ total: 2, success: 2, pending: 0 }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(['cancellation poll', 'original write'])(
|
||||
'waits for active reruns after a %s failure',
|
||||
async (failure) => {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { running: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
if (failure === 'cancellation poll') {
|
||||
runRepository.isCancellationRequested
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockRejectedValueOnce(new Error('Poll failed'))
|
||||
.mockResolvedValue(false);
|
||||
} else {
|
||||
resultRepository.markAsCompleted.mockRejectedValueOnce(new Error('Write failed'));
|
||||
resultRepository.markAsError.mockRejectedValueOnce(new Error('Error write failed'));
|
||||
}
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await vi.waitFor(() => expect(resultRepository.countByStatus).toHaveBeenCalled());
|
||||
// A failed original write may already have been repaired. This running
|
||||
// count is a newer targeted attempt and must not be ignored.
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ success: 1 }));
|
||||
await finished;
|
||||
expect(runRepository.markAsError).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'case_dispatch_failed',
|
||||
expect.anything(),
|
||||
expect.objectContaining({ success: 1, pending: 0 }),
|
||||
counts({ success: 1 }),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('honors a cancellation received while a targeted rerun is finishing', async () => {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { running: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await vi.waitFor(() => expect(resultRepository.countByStatus).toHaveBeenCalled());
|
||||
runRepository.isCancellationRequested.mockResolvedValue(true);
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ success: 1 }));
|
||||
await finished;
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ pending: 0 }),
|
||||
counts({ success: 1 }),
|
||||
);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ends with an error when a result cannot be recorded after execution', async () => {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { running: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockRejectedValue(new Error('Execution failed'));
|
||||
resultRepository.markAsError.mockRejectedValue(new Error('Result write failed'));
|
||||
resultRepository.markUnfinishedAsError
|
||||
.mockRejectedValueOnce(new Error('Repair failed'))
|
||||
.mockImplementationOnce(async () => {
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ error: 1 }));
|
||||
});
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
expect(runRepository.markAsError).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'case_dispatch_failed',
|
||||
expect.anything(),
|
||||
expect.objectContaining({ pending: 0, error: 1 }),
|
||||
counts({ error: 1 }),
|
||||
);
|
||||
expect(resultRepository.markUnfinishedAsError).toHaveBeenCalledTimes(2);
|
||||
expect(resultRepository.markUnfinishedAsError).toHaveBeenLastCalledWith(
|
||||
'res-0',
|
||||
expect.any(Date),
|
||||
'Result write failed',
|
||||
);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('bounds the wait for a result left running by another process', async () => {
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
try {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { running: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await vi.waitFor(() => expect(resultRepository.countByStatus).toHaveBeenCalled());
|
||||
now.mockReturnValue(1_000 + 65 * 60_000);
|
||||
await finished;
|
||||
expect(runRepository.markAsError).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'result_settlement_timeout',
|
||||
expect.anything(),
|
||||
expect.objectContaining({ pending: 1 }),
|
||||
);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves cancellation when an active rerun exceeds the settlement deadline', async () => {
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
try {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { running: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await vi.waitFor(() => expect(resultRepository.countByStatus).toHaveBeenCalled());
|
||||
runRepository.isCancellationRequested.mockResolvedValue(true);
|
||||
now.mockReturnValue(1_000 + 65 * 60_000);
|
||||
await finished;
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ pending: 1 }),
|
||||
);
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(['cancellation poll', 'original write'])(
|
||||
'reports a non-Error rejection from the %s',
|
||||
async (failure) => {
|
||||
seedFor([{ id: 'row-1', question: failure === 'original write' ? '' : 'First' }], {
|
||||
error: 1,
|
||||
});
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
const reason = `${failure} rejected`;
|
||||
if (failure === 'cancellation poll') {
|
||||
runRepository.isCancellationRequested
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockRejectedValueOnce(reason);
|
||||
} else {
|
||||
resultRepository.markAsError.mockRejectedValueOnce(reason);
|
||||
}
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
expect(runRepository.markAsError).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'case_dispatch_failed',
|
||||
expect.objectContaining({ errors: [reason] }),
|
||||
expect.objectContaining({ error: 1, pending: 0 }),
|
||||
counts({ error: 1 }),
|
||||
);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['removed', 'completed'] as const)(
|
||||
'stops settlement when the run was %s by another worker',
|
||||
async (state) => {
|
||||
seedFor([{ id: 'row-1', question: 'First' }], { success: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
runRepository.markAsCompleted.mockResolvedValue(false);
|
||||
runRepository.findById.mockResolvedValue(
|
||||
state === 'removed' ? null : mock({ id: 'run-1', status: 'completed' }),
|
||||
);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
expect(runRepository.markAsCompleted).toHaveBeenCalledOnce();
|
||||
expect(runRepository.findById).toHaveBeenCalledWith('run-1');
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('runs each case with its input verbatim, persists results, aggregates the run', async () => {
|
||||
seedFor(
|
||||
[
|
||||
|
|
@ -369,6 +612,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
user,
|
||||
{ projectId: 'proj-1' },
|
||||
'What is 2+2?',
|
||||
{ onProgress: expect.any(Function) },
|
||||
);
|
||||
expect(runRepository.markAsRunning).toHaveBeenCalledWith('run-1', 'main-1');
|
||||
expect(resultRepository.markAsCompleted).toHaveBeenCalledTimes(2);
|
||||
|
|
@ -379,9 +623,40 @@ describe('AgentEvalRunnerService', () => {
|
|||
error: 0,
|
||||
usage: { inputTokens: 6, outputTokens: 14 },
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('persists in-flight tool calls for the current attempt before completion', async () => {
|
||||
seedFor([{ id: 'row-1', question: 'Send a message' }], { success: 1 });
|
||||
const runAt = new Date();
|
||||
resultRepository.markAsRunning.mockResolvedValue(runAt);
|
||||
evalAgentExecutionService.executeWithLlmMock.mockImplementation(
|
||||
async (_agent, _user, _options, _input, hooks) => {
|
||||
await hooks?.onProgress?.([
|
||||
{
|
||||
tool: 'Slack_Tool',
|
||||
toolCallId: 'call-1',
|
||||
status: 'running',
|
||||
kind: 'node',
|
||||
mocked: false,
|
||||
interceptedRequests: [],
|
||||
},
|
||||
]);
|
||||
expect(resultRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
return successExec() as never;
|
||||
},
|
||||
);
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
expect(resultRepository.updateRunningToolCalls).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
runAt,
|
||||
{ calls: [expect.objectContaining({ status: 'running', toolCallId: 'call-1' })] },
|
||||
);
|
||||
expect(resultRepository.markAsCompleted).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('snapshots input/expectedOutput/criteria onto the seeded case', async () => {
|
||||
seedFor([{ id: 'row-1', question: 'Q', answer: 'A', check: 'C' }], { success: 1 });
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
|
|
@ -459,7 +734,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
expect(resultRepository.markAsRunning).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still completes the run when recording an empty-input case fails', async () => {
|
||||
it('reports an error and processes other cases when recording an empty-input case fails', async () => {
|
||||
seedFor(
|
||||
[
|
||||
{ id: 'row-1', question: ' ' },
|
||||
|
|
@ -475,8 +750,14 @@ describe('AgentEvalRunnerService', () => {
|
|||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
|
||||
expect(runRepository.markAsCompleted).toHaveBeenCalled();
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
expect(runRepository.markAsError).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'case_dispatch_failed',
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
counts({ success: 1, error: 1 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('pages through every row when the table exceeds one page', async () => {
|
||||
|
|
@ -772,6 +1053,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
expect(runRepository.markAsCompleted).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ success: 1 }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -813,6 +1095,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ cancelled: 2 }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -852,6 +1135,9 @@ describe('AgentEvalRunnerService', () => {
|
|||
.mockRejectedValueOnce(new Error('db down'))
|
||||
.mockResolvedValue(undefined as never);
|
||||
|
||||
resultRepository.markUnfinishedAsError.mockImplementationOnce(async () => {
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ cancelled: 1, error: 1 }));
|
||||
});
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
|
||||
|
|
@ -859,7 +1145,8 @@ describe('AgentEvalRunnerService', () => {
|
|||
expect(resultRepository.markAsCancelled).toHaveBeenCalledTimes(2);
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ total: 2, cancelled: 1, pending: 1 }),
|
||||
expect.objectContaining({ total: 2, cancelled: 1, error: 1, pending: 0 }),
|
||||
counts({ cancelled: 1, error: 1 }),
|
||||
);
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -879,6 +1166,9 @@ describe('AgentEvalRunnerService', () => {
|
|||
.mockResolvedValue(false);
|
||||
evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(successExec() as never);
|
||||
|
||||
resultRepository.markUnfinishedAsError.mockImplementationOnce(async () => {
|
||||
resultRepository.countByStatus.mockResolvedValue(counts({ success: 1, error: 1 }));
|
||||
});
|
||||
const { finished } = await service.startRun('ds-1', 'proj-1', user);
|
||||
await finished;
|
||||
|
||||
|
|
@ -887,7 +1177,8 @@ describe('AgentEvalRunnerService', () => {
|
|||
'run-1',
|
||||
'case_dispatch_failed',
|
||||
expect.objectContaining({ errors: ['db down'] }),
|
||||
expect.objectContaining({ total: 2, success: 1 }),
|
||||
expect.objectContaining({ total: 2, success: 1, error: 1, pending: 0 }),
|
||||
counts({ success: 1, error: 1 }),
|
||||
);
|
||||
expect(runRepository.markAsCompleted).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -910,6 +1201,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
'case_dispatch_failed',
|
||||
expect.objectContaining({ errors: ['db down'] }),
|
||||
expect.objectContaining({ total: 1, success: 1 }),
|
||||
counts({ success: 1 }),
|
||||
);
|
||||
// Not the tally-less `run_failed` path.
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalledWith(
|
||||
|
|
@ -957,6 +1249,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
}),
|
||||
// counts land under `metrics`, same as every other terminal status
|
||||
expect.objectContaining({ total: 2, cancelled: 2 }),
|
||||
expect.anything(),
|
||||
);
|
||||
// A deadline is a failure, not the user's Stop.
|
||||
expect(runRepository.markAsCancelled).not.toHaveBeenCalled();
|
||||
|
|
@ -976,7 +1269,11 @@ describe('AgentEvalRunnerService', () => {
|
|||
await vi.advanceTimersByTimeAsync(30 * 60_000);
|
||||
await finished;
|
||||
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith('run-1', expect.anything());
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -1008,6 +1305,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
expect(runRepository.markAsCompleted).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.objectContaining({ total: 1, success: 1 }),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(runRepository.markAsError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -1107,7 +1405,11 @@ describe('AgentEvalRunnerService', () => {
|
|||
executionId: expect.stringContaining('-case-1'),
|
||||
}),
|
||||
);
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith('run-1', expect.anything());
|
||||
expect(runRepository.markAsCancelled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(evalAgentExecutionService.executeWithLlmMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -1239,7 +1541,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
'already running',
|
||||
);
|
||||
|
||||
expect(resultRepository.claimForRerun).toHaveBeenCalledWith('res-1');
|
||||
expect(resultRepository.claimForRerun).toHaveBeenCalledWith('res-1', 'run-1');
|
||||
expect(evalAgentExecutionService.executeWithLlmMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -1311,8 +1613,9 @@ describe('AgentEvalRunnerService', () => {
|
|||
user,
|
||||
{ projectId: 'proj-1' },
|
||||
'What is 2+2?',
|
||||
{ onProgress: expect.any(Function) },
|
||||
);
|
||||
expect(resultRepository.markAsRunning).toHaveBeenCalledWith('res-1');
|
||||
expect(resultRepository.markAsRunning).toHaveBeenCalledWith('res-1', expect.any(Date));
|
||||
expect(resultRepository.markAsCompleted).toHaveBeenCalledWith(
|
||||
'res-1',
|
||||
expect.objectContaining({ output: expect.objectContaining({ finalText: 'the answer' }) }),
|
||||
|
|
@ -1376,6 +1679,7 @@ describe('AgentEvalRunnerService', () => {
|
|||
user,
|
||||
{ projectId: 'proj-1' },
|
||||
'What is 2+2?',
|
||||
{ onProgress: expect.any(Function) },
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import type { AgentEvalVerdict, AgentEvalRunSummary } from '@n8n/api-types';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import {
|
||||
INSTANCE_AI_EVAL_AGENT_MAX_TIMEOUT_MS,
|
||||
type AgentEvalVerdict,
|
||||
type AgentEvalRunSummary,
|
||||
} from '@n8n/api-types';
|
||||
import { Logger, ModuleRegistry } from '@n8n/backend-common';
|
||||
import { GlobalConfig } from '@n8n/config';
|
||||
import type { AgentEvalDataset, AgentEvalResult, User } from '@n8n/db';
|
||||
|
|
@ -26,6 +31,7 @@ import { resolveEvaluationConcurrencyLimit } from '@/evaluation.ee/evaluation-co
|
|||
import { License } from '@/license';
|
||||
import { AgentConfigService } from '@/modules/agents/agent-config.service';
|
||||
import { AgentsSettingsService } from '@/modules/agents/agents-settings.service';
|
||||
import type { AgentRunSource } from '@/modules/agents/utils/agent-published-snapshot';
|
||||
import { AgentRepository } from '@/modules/agents/repositories/agent.repository';
|
||||
import { DataTableService } from '@/modules/data-table/data-table.service';
|
||||
import { EvalAgentExecutionService } from '@/modules/instance-ai/eval/agent-execution.service';
|
||||
|
|
@ -46,6 +52,9 @@ const MAX_CASES = 500;
|
|||
// `setTimeout` holds its delay in a 32-bit int and silently coerces anything
|
||||
// larger to 1ms, so a deadline past ~24.8 days must be clamped, not passed on.
|
||||
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
||||
// Cover the maximum agent execution budget, plus five minutes for judging.
|
||||
const RESULT_SETTLEMENT_TIMEOUT_MS = INSTANCE_AI_EVAL_AGENT_MAX_TIMEOUT_MS + 5 * 60_000;
|
||||
const RESULT_SETTLEMENT_POLL_MS = 1_000;
|
||||
|
||||
/** A dataset row resolved into a runnable case via the dataset's column mapping. */
|
||||
interface ResolvedCase {
|
||||
|
|
@ -62,6 +71,12 @@ interface CaseUsage {
|
|||
outputTokens: number;
|
||||
}
|
||||
|
||||
interface RunDeadline {
|
||||
hasExpired: () => boolean;
|
||||
clear: () => void;
|
||||
deadlineMinutes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an agent-eval dataset against a real agent and persists per-case
|
||||
* results, then aggregates run-level status.
|
||||
|
|
@ -148,13 +163,11 @@ export class AgentEvalRunnerService {
|
|||
throw new BadRequestError('The dataset has no rows to run.');
|
||||
}
|
||||
|
||||
const source = await this.evalAgentExecutionService.captureRunSource(agent);
|
||||
|
||||
const run = await this.runRepository.createRun({
|
||||
datasetId: dataset.id,
|
||||
// Left unpinned: execution reloads the live agent by id (there is no
|
||||
// snapshot-execution path yet), so tagging a specific version would
|
||||
// misrepresent what actually ran — mirrors the workflow eval, which
|
||||
// records a version only when it executes that pinned snapshot.
|
||||
// Version-pinned execution is a follow-up.
|
||||
// This is a detached draft snapshot, not a published agent version.
|
||||
agentVersionId: null,
|
||||
createdById: user.id,
|
||||
});
|
||||
|
|
@ -191,6 +204,7 @@ export class AgentEvalRunnerService {
|
|||
user,
|
||||
cases,
|
||||
seeded,
|
||||
source,
|
||||
timeoutMs: options.timeoutMs,
|
||||
});
|
||||
|
||||
|
|
@ -242,7 +256,7 @@ export class AgentEvalRunnerService {
|
|||
|
||||
// Atomic, unlike the caller's snapshot status check: of two concurrent
|
||||
// requests for the same result, only one wins the claim and executes.
|
||||
if (!(await this.resultRepository.claimForRerun(result.id))) {
|
||||
if (!(await this.resultRepository.claimForRerun(result.id, result.runId))) {
|
||||
throw new BadRequestError(`Agent eval result ${result.id} is already running.`);
|
||||
}
|
||||
|
||||
|
|
@ -299,6 +313,7 @@ export class AgentEvalRunnerService {
|
|||
user: User;
|
||||
cases: ResolvedCase[];
|
||||
seeded: AgentEvalResult[];
|
||||
source: AgentRunSource;
|
||||
timeoutMs?: number;
|
||||
}): Promise<void> {
|
||||
const { runId, cases, seeded } = ctx;
|
||||
|
|
@ -350,8 +365,7 @@ export class AgentEvalRunnerService {
|
|||
};
|
||||
|
||||
// An empty input can't produce an execution, so its verdict is recorded
|
||||
// without taking a queue slot. Self-contained like `runCase`: one unusable
|
||||
// case must not fail the run just because recording it failed.
|
||||
// without taking a queue slot. A failed write reaches the batch error path.
|
||||
const markEmptyInput = async (resultRow: AgentEvalResult) => {
|
||||
try {
|
||||
await this.resultRepository.markAsError(resultRow.id, 'empty_input', {
|
||||
|
|
@ -362,10 +376,12 @@ export class AgentEvalRunnerService {
|
|||
`[AgentEvalRunner] Could not record empty input for case ${resultRow.id}`,
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const deadline = this.startRunDeadline(abort);
|
||||
const originalAttempts = new Map<string, Date>();
|
||||
|
||||
// The stop reads/writes around `runCase` sit outside its safety net, so
|
||||
// collect their throws instead of letting one reject the whole pool.
|
||||
|
|
@ -401,7 +417,10 @@ export class AgentEvalRunnerService {
|
|||
await stopCase(resultRow);
|
||||
return;
|
||||
}
|
||||
const usage = await this.runCase(resultRow, resolvedCase, ctx);
|
||||
const usage = await this.runCase(resultRow, resolvedCase, {
|
||||
...ctx,
|
||||
onAttemptStarted: (runAt) => originalAttempts.set(resultRow.id, runAt),
|
||||
});
|
||||
if (usage) {
|
||||
totalUsage.inputTokens += usage.inputTokens;
|
||||
totalUsage.outputTokens += usage.outputTokens;
|
||||
|
|
@ -416,22 +435,13 @@ export class AgentEvalRunnerService {
|
|||
deadline.clear();
|
||||
|
||||
const dispatchFailures: string[] = [];
|
||||
for (const settlement of settlements) {
|
||||
const unfinishedOriginals = new Map<string, string>();
|
||||
for (const [index, settlement] of settlements.entries()) {
|
||||
if (settlement.status !== 'rejected') continue;
|
||||
const reason: unknown = settlement.reason;
|
||||
dispatchFailures.push(reason instanceof Error ? reason.message : String(reason));
|
||||
}
|
||||
// Re-read once more: a cancel that arrived after every case had already
|
||||
// settled is never observed above, so honor it rather than reporting the
|
||||
// run completed with the flag still set. Collected like the pool's throws,
|
||||
// since letting this one escape would lose the tally it guards.
|
||||
let wasCancelled = cancelObserved;
|
||||
if (!wasCancelled) {
|
||||
try {
|
||||
wasCancelled = await this.runRepository.isCancellationRequested(runId);
|
||||
} catch (error) {
|
||||
dispatchFailures.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
const message = reason instanceof Error ? reason.message : String(reason);
|
||||
dispatchFailures.push(message);
|
||||
unfinishedOriginals.set(seeded[index].id, message);
|
||||
}
|
||||
|
||||
if (dispatchFailures.length > 0) {
|
||||
|
|
@ -441,25 +451,111 @@ export class AgentEvalRunnerService {
|
|||
);
|
||||
}
|
||||
|
||||
await this.settleRun({
|
||||
runId,
|
||||
unfinishedOriginals,
|
||||
originalAttempts,
|
||||
dispatchFailures,
|
||||
cancelObserved,
|
||||
totalUsage,
|
||||
deadline,
|
||||
stoppedCases,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await this.failRun(runId, message);
|
||||
}
|
||||
}
|
||||
|
||||
private async settleRun(ctx: {
|
||||
runId: string;
|
||||
unfinishedOriginals: Map<string, string>;
|
||||
originalAttempts: Map<string, Date>;
|
||||
dispatchFailures: string[];
|
||||
cancelObserved: boolean;
|
||||
totalUsage: CaseUsage;
|
||||
deadline: RunDeadline;
|
||||
stoppedCases: number;
|
||||
}): Promise<void> {
|
||||
const {
|
||||
runId,
|
||||
unfinishedOriginals,
|
||||
originalAttempts,
|
||||
dispatchFailures,
|
||||
cancelObserved,
|
||||
totalUsage,
|
||||
deadline,
|
||||
stoppedCases,
|
||||
} = ctx;
|
||||
let wasCancelled = cancelObserved;
|
||||
|
||||
// A settled sibling can be rerun while the original pool is active.
|
||||
// Fence the terminal write against those attempts, including other mains.
|
||||
const settlementDeadline = Date.now() + RESULT_SETTLEMENT_TIMEOUT_MS;
|
||||
while (true) {
|
||||
for (const [resultId, message] of unfinishedOriginals) {
|
||||
try {
|
||||
await this.resultRepository.markUnfinishedAsError(
|
||||
resultId,
|
||||
originalAttempts.get(resultId),
|
||||
message,
|
||||
);
|
||||
unfinishedOriginals.delete(resultId);
|
||||
} catch (error) {
|
||||
this.logger.warn(`[AgentEvalRunner] Could not settle original case ${resultId}`, {
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!wasCancelled) {
|
||||
try {
|
||||
wasCancelled = await this.runRepository.isCancellationRequested(runId);
|
||||
} catch (error) {
|
||||
dispatchFailures.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
const counts = await this.resultRepository.countByStatus(runId);
|
||||
const metrics: IDataObject = { ...toSummaryCounts(counts), usage: { ...totalUsage } };
|
||||
if (Date.now() >= settlementDeadline) {
|
||||
if (wasCancelled) {
|
||||
await this.runRepository.markAsCancelled(runId, metrics);
|
||||
} else {
|
||||
await this.runRepository.markAsError(
|
||||
runId,
|
||||
'result_settlement_timeout',
|
||||
{
|
||||
message: 'A result did not settle after the original batch finished.',
|
||||
},
|
||||
metrics,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (counts.new > 0 || counts.running > 0) {
|
||||
await sleep(RESULT_SETTLEMENT_POLL_MS);
|
||||
continue;
|
||||
}
|
||||
// Dispatch failures use the same claim fence as successful completion.
|
||||
const finalCounts = counts;
|
||||
|
||||
let settled;
|
||||
// Every branch records the tally under `metrics`, so pollers never find a
|
||||
// settled run whose counts are missing or filed elsewhere.
|
||||
if (wasCancelled) {
|
||||
// A user stop outranks the deadline: it's intent, not a failure.
|
||||
await this.runRepository.markAsCancelled(runId, metrics);
|
||||
settled = await this.runRepository.markAsCancelled(runId, metrics, finalCounts);
|
||||
} else if (deadline.hasExpired() && stoppedCases > 0) {
|
||||
await this.runRepository.markAsError(
|
||||
settled = await this.runRepository.markAsError(
|
||||
runId,
|
||||
'timeout',
|
||||
{
|
||||
message: `Run exceeded its ${deadline.deadlineMinutes}-minute deadline; ${stoppedCases} case(s) were not started.`,
|
||||
},
|
||||
metrics,
|
||||
finalCounts,
|
||||
);
|
||||
} else if (dispatchFailures.length > 0) {
|
||||
await this.runRepository.markAsError(
|
||||
settled = await this.runRepository.markAsError(
|
||||
runId,
|
||||
'case_dispatch_failed',
|
||||
{
|
||||
|
|
@ -467,6 +563,7 @@ export class AgentEvalRunnerService {
|
|||
errors: dispatchFailures,
|
||||
},
|
||||
metrics,
|
||||
finalCounts,
|
||||
);
|
||||
} else {
|
||||
if (deadline.hasExpired()) {
|
||||
|
|
@ -476,11 +573,12 @@ export class AgentEvalRunnerService {
|
|||
`[AgentEvalRunner] Run ${runId} overran its ${deadline.deadlineMinutes}-minute deadline, but every case had finished`,
|
||||
);
|
||||
}
|
||||
await this.runRepository.markAsCompleted(runId, metrics);
|
||||
settled = await this.runRepository.markAsCompleted(runId, metrics, finalCounts);
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await this.failRun(runId, message);
|
||||
if (settled) break;
|
||||
const current = await this.runRepository.findById(runId);
|
||||
if (current?.status !== 'running') break;
|
||||
await sleep(RESULT_SETTLEMENT_POLL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -488,11 +586,7 @@ export class AgentEvalRunnerService {
|
|||
* Arm the run-level deadline; expiring trips the run's abort. Non-positive
|
||||
* disables. Bounds when the last case may *start*, not when the run ends.
|
||||
*/
|
||||
private startRunDeadline(abort: AbortController): {
|
||||
hasExpired: () => boolean;
|
||||
clear: () => void;
|
||||
deadlineMinutes: number;
|
||||
} {
|
||||
private startRunDeadline(abort: AbortController): RunDeadline {
|
||||
const deadlineMinutes = this.globalConfig.evaluation.agentEvalsRunTimeoutMinutes;
|
||||
if (deadlineMinutes <= 0) {
|
||||
return { hasExpired: () => false, clear: () => undefined, deadlineMinutes };
|
||||
|
|
@ -593,10 +687,9 @@ export class AgentEvalRunnerService {
|
|||
|
||||
/**
|
||||
* Execute one case and persist its result. Returns the case's token usage.
|
||||
* Fully self-contained: a thrown execution or DB error is converted into a
|
||||
* per-case error so one case can never abort the batch or leave its result
|
||||
* stuck `running`. Assumes a non-empty input — the caller screens those out
|
||||
* before taking a queue slot.
|
||||
* Execution errors become per-case errors. If that write also fails, the batch
|
||||
* reports the persistence failure. The caller screens out empty input before
|
||||
* taking a queue slot.
|
||||
*
|
||||
* On a successful execution, also grades the output via {@link judgeCase}
|
||||
* and persists the verdict — this is the one place both `executeRun`'s pool
|
||||
|
|
@ -607,16 +700,35 @@ export class AgentEvalRunnerService {
|
|||
private async runCase(
|
||||
resultRow: AgentEvalResult,
|
||||
resolvedCase: ResolvedCase,
|
||||
ctx: { agentId: string; projectId: string; user: User; timeoutMs?: number },
|
||||
ctx: {
|
||||
agentId: string;
|
||||
projectId: string;
|
||||
user: User;
|
||||
timeoutMs?: number;
|
||||
source?: AgentRunSource;
|
||||
onAttemptStarted?: (runAt: Date) => void;
|
||||
},
|
||||
): Promise<CaseUsage | undefined> {
|
||||
try {
|
||||
await this.resultRepository.markAsRunning(resultRow.id);
|
||||
const attemptStartedAt = new Date();
|
||||
ctx.onAttemptStarted?.(attemptStartedAt);
|
||||
const runAt = await this.resultRepository.markAsRunning(resultRow.id, attemptStartedAt);
|
||||
|
||||
const execResult = await this.evalAgentExecutionService.executeWithLlmMock(
|
||||
ctx.agentId,
|
||||
ctx.user,
|
||||
{ projectId: ctx.projectId, ...(ctx.timeoutMs ? { timeoutMs: ctx.timeoutMs } : {}) },
|
||||
resolvedCase.input,
|
||||
{
|
||||
...(ctx.source ? { source: ctx.source } : {}),
|
||||
onProgress: async (calls) => {
|
||||
await this.resultRepository.updateRunningToolCalls(
|
||||
resultRow.id,
|
||||
runAt,
|
||||
toJsonObject({ calls }),
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const usage = normalizeUsage(execResult.usage);
|
||||
|
|
@ -673,6 +785,7 @@ export class AgentEvalRunnerService {
|
|||
this.logger.error(`[AgentEvalRunner] Could not record failure for case ${resultRow.id}`, {
|
||||
error: markError instanceof Error ? markError.message : String(markError),
|
||||
});
|
||||
throw markError;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { GenerateResult } from '@n8n/agents';
|
||||
import { AgentEvent, type AgentEventHandler, type GenerateResult } from '@n8n/agents';
|
||||
import type { AgentJsonConfig } from '@n8n/api-types';
|
||||
import type { Logger, ModuleRegistry } from '@n8n/backend-common';
|
||||
import type { OutboundHttp } from '@n8n/backend-network';
|
||||
|
|
@ -29,6 +29,7 @@ import {
|
|||
import { generateAgentScenarioSeed } from '../agent-scenario-seed';
|
||||
import { createMcpMockFetch } from '../mcp-mock-fetch';
|
||||
import { createLlmMockHandler } from '../mock-handler';
|
||||
import { createWebSearchMock } from '../web-search-mock';
|
||||
|
||||
// The service resolves the agents-module surface lazily; top-level vi.mock
|
||||
// keeps those specifiers from loading the real module graph (and avoids the
|
||||
|
|
@ -54,6 +55,7 @@ vi.mock('@/utils/ai-proxy-fetch', () => ({ createAiProxyFetch: vi.fn(() => vi.fn
|
|||
vi.mock('../agent-scenario-seed', () => ({ generateAgentScenarioSeed: vi.fn() }));
|
||||
vi.mock('../mcp-mock-fetch', () => ({ createMcpMockFetch: vi.fn(() => vi.fn()) }));
|
||||
vi.mock('../mock-handler', () => ({ createLlmMockHandler: vi.fn() }));
|
||||
vi.mock('../web-search-mock', () => ({ createWebSearchMock: vi.fn(() => vi.fn()) }));
|
||||
|
||||
const logger = mock<Logger>();
|
||||
const settingsService = mock<AgentsSettingsService>();
|
||||
|
|
@ -160,6 +162,500 @@ describe('EvalAgentExecutionService.executeWithLlmMock', () => {
|
|||
|
||||
const request = { projectId: 'proj-1' };
|
||||
|
||||
it('keeps a detached root config, tools, and resolved skills for each batch case', async () => {
|
||||
const entity = makeEntity({ ...baseConfig });
|
||||
const skills = { skill1: { name: 'Support', description: 'Before', content: 'Old skill' } };
|
||||
const skillRefs = [{ type: 'skill', id: 'skill1' }];
|
||||
const skillService = createAgentSkillRefsService();
|
||||
vi.spyOn(skillService, 'readDraft').mockResolvedValue({ skillRefs, skills } as never);
|
||||
Container.set(AgentSkillRefsService, skillService);
|
||||
const service = buildService();
|
||||
const source = await service.captureRunSource(entity);
|
||||
entity.schema!.instructions = 'Changed instructions';
|
||||
entity.tools!['t-1'].code = 'Changed code';
|
||||
skills.skill1.content = 'Changed skill';
|
||||
findByIdAndProjectId.mockResolvedValue(entity);
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: { generate: vi.fn().mockResolvedValue(makeGenerateResult()), close: vi.fn() },
|
||||
});
|
||||
|
||||
await service.executeWithLlmMock('agent-1', user, { projectId: 'proj-1' }, 'First', { source });
|
||||
await service.executeWithLlmMock('agent-1', user, { projectId: 'proj-1' }, 'Second', {
|
||||
source,
|
||||
});
|
||||
|
||||
expect(skillService.readDraft).toHaveBeenCalledOnce();
|
||||
expect(reconstructFromAgentEntity).toHaveBeenCalledTimes(2);
|
||||
for (const [snapshot] of reconstructFromAgentEntity.mock.calls) {
|
||||
expect(snapshot.agent.schema.instructions).toBe(baseConfig.instructions);
|
||||
expect(snapshot.agent.tools['t-1'].code).toBe('export default 1');
|
||||
expect(snapshot.skills.skill1.content).toBe('Old skill');
|
||||
}
|
||||
expect(source.agent.schema!.instructions).toBe(baseConfig.instructions);
|
||||
expect(userHasScopes).toHaveBeenCalledTimes(2);
|
||||
await service.executeWithLlmMock('agent-1', user, { projectId: 'proj-1' }, 'Rerun');
|
||||
const [current] = reconstructFromAgentEntity.mock.calls[2];
|
||||
expect(current.agent.schema.instructions).toBe('Changed instructions');
|
||||
expect(current.agent.tools['t-1'].code).toBe('Changed code');
|
||||
expect(current.skills.skill1.content).toBe('Changed skill');
|
||||
expect(skillService.readDraft).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('still checks permissions and project identity for a captured source', async () => {
|
||||
const service = buildService();
|
||||
const source = await service.captureRunSource(makeEntity({ ...baseConfig }));
|
||||
vi.mocked(userHasScopes).mockResolvedValue(false);
|
||||
expect(
|
||||
await service.executeWithLlmMock('agent-1', user, { projectId: 'proj-1' }, 'Case', {
|
||||
source,
|
||||
}),
|
||||
).toMatchObject({ success: false });
|
||||
vi.mocked(userHasScopes).mockResolvedValue(true);
|
||||
expect(
|
||||
await service.executeWithLlmMock('agent-1', user, { projectId: 'other-project' }, 'Case', {
|
||||
source,
|
||||
}),
|
||||
).toMatchObject({ success: false });
|
||||
expect(reconstructFromAgentEntity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses a captured source when the agent is no longer in the project', async () => {
|
||||
const service = buildService();
|
||||
const source = await service.captureRunSource(makeEntity({ ...baseConfig }));
|
||||
findByIdAndProjectId.mockResolvedValue(null);
|
||||
const result = await service.executeWithLlmMock(
|
||||
'agent-1',
|
||||
user,
|
||||
{ projectId: 'proj-1' },
|
||||
'Case',
|
||||
{ source },
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.errors).toEqual(['Agent agent-1 not found or not accessible']);
|
||||
expect(reconstructFromAgentEntity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'keeps runtime IDs for repeated calls at completion (identical: %s)',
|
||||
async (identical) => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const first = { tool: 'Slack_Tool', input: { text: 'first' }, output: { sent: 'first' } };
|
||||
const second = identical
|
||||
? { ...first }
|
||||
: { ...first, input: { text: 'second' }, output: { sent: 'second' } };
|
||||
const skipped = { ...first, output: 'Skipped', canceled: true };
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
close: vi.fn(),
|
||||
generate: vi.fn(async () => {
|
||||
for (const [toolCallId, entry] of [
|
||||
['first', first],
|
||||
['second', second],
|
||||
] as const) {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId,
|
||||
toolName: entry.tool,
|
||||
args: entry.input,
|
||||
});
|
||||
}
|
||||
// Completion order can differ from the original tool-call order.
|
||||
for (const [toolCallId, entry] of [
|
||||
['second', second],
|
||||
['first', first],
|
||||
] as const) {
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolCallId,
|
||||
toolName: entry.tool,
|
||||
result: entry.output,
|
||||
isError: false,
|
||||
});
|
||||
}
|
||||
return makeGenerateResult({ toolCalls: [first, second, skipped] });
|
||||
}),
|
||||
},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock(
|
||||
'agent-1',
|
||||
user,
|
||||
request,
|
||||
'Send twice',
|
||||
{ onProgress: vi.fn() },
|
||||
);
|
||||
expect(result.toolCalls).toEqual([
|
||||
expect.objectContaining({ ...first, toolCallId: 'first', status: 'completed' }),
|
||||
expect.objectContaining({ ...second, toolCallId: 'second', status: 'completed' }),
|
||||
expect.objectContaining({ tool: 'Slack_Tool', output: 'Skipped', error: 'canceled' }),
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it('records search progress only for the matching active call and ignores late endings', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
const search = { query: 'shipping' };
|
||||
const searchResult = { query: 'shipping', results: [] };
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
close: vi.fn(),
|
||||
generate: vi.fn(async () => {
|
||||
for (const toolName of ['Slack_Tool', 'web_search']) {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolName,
|
||||
toolCallId: toolName,
|
||||
args: search,
|
||||
});
|
||||
}
|
||||
const { onSearch } = vi.mocked(createWebSearchMock).mock.calls[0][0];
|
||||
onSearch(search, searchResult);
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'web_search',
|
||||
toolCallId: 'web_search',
|
||||
result: searchResult,
|
||||
isError: false,
|
||||
});
|
||||
// A delegated search has no active parent call to update.
|
||||
onSearch({ query: 'returns' }, { query: 'returns', results: [] });
|
||||
// A runtime can report completion without a prior start event.
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'lookup',
|
||||
toolCallId: 'orphan',
|
||||
result: 'found',
|
||||
isError: false,
|
||||
});
|
||||
return makeGenerateResult({
|
||||
toolCalls: [{ tool: 'lookup', input: {}, output: 'found' }],
|
||||
});
|
||||
}),
|
||||
},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock('agent-1', user, request, 'Search', {
|
||||
onProgress,
|
||||
});
|
||||
expect(onProgress.mock.calls[2][0]).toEqual([
|
||||
expect.objectContaining({ tool: 'Slack_Tool', mocked: false, interceptedRequests: [] }),
|
||||
expect.objectContaining({
|
||||
tool: 'web_search',
|
||||
status: 'running',
|
||||
mocked: true,
|
||||
interceptedRequests: [expect.objectContaining({ requestBody: search })],
|
||||
}),
|
||||
]);
|
||||
expect(onProgress).toHaveBeenCalledTimes(6);
|
||||
expect(
|
||||
result.toolCalls.find((call) => call.tool === 'web_search')?.interceptedRequests,
|
||||
).toHaveLength(2);
|
||||
expect(result.toolCalls.find((call) => call.toolCallId === 'orphan')).toMatchObject({
|
||||
kind: 'other',
|
||||
output: 'found',
|
||||
});
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolName: 'lookup',
|
||||
toolCallId: 'late',
|
||||
result: 'ignored',
|
||||
isError: false,
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(onProgress).toHaveBeenCalledTimes(6);
|
||||
});
|
||||
|
||||
it('classifies delegated MCP records from their intercepted request metadata', async () => {
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
close: vi.fn(),
|
||||
generate: vi.fn(async () => {
|
||||
vi.mocked(createMcpMockFetch).mock.calls[0][0].onToolCall({
|
||||
serverName: 'child',
|
||||
toolName: 'lookup',
|
||||
args: {},
|
||||
result: { text: 'found', isError: false },
|
||||
});
|
||||
const instrumentation = reconstructFromAgentEntity.mock.calls[0][5] as {
|
||||
onMcpToolCallSettled: (event: {
|
||||
serverName: string;
|
||||
toolName: string;
|
||||
success: boolean;
|
||||
}) => void;
|
||||
};
|
||||
instrumentation.onMcpToolCallSettled({
|
||||
serverName: 'child',
|
||||
toolName: 'lookup',
|
||||
success: true,
|
||||
});
|
||||
return makeGenerateResult({ toolCalls: [] });
|
||||
}),
|
||||
},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock('agent-1', user, request);
|
||||
expect(result.toolCalls).toEqual([
|
||||
expect.objectContaining({ tool: 'child_lookup', kind: 'mcp', mocked: true }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports tool progress before generation finishes and flushes writes before returning', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
let finish: (value: GenerateResult) => void = () => {};
|
||||
const generate = vi.fn(async () => {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId: 'send-1',
|
||||
toolName: 'Slack_Tool',
|
||||
args: { text: 'hello' },
|
||||
});
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolCallId: 'send-1',
|
||||
toolName: 'Slack_Tool',
|
||||
result: { sent: true },
|
||||
isError: false,
|
||||
});
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId: 'wait-1',
|
||||
toolName: 'wait_for_reply',
|
||||
args: {},
|
||||
});
|
||||
return await new Promise<GenerateResult>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
});
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
generate,
|
||||
close: vi.fn(),
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
},
|
||||
toolRegistry: {},
|
||||
});
|
||||
let settled = false;
|
||||
const running = buildService()
|
||||
.executeWithLlmMock('agent-1', user, request, 'Send a message', { onProgress })
|
||||
.then((value) => {
|
||||
settled = true;
|
||||
return value;
|
||||
});
|
||||
await vi.waitFor(() => expect(onProgress).toHaveBeenCalledTimes(3));
|
||||
expect(settled).toBe(false);
|
||||
expect(onProgress.mock.calls[2][0]).toEqual([
|
||||
expect.objectContaining({
|
||||
toolCallId: 'send-1',
|
||||
status: 'completed',
|
||||
output: { sent: true },
|
||||
}),
|
||||
expect.objectContaining({ toolCallId: 'wait-1', status: 'running' }),
|
||||
]);
|
||||
finish(makeGenerateResult());
|
||||
await running;
|
||||
});
|
||||
|
||||
it('publishes a mocked send while the tool waits for a reply', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
let finish!: (value: GenerateResult) => void;
|
||||
let mockRequest!: EvalLlmMockHandler;
|
||||
const send = async () =>
|
||||
await mockRequest(
|
||||
{ url: 'https://slack.com/api/chat.postMessage', method: 'POST', body: { text: 'Hello' } },
|
||||
{ name: 'Slack_Tool', type: 'n8n-nodes-base.slackTool' } as INode,
|
||||
);
|
||||
const generate = vi.fn(async () => {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId: 'send-1',
|
||||
toolName: 'Slack_Tool',
|
||||
args: { text: 'Hello' },
|
||||
});
|
||||
const instrumentation = reconstructFromAgentEntity.mock.calls[0][5] as {
|
||||
configureToolAdditionalData: (
|
||||
data: Record<string, unknown>,
|
||||
context: { toolName: string; toolKind: string },
|
||||
) => void;
|
||||
};
|
||||
const data: Record<string, unknown> = { credentialsHelper: { getDecrypted: vi.fn() } };
|
||||
instrumentation.configureToolAdditionalData(data, {
|
||||
toolName: 'Slack_Tool',
|
||||
toolKind: 'node',
|
||||
});
|
||||
mockRequest = data.evalLlmMockHandler as EvalLlmMockHandler;
|
||||
await send();
|
||||
return await new Promise<GenerateResult>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
});
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
generate,
|
||||
close: vi.fn(),
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
},
|
||||
});
|
||||
let settled = false;
|
||||
const running = buildService()
|
||||
.executeWithLlmMock('agent-1', user, request, 'Send then wait', { onProgress })
|
||||
.then((result) => {
|
||||
settled = true;
|
||||
return result;
|
||||
});
|
||||
try {
|
||||
await vi.waitFor(() => expect(onProgress).toHaveBeenCalledTimes(2));
|
||||
expect(settled).toBe(false);
|
||||
expect(onProgress).toHaveBeenLastCalledWith([
|
||||
expect.objectContaining({
|
||||
toolCallId: 'send-1',
|
||||
status: 'running',
|
||||
mocked: true,
|
||||
interceptedRequests: [
|
||||
expect.objectContaining({ requestBody: { text: 'Hello' }, mockResponse: { ok: true } }),
|
||||
],
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
finish(makeGenerateResult());
|
||||
await running;
|
||||
}
|
||||
const writes = onProgress.mock.calls.length;
|
||||
await send();
|
||||
await Promise.resolve();
|
||||
expect(onProgress).toHaveBeenCalledTimes(writes);
|
||||
});
|
||||
|
||||
it('serializes progress writes and waits for the last write without failing the run', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
let releaseWrite: () => void = () => {};
|
||||
const onProgress = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseWrite = resolve;
|
||||
}),
|
||||
)
|
||||
.mockRejectedValueOnce(new Error('progress unavailable'))
|
||||
.mockResolvedValue(undefined);
|
||||
const generate = vi.fn(async () => {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'Slack_Tool',
|
||||
args: {},
|
||||
});
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolCallId: 'call-1',
|
||||
toolName: 'Slack_Tool',
|
||||
result: { sent: true },
|
||||
isError: false,
|
||||
});
|
||||
return makeGenerateResult();
|
||||
});
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
generate,
|
||||
close: vi.fn(),
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
},
|
||||
toolRegistry: {},
|
||||
});
|
||||
let settled = false;
|
||||
const running = buildService()
|
||||
.executeWithLlmMock('agent-1', user, request, 'Send a message', { onProgress })
|
||||
.then((value) => {
|
||||
settled = true;
|
||||
return value;
|
||||
});
|
||||
await vi.waitFor(() => expect(onProgress).toHaveBeenCalledOnce());
|
||||
expect(settled).toBe(false);
|
||||
releaseWrite();
|
||||
expect((await running).success).toBe(true);
|
||||
expect(onProgress).toHaveBeenCalledTimes(3);
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId: 'late',
|
||||
toolName: 'Slack_Tool',
|
||||
args: {},
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(onProgress).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('flushes failed and interrupted tool progress before returning a generation error', async () => {
|
||||
const handlers = new Map<AgentEvent, AgentEventHandler>();
|
||||
const onProgress = vi.fn().mockResolvedValue(undefined);
|
||||
const close = vi.fn();
|
||||
reconstructFromAgentEntity.mockResolvedValue({
|
||||
agent: {
|
||||
generate: vi.fn(async () => {
|
||||
for (const toolCallId of ['failed', 'interrupted']) {
|
||||
handlers.get(AgentEvent.ToolExecutionStart)?.({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
toolCallId,
|
||||
toolName: 'Slack_Tool',
|
||||
args: {},
|
||||
});
|
||||
}
|
||||
handlers.get(AgentEvent.ToolExecutionEnd)?.({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
toolCallId: 'failed',
|
||||
toolName: 'Slack_Tool',
|
||||
result: 'Tool failed',
|
||||
isError: true,
|
||||
});
|
||||
throw new Error('Generation stopped');
|
||||
}),
|
||||
close,
|
||||
on: (event: AgentEvent, handler: AgentEventHandler) => handlers.set(event, handler),
|
||||
},
|
||||
toolRegistry: {},
|
||||
});
|
||||
const result = await buildService().executeWithLlmMock(
|
||||
'agent-1',
|
||||
user,
|
||||
request,
|
||||
'Send a message',
|
||||
{ onProgress },
|
||||
);
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.errors).toContain('Agent run failed: Generation stopped');
|
||||
expect(result.toolCalls).toEqual([
|
||||
expect.objectContaining({
|
||||
toolCallId: 'failed',
|
||||
status: 'error',
|
||||
error: 'tool_execution_failed',
|
||||
output: 'Tool failed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
toolCallId: 'interrupted',
|
||||
status: 'error',
|
||||
error: 'execution_stopped',
|
||||
}),
|
||||
]);
|
||||
expect(onProgress).toHaveBeenLastCalledWith([
|
||||
expect.objectContaining({
|
||||
toolCallId: 'failed',
|
||||
status: 'error',
|
||||
error: 'tool_execution_failed',
|
||||
output: 'Tool failed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
toolCallId: 'interrupted',
|
||||
status: 'error',
|
||||
error: 'execution_stopped',
|
||||
}),
|
||||
]);
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(onProgress.mock.invocationCallOrder.at(-1)).toBeLessThan(
|
||||
close.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
it('refuses queue mode upfront', async () => {
|
||||
const result = await buildService({ queueMode: true }).executeWithLlmMock(
|
||||
'agent-1',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import { isDeepStrictEqual } from 'node:util';
|
||||
import {
|
||||
AgentEvent,
|
||||
sanitizeToolName as sanitizeMcpToolName,
|
||||
type Agent as RuntimeAgent,
|
||||
type GenerateResult,
|
||||
|
|
@ -22,7 +24,7 @@ import { ExecutionsConfig } from '@n8n/config';
|
|||
import type { User } from '@n8n/db';
|
||||
import { Container, Service } from '@n8n/di';
|
||||
import type { EvalLlmMockHandler } from 'n8n-core';
|
||||
import { nodeNameToToolName } from 'n8n-workflow';
|
||||
import { deepCopy, nodeNameToToolName } from 'n8n-workflow';
|
||||
|
||||
import { CredentialsService } from '@/credentials/credentials.service';
|
||||
// Static agents-module imports are safe here: the ModuleRegistry gate decides
|
||||
|
|
@ -31,6 +33,7 @@ import { AgentRuntimeReconstructionService } from '@/modules/agents/agent-runtim
|
|||
import { hashAgentSandboxPrincipal } from '@/modules/agents/agent-sandbox-principal';
|
||||
import { AgentsSettingsService } from '@/modules/agents/agents-settings.service';
|
||||
import type { Agent as AgentEntity } from '@/modules/agents/entities/agent.entity';
|
||||
import type { AgentRunSource } from '@/modules/agents/utils/agent-published-snapshot';
|
||||
import { AgentSkillRefsService } from '@/modules/agents/agent-skill-refs.service';
|
||||
import { sanitizeToolName } from '@/modules/agents/json-config/agent-config-composition';
|
||||
import { fromAgentDocument, toAgentDocument } from '@/modules/agents/json-config/agent-document';
|
||||
|
|
@ -157,6 +160,19 @@ export class EvalAgentExecutionService {
|
|||
private readonly credentialsService: CredentialsService,
|
||||
) {}
|
||||
|
||||
/** Keep the root agent and resolved skills fixed while a batch is in flight. */
|
||||
async captureRunSource(agent: AgentEntity): Promise<AgentRunSource> {
|
||||
const detached = {
|
||||
...agent,
|
||||
schema: deepCopy(agent.schema),
|
||||
tools: deepCopy(agent.tools),
|
||||
skills: deepCopy(agent.skills),
|
||||
integrations: deepCopy(agent.integrations),
|
||||
};
|
||||
const skills = await Container.get(AgentSkillRefsService).readDraft(detached, {});
|
||||
return { agent: detached, ...deepCopy(skills) };
|
||||
}
|
||||
|
||||
async executeWithLlmMock(
|
||||
agentId: string,
|
||||
user: User,
|
||||
|
|
@ -166,6 +182,10 @@ export class EvalAgentExecutionService {
|
|||
// per-tool hints keep the tool mocks coherent), but the message the agent
|
||||
// receives is this input verbatim — how a fixed eval case is executed.
|
||||
caseInput?: string,
|
||||
hooks?: {
|
||||
onProgress?: (calls: InstanceAiEvalAgentToolCallRecord[]) => Promise<void>;
|
||||
source?: AgentRunSource;
|
||||
},
|
||||
): Promise<InstanceAiEvalAgentExecutionResult> {
|
||||
// Workflow-tool sub-executions carry a configureAdditionalData closure
|
||||
// that doesn't survive queue serialization — refuse upfront so tool
|
||||
|
|
@ -190,19 +210,33 @@ export class EvalAgentExecutionService {
|
|||
return this.errorResult(`Agent ${agentId} not found or not accessible`);
|
||||
}
|
||||
|
||||
const agentEntity = await Container.get(AgentRepository).findByIdAndProjectId(
|
||||
const source = hooks?.source;
|
||||
if (source && (source.agent.id !== agentId || source.agent.projectId !== projectId)) {
|
||||
return this.errorResult(`Agent ${agentId} not found or not accessible`);
|
||||
}
|
||||
// Keep the scoped existence check even when content comes from the batch snapshot.
|
||||
const currentAgent = await Container.get(AgentRepository).findByIdAndProjectId(
|
||||
agentId,
|
||||
projectId,
|
||||
);
|
||||
if (!agentEntity) {
|
||||
if (!currentAgent) {
|
||||
return this.errorResult(`Agent ${agentId} not found or not accessible`);
|
||||
}
|
||||
if (!agentEntity.schema) {
|
||||
const storedAgent = source?.agent ?? currentAgent;
|
||||
if (!storedAgent.schema) {
|
||||
return this.errorResult(`Agent ${agentId} has no JSON config to run.`);
|
||||
}
|
||||
|
||||
const agentSkillRefs = Container.get(AgentSkillRefsService);
|
||||
const skillRefs = await agentSkillRefs.refsForDraft(agentEntity, {});
|
||||
// Each case gets its own mutable copy; pruning must not change the batch source.
|
||||
const runSource = source ?? (await this.captureRunSource(storedAgent));
|
||||
const agentEntity = {
|
||||
...runSource.agent,
|
||||
schema: deepCopy(storedAgent.schema),
|
||||
tools: deepCopy(runSource.agent.tools),
|
||||
skills: deepCopy(runSource.agent.skills),
|
||||
integrations: deepCopy(runSource.agent.integrations),
|
||||
};
|
||||
const skillRefs = deepCopy(runSource.skillRefs);
|
||||
const { config, skippedFeatures } = pruneConfigForEval(
|
||||
toAgentDocument(agentEntity.schema, skillRefs),
|
||||
);
|
||||
|
|
@ -262,6 +296,9 @@ export class EvalAgentExecutionService {
|
|||
});
|
||||
|
||||
const toolLedger: ToolLedger = new Map();
|
||||
let onLedgerChange:
|
||||
| ((toolName: string, entries: InstanceAiEvalInterceptedRequest[]) => void)
|
||||
| undefined;
|
||||
const credentialHelpers: EvalMockedCredentialsHelper[] = [];
|
||||
const recorder = createAgentModelTurnRecorder(
|
||||
createAiProxyFetch(this.outboundHttp),
|
||||
|
|
@ -320,6 +357,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: call.args,
|
||||
mockResponse: call.result,
|
||||
});
|
||||
onLedgerChange?.(key, entries);
|
||||
};
|
||||
|
||||
// Fallback web_search mock — always created (sub-agents may enable web
|
||||
|
|
@ -343,6 +381,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: args,
|
||||
mockResponse: result,
|
||||
});
|
||||
onLedgerChange?.('web_search', entries);
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -360,7 +399,7 @@ export class EvalAgentExecutionService {
|
|||
{
|
||||
agent: agentEntity,
|
||||
skillRefs: pruned.skillRefs,
|
||||
skills: await agentSkillRefs.skillsForRefs(agentEntity, pruned.skillRefs, {}),
|
||||
skills: deepCopy(runSource.skills),
|
||||
},
|
||||
credentialProvider,
|
||||
'test',
|
||||
|
|
@ -397,6 +436,7 @@ export class EvalAgentExecutionService {
|
|||
mockHandler,
|
||||
toolContext.toolName,
|
||||
toolLedger,
|
||||
(entries) => onLedgerChange?.(toolContext.toolName, entries),
|
||||
);
|
||||
},
|
||||
},
|
||||
|
|
@ -408,6 +448,70 @@ export class EvalAgentExecutionService {
|
|||
return this.errorResult(`Failed to build agent runtime: ${message}`, seed, skippedFeatures);
|
||||
}
|
||||
|
||||
// Event handlers cannot await writes. Serialize snapshots and drain them before
|
||||
// returning, so progress cannot overwrite completion or a later attempt.
|
||||
const progressCalls = new Map<string, InstanceAiEvalAgentToolCallRecord>();
|
||||
let progressWrites = Promise.resolve();
|
||||
let progressClosed = false;
|
||||
const onProgress = hooks?.onProgress;
|
||||
const publishProgress = (
|
||||
writeProgress: (calls: InstanceAiEvalAgentToolCallRecord[]) => Promise<void>,
|
||||
) => {
|
||||
const calls = [...progressCalls.values()].map((call) => ({ ...call }));
|
||||
progressWrites = progressWrites
|
||||
.then(async () => await writeProgress(calls))
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn('[EvalAgentMock] Could not record tool progress', { error });
|
||||
});
|
||||
};
|
||||
if (onProgress) {
|
||||
// A send-and-wait tool records its mocked send before the tool ends.
|
||||
// Notify synchronously; the existing queue owns persistence and ordering.
|
||||
onLedgerChange = (toolName, interceptedRequests) => {
|
||||
let changed = false;
|
||||
for (const [id, call] of progressCalls) {
|
||||
if (call.tool !== toolName || call.status !== 'running') continue;
|
||||
progressCalls.set(id, {
|
||||
...call,
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests: [...interceptedRequests],
|
||||
});
|
||||
changed = true;
|
||||
}
|
||||
if (changed) publishProgress(onProgress);
|
||||
};
|
||||
agent.on(AgentEvent.ToolExecutionStart, (event) => {
|
||||
if (progressClosed || event.type !== AgentEvent.ToolExecutionStart) return;
|
||||
progressCalls.set(event.toolCallId, {
|
||||
toolCallId: event.toolCallId,
|
||||
tool: event.toolName,
|
||||
kind: toolSummaries.find((tool) => tool.name === event.toolName)?.kind ?? 'other',
|
||||
input: truncateRecordedValue(event.args),
|
||||
status: 'running',
|
||||
mocked: false,
|
||||
interceptedRequests: [],
|
||||
});
|
||||
publishProgress(onProgress);
|
||||
});
|
||||
agent.on(AgentEvent.ToolExecutionEnd, (event) => {
|
||||
if (progressClosed || event.type !== AgentEvent.ToolExecutionEnd) return;
|
||||
const previous = progressCalls.get(event.toolCallId);
|
||||
const interceptedRequests = toolLedger.get(event.toolName) ?? [];
|
||||
progressCalls.set(event.toolCallId, {
|
||||
...previous,
|
||||
toolCallId: event.toolCallId,
|
||||
tool: event.toolName,
|
||||
kind: previous?.kind ?? 'other',
|
||||
output: truncateRecordedValue(event.result),
|
||||
status: event.isError ? 'error' : 'completed',
|
||||
...(event.isError ? { error: 'tool_execution_failed' } : {}),
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests: [...interceptedRequests],
|
||||
});
|
||||
publishProgress(onProgress);
|
||||
});
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
// GenerateResult.toolCalls carry no toolCallId — approvals are attributed
|
||||
// by tool name, enough to flag auto-approval for the judge.
|
||||
|
|
@ -427,6 +531,69 @@ export class EvalAgentExecutionService {
|
|||
if (!segmentToolCalls.includes(entry)) segmentToolCalls.push(entry);
|
||||
}
|
||||
};
|
||||
const terminalToolCalls = (): InstanceAiEvalAgentToolCallRecord[] => {
|
||||
const kindByToolName = new Map(toolSummaries.map((tool) => [tool.name, tool.kind]));
|
||||
const kindForTool = (tool: string): InstanceAiEvalAgentToolCallRecord['kind'] =>
|
||||
kindByToolName.get(tool) ??
|
||||
(mcpServers.some((server) => tool.startsWith(sanitizeMcpToolName(`${server.name}_`))) ||
|
||||
(toolLedger.get(tool) ?? []).some((request) => request.nodeType?.startsWith('mcp:'))
|
||||
? 'mcp'
|
||||
: 'other');
|
||||
const reported = segmentToolCalls.map(
|
||||
(entry): InstanceAiEvalAgentToolCallRecord => ({
|
||||
tool: entry.tool,
|
||||
kind: kindForTool(entry.tool),
|
||||
input: truncateRecordedValue(entry.input),
|
||||
output: truncateRecordedValue(entry.output),
|
||||
...(entry.canceled ? { error: 'canceled' } : {}),
|
||||
mocked: false,
|
||||
interceptedRequests: [],
|
||||
}),
|
||||
);
|
||||
// Events own the IDs shown during progress. Consume matching SDK entries
|
||||
// once, so repeated calls remain distinct even when they finish out of order.
|
||||
const calls = [...progressCalls.values()].map((call) => {
|
||||
const index = reported.findIndex(
|
||||
(entry) =>
|
||||
call.status === 'completed' &&
|
||||
entry.tool === call.tool &&
|
||||
(call.input === undefined || isDeepStrictEqual(entry.input, call.input)) &&
|
||||
isDeepStrictEqual(entry.output, call.output),
|
||||
);
|
||||
const entry = index >= 0 ? reported.splice(index, 1)[0] : undefined;
|
||||
return {
|
||||
...entry,
|
||||
...call,
|
||||
kind: kindForTool(call.tool),
|
||||
...(call.status === 'running'
|
||||
? { status: 'error' as const, error: 'execution_stopped' }
|
||||
: {}),
|
||||
};
|
||||
});
|
||||
// Some cancelled calls never emit execution events. Keep those records too.
|
||||
calls.push(...reported);
|
||||
const recordedTools = new Set(calls.map((call) => call.tool));
|
||||
for (const [tool] of toolLedger) {
|
||||
if (recordedTools.has(tool)) continue;
|
||||
calls.push({
|
||||
tool,
|
||||
kind: kindForTool(tool),
|
||||
error:
|
||||
'Not attributed to a reported tool call (a delegated sub-agent call, or an errored call) — see interceptedRequests',
|
||||
mocked: false,
|
||||
interceptedRequests: [],
|
||||
});
|
||||
}
|
||||
return calls.map((call) => {
|
||||
const interceptedRequests = toolLedger.get(call.tool) ?? [];
|
||||
return {
|
||||
...call,
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests,
|
||||
...(autoApprovedToolNames.has(call.tool) ? { autoApproved: true } : {}),
|
||||
};
|
||||
});
|
||||
};
|
||||
const budgetSignal = AbortSignal.timeout(timeoutMs);
|
||||
try {
|
||||
result = await agent.generate(seed.openingMessage, {
|
||||
|
|
@ -470,15 +637,27 @@ export class EvalAgentExecutionService {
|
|||
`Agent run exceeded its ${seconds}s eval budget and was stopped`,
|
||||
seed,
|
||||
skippedFeatures,
|
||||
{ modelTurns: recorder.turns, toolLedger, credentialHelpers },
|
||||
{
|
||||
modelTurns: recorder.turns,
|
||||
credentialHelpers,
|
||||
toolCalls: terminalToolCalls(),
|
||||
},
|
||||
);
|
||||
}
|
||||
return this.errorResult(`Agent run failed: ${message}`, seed, skippedFeatures, {
|
||||
modelTurns: recorder.turns,
|
||||
toolLedger,
|
||||
credentialHelpers,
|
||||
toolCalls: terminalToolCalls(),
|
||||
});
|
||||
} finally {
|
||||
onLedgerChange = undefined;
|
||||
for (const [id, call] of progressCalls) {
|
||||
if (call.status === 'running')
|
||||
progressCalls.set(id, { ...call, status: 'error', error: 'execution_stopped' });
|
||||
}
|
||||
if (onProgress && progressCalls.size > 0) publishProgress(onProgress);
|
||||
progressClosed = true;
|
||||
await progressWrites;
|
||||
await recorder.flush();
|
||||
try {
|
||||
await agent.close();
|
||||
|
|
@ -495,47 +674,6 @@ export class EvalAgentExecutionService {
|
|||
);
|
||||
}
|
||||
|
||||
const kindByToolName = new Map(toolSummaries.map((tool) => [tool.name, tool.kind]));
|
||||
// MCP summaries are keyed by SERVER name; the model calls the client-side
|
||||
// prefixed `<server>_<tool>` names. Sub-agent-brought servers aren't in
|
||||
// this agent's summaries — their ledger entries carry an `mcp:` nodeType.
|
||||
const kindForTool = (tool: string): InstanceAiEvalAgentToolCallRecord['kind'] =>
|
||||
kindByToolName.get(tool) ??
|
||||
(mcpServers.some((server) => tool.startsWith(sanitizeMcpToolName(`${server.name}_`))) ||
|
||||
(toolLedger.get(tool) ?? []).some((request) => request.nodeType?.startsWith('mcp:'))
|
||||
? 'mcp'
|
||||
: 'other');
|
||||
const toolCalls = segmentToolCalls.map((entry): InstanceAiEvalAgentToolCallRecord => {
|
||||
const interceptedRequests = toolLedger.get(entry.tool) ?? [];
|
||||
return {
|
||||
tool: entry.tool,
|
||||
kind: kindForTool(entry.tool),
|
||||
input: truncateRecordedValue(entry.input),
|
||||
output: truncateRecordedValue(entry.output),
|
||||
...(entry.canceled ? { error: 'canceled' } : {}),
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests,
|
||||
...(autoApprovedToolNames.has(entry.tool) ? { autoApproved: true } : {}),
|
||||
};
|
||||
});
|
||||
// Ledger entries with no matching toolCalls record are either errored
|
||||
// calls (GenerateResult omits those) or a delegated sub-agent's calls
|
||||
// (never reported on the parent) — surface them without asserting
|
||||
// failure so the judge can weigh the intercepted traffic itself.
|
||||
const reportedTools = new Set(toolCalls.map((entry) => entry.tool));
|
||||
for (const [tool, interceptedRequests] of toolLedger) {
|
||||
if (reportedTools.has(tool)) continue;
|
||||
toolCalls.push({
|
||||
tool,
|
||||
kind: kindForTool(tool),
|
||||
error:
|
||||
'Not attributed to a reported tool call (a delegated sub-agent call, or an errored call) — see interceptedRequests',
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests,
|
||||
...(autoApprovedToolNames.has(tool) ? { autoApproved: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
runId: result.runId,
|
||||
success: errors.length === 0 && result.finishReason !== 'error',
|
||||
|
|
@ -543,7 +681,7 @@ export class EvalAgentExecutionService {
|
|||
finalText: extractFinalAssistantText(result),
|
||||
model: result.model,
|
||||
finishReason: result.finishReason,
|
||||
toolCalls,
|
||||
toolCalls: terminalToolCalls(),
|
||||
modelTurns: recorder.turns,
|
||||
...(result.usage
|
||||
? {
|
||||
|
|
@ -568,6 +706,7 @@ export class EvalAgentExecutionService {
|
|||
mockHandler: EvalLlmMockHandler,
|
||||
toolName: string,
|
||||
ledger: ToolLedger,
|
||||
onRecorded?: (entries: InstanceAiEvalInterceptedRequest[]) => void,
|
||||
): EvalLlmMockHandler {
|
||||
return async (requestOptions, node) => {
|
||||
const response = await mockHandler(requestOptions, node);
|
||||
|
|
@ -583,6 +722,7 @@ export class EvalAgentExecutionService {
|
|||
requestBody: requestOptions.body,
|
||||
mockResponse: snapshotLedgerBody(response?.body),
|
||||
});
|
||||
onRecorded?.(entries);
|
||||
this.logger.debug(
|
||||
`[EvalAgentMock] Intercepted ${requestOptions.method ?? 'GET'} ${requestOptions.url} from tool "${toolName}" (${node.type})`,
|
||||
);
|
||||
|
|
@ -596,8 +736,8 @@ export class EvalAgentExecutionService {
|
|||
skippedFeatures: InstanceAiEvalAgentSkippedFeature[] = [],
|
||||
partial?: {
|
||||
modelTurns: InstanceAiEvalAgentExecutionResult['modelTurns'];
|
||||
toolLedger: ToolLedger;
|
||||
credentialHelpers: EvalMockedCredentialsHelper[];
|
||||
toolCalls: InstanceAiEvalAgentToolCallRecord[];
|
||||
},
|
||||
): InstanceAiEvalAgentExecutionResult {
|
||||
this.logger.error(`[EvalAgentMock] ${message}`);
|
||||
|
|
@ -606,14 +746,7 @@ export class EvalAgentExecutionService {
|
|||
success: false,
|
||||
errors: [message],
|
||||
finalText: '',
|
||||
toolCalls: partial
|
||||
? [...partial.toolLedger.entries()].map(([tool, interceptedRequests]) => ({
|
||||
tool,
|
||||
kind: salvagedToolKind(interceptedRequests),
|
||||
mocked: interceptedRequests.length > 0,
|
||||
interceptedRequests,
|
||||
}))
|
||||
: [],
|
||||
toolCalls: partial?.toolCalls ?? [],
|
||||
modelTurns: partial?.modelTurns ?? [],
|
||||
seed: seed ?? { openingMessage: '', globalContext: '', toolHints: {}, warnings: [] },
|
||||
skippedFeatures,
|
||||
|
|
@ -688,22 +821,6 @@ export function pruneConfigForEval(original: AgentJsonConfig): {
|
|||
* tools, `sanitizeToolName` for workflow tools, the stored descriptor name
|
||||
* for custom tools.
|
||||
*/
|
||||
/**
|
||||
* Best-effort kind for a salvaged ledger entry on the failure path, where the
|
||||
* summary map is gone: MCP and web-search entries are self-describing via
|
||||
* nodeType; anything else with a nodeType came through the node/workflow
|
||||
* engine seam.
|
||||
*/
|
||||
function salvagedToolKind(
|
||||
interceptedRequests: InstanceAiEvalInterceptedRequest[],
|
||||
): InstanceAiEvalAgentToolCallRecord['kind'] {
|
||||
const nodeType = interceptedRequests.find((request) => request.nodeType)?.nodeType;
|
||||
if (!nodeType) return 'other';
|
||||
if (nodeType.startsWith('mcp:')) return 'mcp';
|
||||
if (nodeType === 'web-search:fallback') return 'other';
|
||||
return 'node';
|
||||
}
|
||||
|
||||
export function summarizeTools(
|
||||
config: AgentJsonConfig,
|
||||
customTools: NonNullable<AgentEntity['tools']>,
|
||||
|
|
|
|||
|
|
@ -9176,6 +9176,11 @@
|
|||
"agents.builder.agentEvals.checks.addCheckMore": "+{count} more",
|
||||
"agents.builder.agentEvals.checks.addCheckFewer": "Show fewer",
|
||||
"agents.builder.agentEvals.checks.status.passes": "Passes",
|
||||
"agents.builder.agentEvals.checks.mockedRequestRunning": "Mocked request sent; tool still running | Mocked requests sent; tool still running",
|
||||
"agents.builder.agentEvals.checks.mockedRequests": "Mocked requests",
|
||||
"agents.builder.agentEvals.checks.manageChecks": "Manage checks",
|
||||
"agents.builder.agentEvals.checks.savedForNextRun": "{count} added check is saved for the next run. | {count} added check is saved for the next run. | {count} added checks are saved for the next run.",
|
||||
"agents.builder.agentEvals.checks.waitingForResponse": "Waiting for the agent’s response…",
|
||||
"agents.builder.agentEvals.checks.status.running": "Running",
|
||||
"agents.builder.agentEvals.checks.status.neverRan": "Never ran",
|
||||
"agents.builder.agentEvals.checks.status.needsWork": "Needs work",
|
||||
|
|
|
|||
|
|
@ -28,7 +28,12 @@ const draft = (index: number): AgentEvalDraftCase => ({
|
|||
scenario: `Scenario ${index}`,
|
||||
});
|
||||
|
||||
type Props = { disabled?: boolean; busy?: boolean; caseSource?: typeof caseSource | null };
|
||||
type Props = {
|
||||
deferRun?: boolean;
|
||||
disabled?: boolean;
|
||||
busy?: boolean;
|
||||
caseSource?: typeof caseSource | null;
|
||||
};
|
||||
|
||||
const renderPanel = (
|
||||
props: Props = {},
|
||||
|
|
@ -63,6 +68,7 @@ const renderPanel = (
|
|||
caseSource: 'caseSource' in props ? props.caseSource : caseSource,
|
||||
disabled: props.disabled,
|
||||
busy: props.busy,
|
||||
deferRun: props.deferRun,
|
||||
},
|
||||
})({ pinia });
|
||||
|
||||
|
|
@ -74,6 +80,67 @@ describe('AgentEvalAddCheckPanel', () => {
|
|||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['drafting a rule', 'deferRun'],
|
||||
['saving a rule', 'busy'],
|
||||
['saving a prepared case', 'deferRun'],
|
||||
['saving a prepared case', 'busy'],
|
||||
] as const)('remembers a run that starts and finishes while %s (%s)', async (phase, runProp) => {
|
||||
const user = userEvent.setup();
|
||||
const { store, findAllByTestId, getByTestId, rerender, emitted } = renderPanel();
|
||||
await findAllByTestId('agent-eval-add-check-prepared-add');
|
||||
let finish!: () => void;
|
||||
const pending = new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
if (phase === 'drafting a rule') {
|
||||
vi.mocked(store.generateDraftCases).mockImplementationOnce(async () => {
|
||||
await pending;
|
||||
return { cases: [draft(9)] };
|
||||
});
|
||||
} else {
|
||||
vi.mocked(store.createCase).mockImplementationOnce(async () => {
|
||||
await pending;
|
||||
return { rowId: 9, input: 'new message', whatToCheck: 'new rule' };
|
||||
});
|
||||
}
|
||||
if (phase === 'saving a prepared case') {
|
||||
await user.click((await findAllByTestId('agent-eval-add-check-prepared-add'))[0]);
|
||||
} else {
|
||||
await user.type(getByTestId('agent-eval-add-check-rule-input'), 'Be polite{Enter}');
|
||||
}
|
||||
|
||||
await rerender({ [runProp]: true });
|
||||
await rerender({ [runProp]: false });
|
||||
finish();
|
||||
await waitFor(() => expect(emitted('added')).toEqual([[true]]));
|
||||
|
||||
// The next addition has no overlap with that finished run.
|
||||
await user.click((await findAllByTestId('agent-eval-add-check-prepared-add'))[0]);
|
||||
await waitFor(() => expect(emitted('added')).toEqual([[true], [false]]));
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'defers additions if a run is active at either end of the save (initial: %s)',
|
||||
async (initiallyRunning) => {
|
||||
const user = userEvent.setup();
|
||||
const { store, findAllByTestId, rerender, emitted } = renderPanel({
|
||||
deferRun: initiallyRunning,
|
||||
});
|
||||
let save!: () => void;
|
||||
vi.mocked(store.createCase).mockImplementation(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
save = resolve;
|
||||
});
|
||||
return { rowId: 9, input: 'new message', whatToCheck: 'new rule' };
|
||||
});
|
||||
await user.click((await findAllByTestId('agent-eval-add-check-prepared-add'))[0]);
|
||||
await rerender({ deferRun: !initiallyRunning });
|
||||
save();
|
||||
await waitFor(() => expect(emitted('added')).toEqual([[true]]));
|
||||
},
|
||||
);
|
||||
|
||||
describe('the prepared cases', () => {
|
||||
it('drafts a batch of ten without saving anything when it opens', async () => {
|
||||
const { store, findAllByTestId } = renderPanel();
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { configure, within } from '@testing-library/vue';
|
|||
import { createTestingPinia } from '@pinia/testing';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ref } from 'vue';
|
||||
import { nextTick, ref } from 'vue';
|
||||
|
||||
import { ResponseError } from '@n8n/rest-api-client';
|
||||
|
||||
|
|
@ -38,6 +38,7 @@ vi.mock('../components/AgentEvalTryRow.vue', () => ({
|
|||
status: {},
|
||||
input: {},
|
||||
output: {},
|
||||
toolCalls: {},
|
||||
label: {},
|
||||
testId: {},
|
||||
disabled: { type: Boolean },
|
||||
|
|
@ -61,6 +62,8 @@ vi.mock('../components/AgentEvalTryRow.vue', () => ({
|
|||
template: `<div
|
||||
:data-testid="testId"
|
||||
:data-status="status"
|
||||
:data-tool-status="toolCalls?.[0]?.state"
|
||||
:data-tool-id="toolCalls?.[0]?.toolCallId"
|
||||
:data-disabled="disabled"
|
||||
:data-hide-revise="hideRevise"
|
||||
:data-focused="focused"
|
||||
|
|
@ -83,8 +86,15 @@ const { addCheckPanelMounts } = vi.hoisted(() => ({ addCheckPanelMounts: { count
|
|||
vi.mock('../components/AgentEvalAddCheckPanel.vue', () => ({
|
||||
default: {
|
||||
name: 'AgentEvalAddCheckPanel',
|
||||
setup() {
|
||||
setup(props: { deferRun: boolean }) {
|
||||
addCheckPanelMounts.count++;
|
||||
const deferredAtStart = ref(false);
|
||||
return {
|
||||
deferredAtStart,
|
||||
startAdding: () => {
|
||||
deferredAtStart.value = props.deferRun;
|
||||
},
|
||||
};
|
||||
},
|
||||
props: {
|
||||
projectId: {},
|
||||
|
|
@ -92,16 +102,20 @@ vi.mock('../components/AgentEvalAddCheckPanel.vue', () => ({
|
|||
caseSource: {},
|
||||
disabled: { type: Boolean },
|
||||
busy: { type: Boolean },
|
||||
deferRun: { type: Boolean },
|
||||
},
|
||||
emits: ['close', 'added'],
|
||||
template: `<div
|
||||
data-testid="add-check-panel-stub"
|
||||
:data-busy="busy"
|
||||
:data-defer-run="deferRun"
|
||||
:data-disabled="disabled"
|
||||
:data-dataset="caseSource && caseSource.datasetId"
|
||||
>
|
||||
<button @click="$emit('close')">close panel</button>
|
||||
<button @click="$emit('added')">added a check</button>
|
||||
<button @click="$emit('added', deferRun)">added a check</button>
|
||||
<button @click="startAdding">start adding a check</button>
|
||||
<button @click="$emit('added', deferredAtStart)">added after run finished</button>
|
||||
</div>`,
|
||||
},
|
||||
}));
|
||||
|
|
@ -311,6 +325,11 @@ describe('AgentEvalChecksPanel', () => {
|
|||
|
||||
live.value = loadedReview([result('n1', 'new')]);
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(getByTestId('agent-eval-check-n1')).toHaveAttribute('data-disabled', 'true'),
|
||||
);
|
||||
live.value = loadedReview([result('n1', 'success')]);
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(getByTestId('agent-eval-check-n1')).toHaveAttribute('data-disabled', 'false'),
|
||||
);
|
||||
|
|
@ -719,6 +738,30 @@ describe('AgentEvalChecksPanel', () => {
|
|||
expect(store.applySuggestions).toHaveBeenCalledWith('project-1', 'agent-1', ['c1', 'c3']);
|
||||
});
|
||||
|
||||
it('applies settled suggestions during a batch and leaves queued and running checks out', async () => {
|
||||
const user = userEvent.setup();
|
||||
const order: string[] = [];
|
||||
const { getByTestId, store } = render(
|
||||
{
|
||||
results: [
|
||||
failedWith('c1', 'Be brief'),
|
||||
{ ...failedWith('c2', 'Wait'), status: 'running' },
|
||||
{ ...failedWith('c3', 'Still queued'), status: 'new' },
|
||||
],
|
||||
},
|
||||
true,
|
||||
builderWrite(order),
|
||||
);
|
||||
vi.mocked(store.applySuggestions).mockImplementation(async () => {
|
||||
order.push('apply');
|
||||
return { configHash: 'h', results: [] };
|
||||
});
|
||||
expect(getByTestId(BUTTON)).toBeEnabled();
|
||||
await user.click(getByTestId(BUTTON));
|
||||
await vi.waitFor(() => expect(order).toEqual(['lock', 'apply', 'unlock']));
|
||||
expect(store.applySuggestions).toHaveBeenCalledWith('project-1', 'agent-1', ['c1']);
|
||||
});
|
||||
|
||||
it('sends a long list as successive requests of at most one request’s worth', async () => {
|
||||
const user = userEvent.setup();
|
||||
const many = Array.from({ length: MAX_APPLY_SUGGESTIONS + 2 }, (_, i) =>
|
||||
|
|
@ -767,6 +810,10 @@ describe('AgentEvalChecksPanel', () => {
|
|||
// A click that still reaches the composable is ignored, not sent as a second write.
|
||||
await user.click(within(getByTestId('agent-eval-check-c2')).getByText('apply suggestion'));
|
||||
expect(store.applySuggestions).toHaveBeenCalledTimes(1);
|
||||
// A queued button event must also respect the active write.
|
||||
getByTestId(BUTTON).dispatchEvent(new MouseEvent('click', { bubbles: true }));
|
||||
await nextTick();
|
||||
expect(store.applySuggestions).toHaveBeenCalledTimes(1);
|
||||
|
||||
finish({ configHash: 'h', results: [] });
|
||||
});
|
||||
|
|
@ -849,7 +896,8 @@ describe('AgentEvalChecksPanel', () => {
|
|||
loading: false,
|
||||
loadingMore: false,
|
||||
});
|
||||
vi.mocked(store.isRunInFlight).mockReturnValue(options.inFlight ?? false);
|
||||
const inFlight = ref(options.inFlight ?? false);
|
||||
vi.mocked(store.isRunInFlight).mockImplementation(() => inFlight.value);
|
||||
vi.mocked(store.isStartingRun).mockReturnValue(false);
|
||||
vi.mocked(store.getDatasets).mockReturnValue([
|
||||
{ ...dataset, columnMapping: options.columnMapping ?? dataset.columnMapping },
|
||||
|
|
@ -860,6 +908,7 @@ describe('AgentEvalChecksPanel', () => {
|
|||
props: { disabled: options.disabled, rerunning: options.rerunning },
|
||||
}),
|
||||
store,
|
||||
inFlight,
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -954,6 +1003,35 @@ describe('AgentEvalChecksPanel', () => {
|
|||
expect(addCheckPanelMounts.count).toBe(1);
|
||||
});
|
||||
|
||||
it('saves additions for the next run while keeping the active batch visible', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, getByText, emitted } = renderWithDataset({ inFlight: true });
|
||||
await user.click(getByTestId('agent-eval-checks-add-check'));
|
||||
expect(getByTestId('add-check-panel-stub')).toHaveAttribute('data-busy', 'false');
|
||||
await user.click(getByText('added a check'));
|
||||
expect(emitted('rerun')).toBeUndefined();
|
||||
expect(getByTestId('agent-eval-check-c1')).toBeVisible();
|
||||
expect(getByTestId('agent-eval-checks-saved-next-run')).toHaveTextContent(
|
||||
'1 added check is saved for the next run.',
|
||||
);
|
||||
await user.click(getByText('close panel'));
|
||||
expect(getByTestId('agent-eval-checks-saved-next-run')).toBeVisible();
|
||||
});
|
||||
|
||||
it('keeps an addition deferred after the original run finishes', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, getByText, emitted, inFlight } = renderWithDataset({ inFlight: true });
|
||||
await user.click(getByTestId('agent-eval-checks-add-check'));
|
||||
expect(getByTestId('add-check-panel-stub')).toHaveAttribute('data-defer-run', 'true');
|
||||
await user.click(getByText('start adding a check'));
|
||||
inFlight.value = false;
|
||||
await nextTick();
|
||||
expect(getByTestId('add-check-panel-stub')).toHaveAttribute('data-defer-run', 'false');
|
||||
await user.click(getByText('added after run finished'));
|
||||
expect(emitted('rerun')).toBeUndefined();
|
||||
expect(getByTestId('agent-eval-checks-saved-next-run')).toBeVisible();
|
||||
});
|
||||
|
||||
it('runs the checks again once a check was added, since a run only holds the cases it started with', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, getByText, emitted } = renderWithDataset();
|
||||
|
|
@ -1025,17 +1103,17 @@ describe('AgentEvalChecksPanel', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a rerun is starting', { rerunning: true }],
|
||||
['a run is in flight', { inFlight: true }],
|
||||
])('tells the panel it is busy while %s', async (_label, options) => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId } = renderWithDataset(options);
|
||||
it.each([['a rerun is starting', { rerunning: true }]])(
|
||||
'tells the panel it is busy while %s',
|
||||
async (_label, options) => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId } = renderWithDataset(options);
|
||||
|
||||
await user.click(getByTestId('agent-eval-checks-add-check'));
|
||||
await user.click(getByTestId('agent-eval-checks-add-check'));
|
||||
|
||||
expect(getByTestId('add-check-panel-stub')).toHaveAttribute('data-busy', 'true');
|
||||
});
|
||||
expect(getByTestId('add-check-panel-stub')).toHaveAttribute('data-busy', 'true');
|
||||
},
|
||||
);
|
||||
|
||||
it('disables the button and the panel for a read-only viewer', async () => {
|
||||
const { getByTestId } = renderWithDataset({ disabled: true });
|
||||
|
|
@ -1141,6 +1219,54 @@ describe('AgentEvalChecksPanel', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each(['new', 'running'] as const)(
|
||||
'blocks mutations on a %s check during a batch',
|
||||
async (status) => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, store } = render({ results: [result('c1', status)] }, true);
|
||||
const row = getByTestId('agent-eval-check-c1');
|
||||
expect(row).toHaveAttribute('data-disabled', 'true');
|
||||
for (const action of [
|
||||
'actually fine',
|
||||
'run check',
|
||||
'save rule',
|
||||
'delete check',
|
||||
'apply suggestion',
|
||||
]) {
|
||||
// Guard handlers as well as controls against an event already in flight.
|
||||
await user.click(within(row).getByText(action));
|
||||
}
|
||||
expect(store.acceptResult).not.toHaveBeenCalled();
|
||||
expect(store.rerunResult).not.toHaveBeenCalled();
|
||||
expect(store.updateCaseRule).not.toHaveBeenCalled();
|
||||
expect(store.deleteCase).not.toHaveBeenCalled();
|
||||
expect(store.deleteResult).not.toHaveBeenCalled();
|
||||
expect(store.applySuggestions).not.toHaveBeenCalled();
|
||||
expect(showError).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('passes current tool progress to a running check without an answer', () => {
|
||||
const { getByTestId } = render(
|
||||
{
|
||||
results: [
|
||||
{
|
||||
...result('c1', 'running'),
|
||||
output: null,
|
||||
toolCalls: {
|
||||
calls: [{ tool: 'wait_for_reply', toolCallId: 'call-1', status: 'running' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
true,
|
||||
);
|
||||
const row = getByTestId('agent-eval-check-c1');
|
||||
expect(row).toHaveAttribute('data-status', 'waiting');
|
||||
expect(row).toHaveAttribute('data-tool-status', 'running');
|
||||
expect(row).toHaveAttribute('data-tool-id', 'call-1');
|
||||
});
|
||||
|
||||
it('disables "Run all checks" while a run is already in flight', () => {
|
||||
const { getByTestId } = render({ results: [result('c1', 'running')] }, true);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
import { createTestingPinia } from '@pinia/testing';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
|
||||
import { createComponentRenderer } from '@/__tests__/render';
|
||||
import { TOOL_CALL_STATE } from '@/features/ai/shared/agentsChat/constants';
|
||||
import AgentEvalToolCalls from '../components/AgentEvalToolCalls.vue';
|
||||
import { toDisplayToolCalls } from '../utils/agent-eval-tool-calls';
|
||||
|
||||
const renderComponent = createComponentRenderer(AgentEvalToolCalls, {
|
||||
global: { plugins: [createTestingPinia()] },
|
||||
});
|
||||
|
||||
const request = {
|
||||
url: 'https://slack.com/api/chat.postMessage',
|
||||
method: 'POST',
|
||||
nodeType: 'n8n-nodes-base.slack',
|
||||
requestBody: { channel: 'support', text: 'Your order is ready.' },
|
||||
mockResponse: { ok: true, ts: 'mock-message' },
|
||||
};
|
||||
|
||||
function calls(withRequest: boolean, grouped = false) {
|
||||
return toDisplayToolCalls({
|
||||
calls: [
|
||||
{
|
||||
tool: 'send_and_wait',
|
||||
toolCallId: 'wait-1',
|
||||
status: 'running',
|
||||
input: { channel: 'support' },
|
||||
mocked: withRequest,
|
||||
interceptedRequests: withRequest ? [request] : [],
|
||||
},
|
||||
...(grouped ? [{ tool: 'lookup', toolCallId: 'lookup-1', output: 'Order found' }] : []),
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
it.each([false, true])(
|
||||
'shows intercepted requests before completion (grouped: %s)',
|
||||
async (grouped) => {
|
||||
const user = userEvent.setup();
|
||||
const { getByText, queryByText, rerender } = renderComponent({
|
||||
props: { toolCalls: calls(false, grouped) },
|
||||
});
|
||||
if (grouped) await user.click(getByText('2 tool calls'));
|
||||
await user.click(getByText('Send and wait'));
|
||||
expect(getByText('Input')).toBeInTheDocument();
|
||||
|
||||
await rerender({ toolCalls: calls(true, grouped) });
|
||||
|
||||
expect(getByText(/Mocked request sent; tool still running/)).toBeInTheDocument();
|
||||
// The disclosure stays open as live progress arrives.
|
||||
expect(getByText('Mocked requests')).toBeInTheDocument();
|
||||
expect(getByText(/https:\/\/slack.com\/api\/chat.postMessage/)).toHaveTextContent(
|
||||
'Your order is ready.',
|
||||
);
|
||||
expect(getByText(/mock-message/)).toHaveTextContent('POST');
|
||||
expect(queryByText('Output')).not.toBeInTheDocument();
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps legacy calls free of mock claims', () => {
|
||||
const { queryByText } = renderComponent({
|
||||
props: { toolCalls: toDisplayToolCalls({ calls: [{ tool: 'lookup', output: 'Found' }] }) },
|
||||
});
|
||||
expect(queryByText(/Mocked/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'keeps the tool disclosure open when the result completes (grouped: %s)',
|
||||
async (grouped) => {
|
||||
const user = userEvent.setup();
|
||||
const { getByText, getByRole, rerender } = renderComponent({
|
||||
props: { toolCalls: calls(true, grouped) },
|
||||
});
|
||||
if (grouped) await user.click(getByText('2 tool calls'));
|
||||
const trigger = getByRole('button', { name: /Send and wait/ });
|
||||
await user.click(trigger);
|
||||
const contentId = trigger.getAttribute('aria-controls');
|
||||
expect(getByText('Input')).toBeInTheDocument();
|
||||
await rerender({
|
||||
toolCalls: calls(true, grouped).map((call) => ({
|
||||
...call,
|
||||
state: TOOL_CALL_STATE.DONE,
|
||||
output: { ok: true },
|
||||
})),
|
||||
});
|
||||
expect(getByRole('button', { name: /Send and wait/ })).toHaveAttribute(
|
||||
'aria-controls',
|
||||
contentId,
|
||||
);
|
||||
expect(getByRole('button', { name: /Send and wait/ })).toHaveAttribute('aria-expanded', 'true');
|
||||
expect(getByText('Input')).toBeInTheDocument();
|
||||
expect(getByText('Output')).toBeInTheDocument();
|
||||
expect(getByText('Mocked requests')).toBeInTheDocument();
|
||||
},
|
||||
);
|
||||
|
||||
it('opens recorded mock details when the tool has no input or output', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByText } = renderComponent({
|
||||
props: {
|
||||
toolCalls: toDisplayToolCalls({
|
||||
calls: [
|
||||
{
|
||||
tool: 'send_and_wait',
|
||||
status: 'running',
|
||||
mocked: true,
|
||||
interceptedRequests: [request],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
});
|
||||
await user.click(getByText(/Send and wait/));
|
||||
expect(getByText('Mocked requests')).toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -30,6 +30,17 @@ describe('AgentEvalTryRow', () => {
|
|||
});
|
||||
|
||||
describe('view="complete"', () => {
|
||||
it('announces both disclosure states on the complete row', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId } = renderComponent({ props: { testId: 'row' } });
|
||||
const toggle = getByTestId('row-toggle');
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'false');
|
||||
await user.click(toggle);
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'true');
|
||||
await user.click(toggle);
|
||||
expect(toggle).toHaveAttribute('aria-expanded', 'false');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['pass', 'Passes'],
|
||||
['strong', 'Passes'],
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { createPinia, setActivePinia } from 'pinia';
|
||||
import { ResponseError } from '@n8n/rest-api-client';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { useAgentEvalsStore } from '../agentEvals.store';
|
||||
|
|
@ -157,6 +158,9 @@ const mockRun = (options: {
|
|||
};
|
||||
|
||||
describe('useAgentEvalsStore', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia());
|
||||
vi.clearAllMocks();
|
||||
|
|
@ -1368,6 +1372,265 @@ describe('useAgentEvalsStore', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe.each(['rerun', 'suggestion'] as const)('%s progress', (operation) => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
it('does not let an expired confirmation read undo a newer rerun', async () => {
|
||||
vi.useFakeTimers();
|
||||
const old = result('c1');
|
||||
mockRun({ results: [old], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
const request = operation === 'rerun' ? rerunResult : applySuggestions;
|
||||
request.mockRejectedValueOnce(new ResponseError('Forbidden', { httpStatusCode: 403 }));
|
||||
let finishOldRead!: (value: unknown) => void;
|
||||
getRunDetail.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finishOldRead = resolve;
|
||||
}),
|
||||
);
|
||||
const first =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
const rejected = expect(first).rejects.toThrow('Forbidden');
|
||||
await vi.advanceTimersByTimeAsync(30000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(old);
|
||||
let finishNew!: (value: unknown) => void;
|
||||
request.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finishNew = resolve;
|
||||
}),
|
||||
);
|
||||
const second =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
finishOldRead(runDetail([old], 1));
|
||||
await rejected;
|
||||
expect(store.getReview(RUN_ID).results[0]).toMatchObject({ status: 'running', output: null });
|
||||
const progress = {
|
||||
...old,
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
status: 'running' as const,
|
||||
output: null,
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([progress], 1));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(progress);
|
||||
const final = { ...progress, status: 'success' as const };
|
||||
finishNew(operation === 'rerun' ? final : { configHash: 'new', results: [final] });
|
||||
await second;
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(final);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps a confirmed attempt after the confirmation window and later polling errors', async () => {
|
||||
vi.useFakeTimers();
|
||||
const old = result('c1');
|
||||
mockRun({ results: [old], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
const progress = {
|
||||
...old,
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
status: 'running' as const,
|
||||
output: null,
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([progress], 1));
|
||||
(operation === 'rerun' ? rerunResult : applySuggestions).mockRejectedValue(
|
||||
new Error('timeout'),
|
||||
);
|
||||
const pending =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
await expect(pending).rejects.toThrow('timeout');
|
||||
await vi.advanceTimersByTimeAsync(35000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(progress);
|
||||
getRunDetail.mockRejectedValue(new Error('offline'));
|
||||
await vi.advanceTimersByTimeAsync(15000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(progress);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('restores the previous result immediately after an authorization rejection', async () => {
|
||||
vi.useFakeTimers();
|
||||
const old = result('c1');
|
||||
mockRun({ results: [old], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
(operation === 'rerun' ? rerunResult : applySuggestions).mockRejectedValue(
|
||||
new ResponseError('Forbidden', { httpStatusCode: 403 }),
|
||||
);
|
||||
const pending =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
await expect(pending).rejects.toThrow('Forbidden');
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(old);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
'confirms a lost request before restoring an attempt not yet observed (read fails: %s)',
|
||||
async (readFails) => {
|
||||
vi.useFakeTimers();
|
||||
const visibility = vi.spyOn(document, 'hidden', 'get').mockReturnValue(true);
|
||||
const old = result('c1');
|
||||
mockRun({ results: [old], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
(operation === 'rerun' ? rerunResult : applySuggestions).mockRejectedValue(
|
||||
new Error('timeout'),
|
||||
);
|
||||
if (readFails) getRunDetail.mockRejectedValueOnce(new Error('offline'));
|
||||
const pending =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
await expect(pending).rejects.toThrow('timeout');
|
||||
expect(store.getReview(RUN_ID).results[0]).toMatchObject({
|
||||
status: 'running',
|
||||
output: null,
|
||||
});
|
||||
// Whole-run reads must not erase the unconfirmed execution boundary either.
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
expect(store.getReview(RUN_ID).results[0].status).toBe('running');
|
||||
const progress = {
|
||||
...old,
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
status: 'running' as const,
|
||||
output: null,
|
||||
toolCalls: { calls: [{ tool: 'wait_for_reply', status: 'running' }] },
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([progress], 1));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(progress);
|
||||
visibility.mockReturnValue(false);
|
||||
const final = {
|
||||
...progress,
|
||||
status: 'success' as const,
|
||||
output: { finalText: 'Completed' },
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([final], 1));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(final);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, true])(
|
||||
'keeps observed server progress after a request fails (recovery read fails: %s)',
|
||||
async (readFails) => {
|
||||
vi.useFakeTimers();
|
||||
const old = result('c1');
|
||||
mockRun({ results: [old], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let reject!: (error: Error) => void;
|
||||
(operation === 'rerun' ? rerunResult : applySuggestions).mockImplementation(
|
||||
async () =>
|
||||
await new Promise((_, fail) => {
|
||||
reject = fail;
|
||||
}),
|
||||
);
|
||||
const pending =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
const progress = {
|
||||
...old,
|
||||
status: 'running' as const,
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
output: null,
|
||||
toolCalls: { calls: [{ tool: 'wait_for_reply', status: 'running' }] },
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([progress], 1));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
if (readFails) getRunDetail.mockRejectedValueOnce(new Error('offline'));
|
||||
const rejected = expect(pending).rejects.toThrow('timeout');
|
||||
reject(new Error('timeout'));
|
||||
await rejected;
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(progress);
|
||||
const final = {
|
||||
...progress,
|
||||
status: 'success' as const,
|
||||
output: { finalText: 'Finished later' },
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([final], 1));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(final);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
},
|
||||
);
|
||||
it('clears previous output and polls the new attempt on a completed run, ignoring stale reads', async () => {
|
||||
vi.useFakeTimers();
|
||||
const old = { ...result('c1'), toolCalls: { calls: [{ tool: 'old_tool' }] } };
|
||||
mockRun({ results: [old, result('c2')], count: 2, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let finish!: (value: unknown) => void;
|
||||
(operation === 'rerun' ? rerunResult : applySuggestions).mockImplementation(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const pending =
|
||||
operation === 'rerun'
|
||||
? store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')
|
||||
: store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1']);
|
||||
expect(store.getReview(RUN_ID).results[0]).toMatchObject({
|
||||
status: 'running',
|
||||
output: null,
|
||||
toolCalls: null,
|
||||
verdict: null,
|
||||
});
|
||||
// A batch refresh before the server claims the attempt must preserve the
|
||||
// optimistic row while still updating siblings.
|
||||
const sibling = { ...result('c2'), output: { finalText: 'Fresh sibling' } };
|
||||
getRunDetail.mockResolvedValue(runDetail([old, sibling], 2));
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
expect(store.getReview(RUN_ID).results[0].output).toBeNull();
|
||||
expect(store.getReview(RUN_ID).results[1]).toEqual(sibling);
|
||||
const progress = {
|
||||
...old,
|
||||
status: 'running' as const,
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
output: null,
|
||||
toolCalls: { calls: [{ tool: 'wait_for_reply', status: 'running' }] },
|
||||
};
|
||||
getRunDetail.mockResolvedValue(runDetail([progress], 2));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0].toolCalls).toEqual(progress.toolCalls);
|
||||
// A read already in flight cannot overwrite the final request response.
|
||||
let staleRead!: (value: unknown) => void;
|
||||
getRunDetail.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
staleRead = resolve;
|
||||
}),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
const final = {
|
||||
...progress,
|
||||
status: 'success' as const,
|
||||
output: { finalText: 'New response' },
|
||||
};
|
||||
finish(operation === 'rerun' ? final : { configHash: 'new', results: [final] });
|
||||
await pending;
|
||||
staleRead(runDetail([progress], 2));
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(final);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applySuggestions', () => {
|
||||
const failing = (id: string): AgentEvalResultRecord => ({
|
||||
...result(id),
|
||||
|
|
@ -1431,16 +1694,18 @@ describe('useAgentEvalsStore', () => {
|
|||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
applySuggestions.mockRejectedValue(new Error('conflict'));
|
||||
getRunDetail.mockResolvedValue(runDetail([failing('c1')], 1));
|
||||
const fresh = { ...failing('c1'), runAt: '2026-01-01T01:00:00.000Z' };
|
||||
getRunDetail.mockResolvedValue(runDetail([fresh], 1));
|
||||
|
||||
await expect(store.applySuggestions(PROJECT_ID, AGENT_ID, ['c1'])).rejects.toThrow(
|
||||
'conflict',
|
||||
);
|
||||
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(failing('c1'));
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(fresh);
|
||||
});
|
||||
|
||||
it('restores the previous rows when reading them back also fails', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [failing('c1')], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
|
|
@ -1451,11 +1716,85 @@ describe('useAgentEvalsStore', () => {
|
|||
'conflict',
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15000);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(failing('c1'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('rerunResult', () => {
|
||||
it('ignores a timer task already queued when a rerun finishes', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [result('c1')], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let finish!: (value: AgentEvalResultRecord) => void;
|
||||
rerunResult.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const timer = vi.spyOn(globalThis, 'setTimeout');
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
|
||||
const queued = timer.mock.calls.find(([, delay]) => delay === 5000)?.[0];
|
||||
expect(typeof queued).toBe('function');
|
||||
finish(result('c1'));
|
||||
await pending;
|
||||
getRunDetail.mockClear();
|
||||
if (typeof queued !== 'function') throw new Error('Missing progress timer');
|
||||
await queued();
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(result('c1'));
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
timer.mockRestore();
|
||||
});
|
||||
|
||||
it.each([false, true])('handles an uncached rerun (request fails: %s)', async (fails) => {
|
||||
const store = useAgentEvalsStore();
|
||||
if (fails) rerunResult.mockRejectedValueOnce(new Error('Unavailable'));
|
||||
else rerunResult.mockResolvedValueOnce(result('uncached'));
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'uncached');
|
||||
if (fails) await expect(pending).rejects.toThrow('Unavailable');
|
||||
else expect(await pending).toEqual(result('uncached'));
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('preserves a newer attempt when a later refresh returns an older runAt', async () => {
|
||||
const newer = { ...result('c1'), runAt: '2026-01-01T01:00:00.000Z' };
|
||||
mockRun({ results: [newer], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
getRunDetail.mockResolvedValueOnce(runDetail([result('c1')], 1));
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
expect(store.getReview(RUN_ID).results[0]).toEqual(newer);
|
||||
});
|
||||
|
||||
it('pauses normal progress reads while the document is hidden', async () => {
|
||||
vi.useFakeTimers();
|
||||
const visibility = vi.spyOn(document, 'hidden', 'get').mockReturnValue(true);
|
||||
mockRun({ results: [result('c1')], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
let finish!: (value: AgentEvalResultRecord) => void;
|
||||
rerunResult.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
|
||||
getRunDetail.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(getRunDetail).not.toHaveBeenCalled();
|
||||
visibility.mockReturnValue(false);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(getRunDetail).toHaveBeenCalledOnce();
|
||||
finish(result('c1'));
|
||||
await pending;
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
visibility.mockRestore();
|
||||
});
|
||||
|
||||
it('patches the cached result to running before the request lands, then to the real response', async () => {
|
||||
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
|
|
@ -1480,6 +1819,7 @@ describe('useAgentEvalsStore', () => {
|
|||
});
|
||||
|
||||
it('reverts the optimistic patch when the rerun fails', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
|
|
@ -1487,6 +1827,10 @@ describe('useAgentEvalsStore', () => {
|
|||
|
||||
await expect(store.rerunResult(PROJECT_ID, AGENT_ID, 'c1')).rejects.toThrow('timeout');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25000);
|
||||
expect(store.getReview(RUN_ID).results[0].status).toBe('running');
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(store.getReview(RUN_ID).results[0].status).toBe('error');
|
||||
});
|
||||
|
||||
|
|
@ -1501,7 +1845,9 @@ describe('useAgentEvalsStore', () => {
|
|||
);
|
||||
const pending = store.rerunResult(PROJECT_ID, AGENT_ID, 'c1');
|
||||
// A poll lands first and replaces the optimistic row with settled data.
|
||||
getRunDetail.mockResolvedValue(runDetail([{ ...result('c1'), status: 'success' }], 1));
|
||||
getRunDetail.mockResolvedValue(
|
||||
runDetail([{ ...result('c1'), status: 'success', runAt: '2026-01-01T01:00:00.000Z' }], 1),
|
||||
);
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
|
||||
rejectRerun(new Error('timeout'));
|
||||
|
|
@ -1522,7 +1868,14 @@ describe('useAgentEvalsStore', () => {
|
|||
// The backend saved the rule before the failure.
|
||||
getRunDetail.mockResolvedValue(
|
||||
runDetail(
|
||||
[{ ...result('c1'), status: 'error', input: { input: 'q', criteria: 'new' } }],
|
||||
[
|
||||
{
|
||||
...result('c1'),
|
||||
status: 'error',
|
||||
runAt: '2026-01-01T01:00:00.000Z',
|
||||
input: { input: 'q', criteria: 'new' },
|
||||
},
|
||||
],
|
||||
1,
|
||||
),
|
||||
);
|
||||
|
|
@ -1540,6 +1893,7 @@ describe('useAgentEvalsStore', () => {
|
|||
});
|
||||
|
||||
it('falls back to the previous row when reading it back also fails', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
|
|
@ -1550,12 +1904,14 @@ describe('useAgentEvalsStore', () => {
|
|||
store.rerunResult(PROJECT_ID, AGENT_ID, 'c1', { whatToCheck: 'new' }),
|
||||
).rejects.toThrow('timeout');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15000);
|
||||
expect(store.getReview(RUN_ID).results[0].status).toBe('error');
|
||||
});
|
||||
|
||||
// A row deleted ahead of this one shifts the page offset, so the single-row
|
||||
// read lands on a different result and must not be applied.
|
||||
it('falls back to the previous row when the read-back returns a different result', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({
|
||||
results: [
|
||||
{ ...result('c1'), status: 'error' },
|
||||
|
|
@ -1573,6 +1929,7 @@ describe('useAgentEvalsStore', () => {
|
|||
store.rerunResult(PROJECT_ID, AGENT_ID, 'c2', { whatToCheck: 'new' }),
|
||||
).rejects.toThrow('timeout');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30000);
|
||||
const rows = store.getReview(RUN_ID).results;
|
||||
expect(rows.map((r) => r.id)).toEqual(['c1', 'c2']);
|
||||
expect(rows[1].status).toBe('error');
|
||||
|
|
@ -1644,6 +2001,7 @@ describe('useAgentEvalsStore', () => {
|
|||
});
|
||||
|
||||
it('reverts an edited rule along with the status when the rerun fails', async () => {
|
||||
vi.useFakeTimers();
|
||||
mockRun({ results: [{ ...result('c1'), status: 'error' }], count: 1, ratings: [] });
|
||||
const store = useAgentEvalsStore();
|
||||
await store.openRun(PROJECT_ID, AGENT_ID, RUN_ID);
|
||||
|
|
@ -1653,6 +2011,7 @@ describe('useAgentEvalsStore', () => {
|
|||
store.rerunResult(PROJECT_ID, AGENT_ID, 'c1', { whatToCheck: 'New rule.' }),
|
||||
).rejects.toThrow('timeout');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30000);
|
||||
expect(store.getReview(RUN_ID).results[0].input).toEqual(result('c1').input);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { defineStore } from 'pinia';
|
|||
import { ref } from 'vue';
|
||||
|
||||
import { STORES } from '@n8n/stores';
|
||||
import { ResponseError } from '@n8n/rest-api-client';
|
||||
import { useRootStore } from '@n8n/stores/useRootStore';
|
||||
|
||||
import { TIME } from '@/app/constants';
|
||||
|
|
@ -78,10 +79,13 @@ const RUN_POLL_INTERVAL_MS = 5 * TIME.SECOND;
|
|||
const RUN_POLL_TIMEOUT_MS = 10 * TIME.MINUTE;
|
||||
/** One failed poll is a blip worth retrying; a sustained run of them means we've lost the run. */
|
||||
const RUN_POLL_MAX_ERRORS = 3;
|
||||
/** Briefly confirm an execution whose request failed before any attempt was observed. */
|
||||
const RESULT_CONFIRMATION_TIMEOUT_MS = 30 * TIME.SECOND;
|
||||
|
||||
/** A run in one of these states is still doing work. */
|
||||
const isPendingStatus = (status: AgentEvalRunStatus | undefined) =>
|
||||
status === 'new' || status === 'running';
|
||||
const isPendingStatus = (
|
||||
status: AgentEvalRunStatus | AgentEvalResultRecord['status'] | undefined,
|
||||
) => status === 'new' || status === 'running';
|
||||
|
||||
const emptyRunReview = (): RunReviewState => ({
|
||||
run: null,
|
||||
|
|
@ -112,6 +116,12 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
// Keyed by runId: a run's cases can never render under a different run.
|
||||
const reviewByRunId = ref<Record<string, RunReviewState>>({});
|
||||
|
||||
// Local execution boundaries fence responses that started before a rerun or
|
||||
// settled after its final response. The old runAt also fences pre-claim reads.
|
||||
let resultRevision = 0;
|
||||
const revisionByResult = new Map<string, number>();
|
||||
const rerunningResults = new Map<string, { previousRunAt: string | null }>();
|
||||
|
||||
// The newest run per dataset. Lets the review view find something to show
|
||||
// without owning a run list of its own.
|
||||
const latestRunIdByDatasetId = ref<Record<string, string | null>>({});
|
||||
|
|
@ -611,6 +621,25 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
): AgentEvalResultRecord[] =>
|
||||
held.length > refreshed.length ? [...refreshed, ...held.slice(refreshed.length)] : refreshed;
|
||||
|
||||
const mergeReadResults = (
|
||||
held: AgentEvalResultRecord[],
|
||||
refreshed: AgentEvalResultRecord[],
|
||||
readRevision: number,
|
||||
): AgentEvalResultRecord[] =>
|
||||
refreshed.map((fresh) => {
|
||||
const current = held.find((row) => row.id === fresh.id);
|
||||
if (!current) return fresh;
|
||||
if (current.runAt && fresh.runAt && fresh.runAt < current.runAt) return current;
|
||||
if ((revisionByResult.get(fresh.id) ?? 0) > readRevision) return current;
|
||||
const pending = rerunningResults.get(fresh.id);
|
||||
if (
|
||||
pending &&
|
||||
(!fresh.runAt || (pending.previousRunAt && fresh.runAt <= pending.previousRunAt))
|
||||
)
|
||||
return current;
|
||||
return fresh;
|
||||
});
|
||||
|
||||
/**
|
||||
* Reads back a window wider than one request by walking pages.
|
||||
*
|
||||
|
|
@ -661,6 +690,7 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
*/
|
||||
const settleRun = async (projectId: string, agentId: string, runId: string) => {
|
||||
const target = loadedWindow(runId);
|
||||
const readRevision = resultRevision;
|
||||
const [window, ratings] = await Promise.all([
|
||||
readResultsWindow(projectId, agentId, runId, target),
|
||||
agentEvalsApi.listLatestRatingsForRun(rootStore.restApiContext, projectId, agentId, runId),
|
||||
|
|
@ -669,7 +699,10 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
const current = getReview(runId);
|
||||
patchReview(runId, {
|
||||
...(window.run ? { run: window.run } : {}),
|
||||
results: spliceRefreshed(current.results, window.results),
|
||||
results: spliceRefreshed(
|
||||
current.results,
|
||||
mergeReadResults(current.results, window.results, readRevision),
|
||||
),
|
||||
resultsCount: window.total,
|
||||
ratingsByResultId: mergeRatings(current.ratingsByResultId, ratings),
|
||||
});
|
||||
|
|
@ -691,6 +724,7 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
take: number = AGENT_EVAL_RESULTS_DEFAULT_TAKE,
|
||||
) => {
|
||||
const key = openKeyFor(projectId, agentId, runId);
|
||||
const readRevision = resultRevision;
|
||||
latestOpenKey = key;
|
||||
patchReview(runId, { loading: true });
|
||||
|
||||
|
|
@ -714,7 +748,7 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
// the row already marks it unsaved.
|
||||
patchReview(runId, {
|
||||
run,
|
||||
results: results.data,
|
||||
results: mergeReadResults(current.results, results.data, readRevision),
|
||||
resultsCount: results.count,
|
||||
// Newest per case wins — see `mergeRatings`. Neither side can be trusted
|
||||
// to be current: a save that landed during this read is missing from it,
|
||||
|
|
@ -948,6 +982,7 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
* collapse it back to the first page.
|
||||
*/
|
||||
const refreshResults = async (projectId: string, agentId: string, runId: string) => {
|
||||
const readRevision = resultRevision;
|
||||
const detail = await agentEvalsApi.getRunDetail(
|
||||
rootStore.restApiContext,
|
||||
projectId,
|
||||
|
|
@ -958,7 +993,10 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
const { results, ...run } = detail;
|
||||
patchReview(runId, {
|
||||
run,
|
||||
results: spliceRefreshed(getReview(runId).results, results.data),
|
||||
results: spliceRefreshed(
|
||||
getReview(runId).results,
|
||||
mergeReadResults(getReview(runId).results, results.data, readRevision),
|
||||
),
|
||||
resultsCount: results.count,
|
||||
});
|
||||
};
|
||||
|
|
@ -1114,6 +1152,7 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
runId: string,
|
||||
resultId: string,
|
||||
): Promise<boolean> => {
|
||||
const readRevision = resultRevision;
|
||||
const index = getReview(runId).results.findIndex((r) => r.id === resultId);
|
||||
if (index < 0) return false;
|
||||
const detail = await agentEvalsApi.getRunDetail(
|
||||
|
|
@ -1125,17 +1164,124 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
);
|
||||
const fresh = detail.results.data[0];
|
||||
if (fresh?.id !== resultId) return false;
|
||||
const [merged] = mergeReadResults(getReview(runId).results, [fresh], readRevision);
|
||||
if (merged !== fresh) return false;
|
||||
replaceCachedResult(runId, resultId, fresh);
|
||||
return true;
|
||||
};
|
||||
|
||||
const startResultProgress = (
|
||||
projectId: string,
|
||||
agentId: string,
|
||||
runId: string,
|
||||
result: AgentEvalResultRecord,
|
||||
recovering = false,
|
||||
) => {
|
||||
const pending = { previousRunAt: result.runAt };
|
||||
rerunningResults.set(result.id, pending);
|
||||
const ownsProgress = () => rerunningResults.get(result.id) === pending;
|
||||
revisionByResult.set(result.id, ++resultRevision);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let errors = 0;
|
||||
const deadline = Date.now() + RUN_POLL_TIMEOUT_MS;
|
||||
const confirmationDeadline = Date.now() + RESULT_CONFIRMATION_TIMEOUT_MS;
|
||||
const schedule = () => {
|
||||
if (!ownsProgress()) return;
|
||||
timer = setTimeout(async () => {
|
||||
if (!ownsProgress()) return;
|
||||
const current = findCachedResult(result.id)?.result;
|
||||
const confirmationExpired =
|
||||
recovering && current?.runAt === result.runAt && Date.now() >= confirmationDeadline;
|
||||
if (confirmationExpired || Date.now() > deadline) {
|
||||
stop(true);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Confirmation is brief and must still read in a background tab.
|
||||
if (!document.hidden || (recovering && current?.runAt === result.runAt)) {
|
||||
await refreshCachedResult(projectId, agentId, runId, result.id);
|
||||
}
|
||||
errors = 0;
|
||||
} catch {
|
||||
if (++errors >= RUN_POLL_MAX_ERRORS) {
|
||||
stop(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (recovering && !isPendingStatus(findCachedResult(result.id)?.result.status)) {
|
||||
stop();
|
||||
return;
|
||||
}
|
||||
schedule();
|
||||
}, RUN_POLL_INTERVAL_MS);
|
||||
};
|
||||
const stop = (exhausted = false) => {
|
||||
clearTimeout(timer);
|
||||
if (!ownsProgress()) return;
|
||||
const current = findCachedResult(result.id)?.result;
|
||||
if (
|
||||
recovering &&
|
||||
exhausted &&
|
||||
current?.runAt === result.runAt &&
|
||||
current?.status === 'running'
|
||||
) {
|
||||
// Only an unconfirmed optimistic row can fall back after the retry budget.
|
||||
replaceCachedResult(runId, result.id, result);
|
||||
}
|
||||
rerunningResults.delete(result.id);
|
||||
revisionByResult.set(result.id, ++resultRevision);
|
||||
};
|
||||
schedule();
|
||||
return stop;
|
||||
};
|
||||
|
||||
const recoverResult = async (
|
||||
projectId: string,
|
||||
agentId: string,
|
||||
cached: { runId: string; result: AgentEvalResultRecord },
|
||||
error: unknown,
|
||||
) => {
|
||||
// A lost POST response is not proof that execution never started. Keep the
|
||||
// attempt boundary through confirmation reads, including stale batch reads.
|
||||
const stopRecovery = startResultProgress(projectId, agentId, cached.runId, cached.result, true);
|
||||
const recovery = rerunningResults.get(cached.result.id);
|
||||
await refreshCachedResult(projectId, agentId, cached.runId, cached.result.id).catch(
|
||||
() => false,
|
||||
);
|
||||
if (rerunningResults.get(cached.result.id) !== recovery) return;
|
||||
const current = findCachedResult(cached.result.id)?.result;
|
||||
if (!isPendingStatus(current?.status)) {
|
||||
stopRecovery();
|
||||
} else if (
|
||||
current?.runAt === cached.result.runAt &&
|
||||
error instanceof ResponseError &&
|
||||
(error.httpStatusCode === 401 || error.httpStatusCode === 403)
|
||||
) {
|
||||
// Authorization failures reject execution before the runner can claim it.
|
||||
stopRecovery();
|
||||
replaceCachedResult(cached.runId, cached.result.id, cached.result);
|
||||
}
|
||||
};
|
||||
|
||||
const pendingResult = (result: AgentEvalResultRecord): AgentEvalResultRecord => ({
|
||||
...result,
|
||||
status: 'running',
|
||||
output: null,
|
||||
toolCalls: null,
|
||||
metrics: null,
|
||||
verdict: null,
|
||||
completedAt: null,
|
||||
errorCode: null,
|
||||
errorDetails: null,
|
||||
});
|
||||
|
||||
// Re-executes one already-settled case in place — no new run. Patches the
|
||||
// result to `running` in the cache before the request even lands, the same
|
||||
// way a batch run's own seeded rows read while in flight — every consumer
|
||||
// of `getReview` (the checks panel, the suite examples list) picks this up
|
||||
// through the ordinary status → avatar mapping, with nothing rerun-specific
|
||||
// of its own to track. The REST call is a single synchronous round trip, so
|
||||
// this optimistic patch is the only time the UI ever sees `running` at all.
|
||||
// a separate result poll reads tool progress until that request settles.
|
||||
const rerunResult = async (
|
||||
projectId: string,
|
||||
agentId: string,
|
||||
|
|
@ -1145,7 +1291,8 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
const cached = findCachedResult(resultId);
|
||||
// Already showing as running from an earlier click — don't fire a second
|
||||
// request the backend would just reject.
|
||||
if (cached?.result.status === 'running') return cached.result;
|
||||
if (cached?.result.status === 'running' || rerunningResults.has(resultId))
|
||||
return cached?.result;
|
||||
|
||||
if (cached) {
|
||||
// An edited rule reads instantly too, rather than waiting on the round
|
||||
|
|
@ -1156,11 +1303,13 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
? { ...(cached.result.input ?? {}), criteria: options.whatToCheck }
|
||||
: cached.result.input;
|
||||
replaceCachedResult(cached.runId, resultId, {
|
||||
...cached.result,
|
||||
status: 'running',
|
||||
...pendingResult(cached.result),
|
||||
input: optimisticInput,
|
||||
});
|
||||
}
|
||||
const stopProgress = cached
|
||||
? startResultProgress(projectId, agentId, cached.runId, cached.result)
|
||||
: () => {};
|
||||
try {
|
||||
const updated = await agentEvalsApi.rerunResult(
|
||||
rootStore.restApiContext,
|
||||
|
|
@ -1172,28 +1321,11 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
replaceCachedResult(updated.runId, resultId, updated);
|
||||
return updated;
|
||||
} catch (error) {
|
||||
// A poll or refresh may already have replaced the optimistic row with
|
||||
// fresher data — only undo it while it is still the patch we made.
|
||||
if (cached && findCachedResult(resultId)?.result.status === 'running') {
|
||||
if (options.whatToCheck === undefined) {
|
||||
replaceCachedResult(cached.runId, resultId, cached.result);
|
||||
} else {
|
||||
// The backend saves an edited rule before it executes, so a failure
|
||||
// doesn't say whether the old or new rule is stored — read it back
|
||||
// rather than guess. If that read fails or lands on another row, fall
|
||||
// back to the old one so the row doesn't stay on "running".
|
||||
const refreshed = await refreshCachedResult(
|
||||
projectId,
|
||||
agentId,
|
||||
cached.runId,
|
||||
resultId,
|
||||
).catch(() => false);
|
||||
if (!refreshed && findCachedResult(resultId)?.result.status === 'running') {
|
||||
replaceCachedResult(cached.runId, resultId, cached.result);
|
||||
}
|
||||
}
|
||||
}
|
||||
stopProgress();
|
||||
if (cached) await recoverResult(projectId, agentId, cached, error);
|
||||
throw error;
|
||||
} finally {
|
||||
stopProgress();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -1209,14 +1341,17 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
const targets: Array<{ runId: string; result: AgentEvalResultRecord }> = [];
|
||||
for (const resultId of new Set(resultIds)) {
|
||||
const cached = findCachedResult(resultId);
|
||||
if (cached?.result.status === 'running') continue;
|
||||
if (cached?.result.status === 'running' || rerunningResults.has(resultId)) continue;
|
||||
if (cached) targets.push(cached);
|
||||
}
|
||||
if (targets.length === 0) return null;
|
||||
|
||||
for (const { runId, result } of targets) {
|
||||
replaceCachedResult(runId, result.id, { ...result, status: 'running' });
|
||||
replaceCachedResult(runId, result.id, pendingResult(result));
|
||||
}
|
||||
const stopProgress = targets.map(({ runId, result }) =>
|
||||
startResultProgress(projectId, agentId, runId, result),
|
||||
);
|
||||
try {
|
||||
const applied = await agentEvalsApi.applySuggestions(
|
||||
rootStore.restApiContext,
|
||||
|
|
@ -1229,21 +1364,13 @@ export const useAgentEvalsStore = defineStore(STORES.AGENT_EVALS, () => {
|
|||
}
|
||||
return applied;
|
||||
} catch (error) {
|
||||
// A failure after the config was saved may still have rerun some results, so
|
||||
// read each back before assuming it is unchanged. Only touch a row that still
|
||||
// shows the optimistic patch: a poll may already have replaced it.
|
||||
stopProgress.forEach((stop) => stop());
|
||||
await Promise.all(
|
||||
targets.map(async ({ runId, result }) => {
|
||||
if (findCachedResult(result.id)?.result.status !== 'running') return;
|
||||
const refreshed = await refreshCachedResult(projectId, agentId, runId, result.id).catch(
|
||||
() => false,
|
||||
);
|
||||
if (!refreshed && findCachedResult(result.id)?.result.status === 'running') {
|
||||
replaceCachedResult(runId, result.id, result);
|
||||
}
|
||||
}),
|
||||
targets.map(async (cached) => await recoverResult(projectId, agentId, cached, error)),
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
stopProgress.forEach((stop) => stop());
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import AgentChatToolPolicyRefusal from './AgentChatToolPolicyRefusal.vue';
|
|||
|
||||
const props = defineProps<{
|
||||
toolCalls: ToolCall[];
|
||||
additionalData?: Record<string, Array<{ label: string; value: unknown }>>;
|
||||
projectId?: string;
|
||||
canFixWithAssistant?: boolean;
|
||||
dismissedToolCallIds?: string[];
|
||||
|
|
@ -97,6 +98,7 @@ const { subAgentNameById } = useSubAgentNames(projectIdRef, () =>
|
|||
);
|
||||
|
||||
interface ToolStepDisplay {
|
||||
id: string;
|
||||
label: string;
|
||||
details: string;
|
||||
hasRawData: boolean;
|
||||
|
|
@ -166,15 +168,22 @@ function isEmptyToolErrorPayload(value: unknown): boolean {
|
|||
}
|
||||
|
||||
function toolStepView(tc: ToolCall): ToolStepDisplay {
|
||||
const isCompact = tc.state === TOOL_CALL_STATE.DONE && isCompactToolName(tc.tool, tc.output);
|
||||
const hasAdditionalData = Boolean(props.additionalData?.[tc.toolCallId]?.length);
|
||||
const isCompact =
|
||||
!hasAdditionalData &&
|
||||
tc.state === TOOL_CALL_STATE.DONE &&
|
||||
isCompactToolName(tc.tool, tc.output);
|
||||
const details = isCompact ? '' : (getToolCallDetails(tc, i18n, subAgentNameById.value) ?? '');
|
||||
const metadata = toolStepMetadata(tc);
|
||||
const hasChildProgress = Boolean(tc.childProgress);
|
||||
return {
|
||||
id: tc.toolCallId,
|
||||
label: [toolStepLabel(tc, isCompact), ...metadata].join(' · '),
|
||||
details,
|
||||
hasRawData: !isCompact && details.length === 0 && hasToolData(tc) && !hasChildProgress,
|
||||
expandable: !isCompact && (details.length > 0 || hasToolData(tc) || hasChildProgress),
|
||||
expandable:
|
||||
!isCompact &&
|
||||
(details.length > 0 || hasToolData(tc) || hasChildProgress || hasAdditionalData),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -239,7 +248,7 @@ function hasActiveToolCall(): boolean {
|
|||
<template v-for="tc in toolCalls" :key="tc.toolCallId">
|
||||
<N8nAiActivityStep
|
||||
v-for="view in [toolStepView(tc)]"
|
||||
:key="`${tc.toolCallId}-${view.label}`"
|
||||
:key="view.id"
|
||||
:label="view.label"
|
||||
:loading="isToolStepLoading(tc)"
|
||||
:error="toolStepError(tc)"
|
||||
|
|
@ -302,6 +311,16 @@ function hasActiveToolCall(): boolean {
|
|||
<pre :class="$style.toolDataContent">{{ formatToolData(tc.output) }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="additionalData?.[tc.toolCallId]?.length" :class="$style.toolDataList">
|
||||
<div
|
||||
v-for="entry in additionalData[tc.toolCallId]"
|
||||
:key="entry.label"
|
||||
:class="$style.toolDataSection"
|
||||
>
|
||||
<span :class="$style.toolDataLabel">{{ entry.label }}</span>
|
||||
<pre :class="$style.toolDataContent">{{ formatToolData(entry.value) }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
</N8nAiActivityStep>
|
||||
</template>
|
||||
</N8nAiActivityStepGroup>
|
||||
|
|
@ -323,7 +342,7 @@ function hasActiveToolCall(): boolean {
|
|||
:fallback-name="toolStepLabel(tc)"
|
||||
/>
|
||||
</template>
|
||||
<template v-for="view in [toolStepView(tc)]" :key="view.label">
|
||||
<template v-for="view in [toolStepView(tc)]" :key="view.id">
|
||||
<div
|
||||
v-if="tc.childProgress"
|
||||
:class="$style.childProgress"
|
||||
|
|
@ -373,6 +392,16 @@ function hasActiveToolCall(): boolean {
|
|||
<pre :class="$style.toolDataContent">{{ formatToolData(tc.output) }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="additionalData?.[tc.toolCallId]?.length" :class="$style.toolDataList">
|
||||
<div
|
||||
v-for="entry in additionalData[tc.toolCallId]"
|
||||
:key="entry.label"
|
||||
:class="$style.toolDataSection"
|
||||
>
|
||||
<span :class="$style.toolDataLabel">{{ entry.label }}</span>
|
||||
<pre :class="$style.toolDataContent">{{ formatToolData(entry.value) }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
</N8nAiActivityStep>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* Either way the case is written to the run's dataset, and the parent is told
|
||||
* so it can run the checks again — a run only holds the cases it started with.
|
||||
*/
|
||||
import { computed, onBeforeUnmount, onMounted, ref } from 'vue';
|
||||
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue';
|
||||
import type { AgentEvalDraftCase } from '@n8n/api-types';
|
||||
import { N8nButton, N8nIcon, N8nSpinner, N8nText } from '@n8n/design-system';
|
||||
import { useToast } from '@n8n/composables/useToast';
|
||||
|
|
@ -22,14 +22,16 @@ const props = defineProps<{
|
|||
caseSource: AgentEvalCaseSource | null;
|
||||
/** No `agent:update` — nothing can be added. */
|
||||
disabled?: boolean;
|
||||
/** The checks are starting or running, so another add would start a second run. */
|
||||
/** A replacement run is starting, so its dataset is not ready for another write. */
|
||||
busy?: boolean;
|
||||
/** Save new cases for the next run while this run keeps its original cases. */
|
||||
deferRun?: boolean;
|
||||
}>();
|
||||
|
||||
const emit = defineEmits<{
|
||||
close: [];
|
||||
/** A case was written to the dataset. */
|
||||
added: [];
|
||||
added: [deferRun: boolean];
|
||||
}>();
|
||||
|
||||
const i18n = useI18n();
|
||||
|
|
@ -47,6 +49,16 @@ const loadingPrepared = ref(false);
|
|||
const expanded = ref(false);
|
||||
// 'rule' for the typed rule, or the key of the prepared case being added.
|
||||
const adding = ref<'rule' | number | null>(null);
|
||||
let deferAddedCheck = false;
|
||||
|
||||
// A run can start and finish before drafting or saving the new check completes.
|
||||
watch(
|
||||
() => props.deferRun || props.busy,
|
||||
(busy) => {
|
||||
if (adding.value !== null && busy) deferAddedCheck = true;
|
||||
},
|
||||
{ flush: 'sync' },
|
||||
);
|
||||
|
||||
const canAdd = computed(
|
||||
() => !props.disabled && props.caseSource !== null && !props.busy && adding.value === null,
|
||||
|
|
@ -85,7 +97,7 @@ onBeforeUnmount(() => {
|
|||
isMounted = false;
|
||||
});
|
||||
|
||||
/** Writes the case to the dataset, then tells the parent to run the checks again. */
|
||||
/** Reports the saved case and whether it must wait for the next run. */
|
||||
async function writeCase(value: { input: string; whatToCheck: string }): Promise<boolean> {
|
||||
const source = props.caseSource;
|
||||
if (!source) return false;
|
||||
|
|
@ -97,7 +109,7 @@ async function writeCase(value: { input: string; whatToCheck: string }): Promise
|
|||
if (isMounted) toast.showError(error, i18n.baseText('agents.builder.agentEvals.case.addError'));
|
||||
return false;
|
||||
}
|
||||
emit('added');
|
||||
emit('added', deferAddedCheck || Boolean(props.deferRun || props.busy));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -105,6 +117,7 @@ async function onAddRule() {
|
|||
const rule = ruleInput.value.trim();
|
||||
if (!rule || !canAdd.value) return;
|
||||
|
||||
deferAddedCheck = Boolean(props.deferRun || props.busy);
|
||||
adding.value = 'rule';
|
||||
try {
|
||||
let input: string;
|
||||
|
|
@ -133,6 +146,7 @@ async function onAddRule() {
|
|||
async function onAddPrepared(row: PreparedCase) {
|
||||
if (!canAdd.value) return;
|
||||
|
||||
deferAddedCheck = Boolean(props.deferRun || props.busy);
|
||||
adding.value = row.key;
|
||||
try {
|
||||
const written = await writeCase({ input: row.input, whatToCheck: row.whatToCheck });
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
* never passes `label` to `AgentEvalTryRow`.
|
||||
*/
|
||||
import { computed, onBeforeUnmount, onMounted, ref, watch } from 'vue';
|
||||
import { N8nButton, N8nIcon } from '@n8n/design-system';
|
||||
import { N8nButton, N8nIcon, N8nText } from '@n8n/design-system';
|
||||
import { useToast } from '@n8n/composables/useToast';
|
||||
import { useI18n, type BaseTextKey } from '@n8n/i18n';
|
||||
|
||||
|
|
@ -209,7 +209,22 @@ watch(
|
|||
{ immediate: true },
|
||||
);
|
||||
|
||||
function isSettledCheck(row: CheckRow) {
|
||||
return row.status !== 'waiting' && row.status !== 'idle';
|
||||
}
|
||||
|
||||
function canChangeResult(resultId: string) {
|
||||
return (
|
||||
!props.disabled &&
|
||||
!props.rerunning &&
|
||||
!showingPreviousRun.value &&
|
||||
!isApplyingSuggestions.value &&
|
||||
rows.value.some((row) => row.id === resultId && isSettledCheck(row))
|
||||
);
|
||||
}
|
||||
|
||||
async function onActuallyFine(resultId: string) {
|
||||
if (!canChangeResult(resultId)) return;
|
||||
try {
|
||||
await store.acceptResult(props.projectId, props.agentId, resultId);
|
||||
} catch (error) {
|
||||
|
|
@ -218,6 +233,7 @@ async function onActuallyFine(resultId: string) {
|
|||
}
|
||||
|
||||
async function onRerunCheck(resultId: string) {
|
||||
if (!canChangeResult(resultId)) return;
|
||||
try {
|
||||
await store.rerunResult(props.projectId, props.agentId, resultId);
|
||||
} catch (error) {
|
||||
|
|
@ -234,7 +250,7 @@ const applyingAll = ref(false);
|
|||
// composable's concern, not this button's.
|
||||
const applicableSuggestionIds = computed(() =>
|
||||
rows.value
|
||||
.filter((row) => row.fixSuggestion !== null && row.status !== 'waiting')
|
||||
.filter((row) => row.fixSuggestion !== null && isSettledCheck(row))
|
||||
.map((row) => row.id),
|
||||
);
|
||||
|
||||
|
|
@ -243,10 +259,13 @@ const applicableSuggestionIds = computed(() =>
|
|||
const isApplyingSuggestions = computed(() => applyingSuggestionIds.value.length > 0);
|
||||
|
||||
async function onApplySuggestion(resultId: string) {
|
||||
if (!canChangeResult(resultId)) return;
|
||||
await applySuggestions([resultId]);
|
||||
}
|
||||
|
||||
async function onApplyAllSuggestions() {
|
||||
if (props.disabled || props.rerunning || showingPreviousRun.value || isApplyingSuggestions.value)
|
||||
return;
|
||||
applyingAll.value = true;
|
||||
try {
|
||||
await applySuggestions(applicableSuggestionIds.value);
|
||||
|
|
@ -260,6 +279,7 @@ async function onApplyAllSuggestions() {
|
|||
// this result's snapshot. A result with no row to write to (an older run, or a dataset
|
||||
// with no rule column) keeps the snapshot-only edit.
|
||||
async function onSaveWhatToCheck(resultId: string, whatToCheck: string) {
|
||||
if (!canChangeResult(resultId)) return;
|
||||
// `Number(null)` is 0, which would aim the write at a row that is not this check's.
|
||||
const sourceRowId = rows.value.find((row) => row.id === resultId)?.sourceRowId;
|
||||
const rowId = sourceRowId ? Number(sourceRowId) : Number.NaN;
|
||||
|
|
@ -299,6 +319,21 @@ const addCheckSource = computed(() => {
|
|||
});
|
||||
|
||||
const addCheckOpen = ref(false);
|
||||
const savedForNextRunCount = ref(0);
|
||||
watch(
|
||||
() => props.runId,
|
||||
() => {
|
||||
savedForNextRunCount.value = 0;
|
||||
},
|
||||
);
|
||||
|
||||
function onCheckAdded(deferRun: boolean) {
|
||||
if (deferRun || inFlight.value || props.rerunning) {
|
||||
savedForNextRunCount.value++;
|
||||
return;
|
||||
}
|
||||
emit('rerun');
|
||||
}
|
||||
// Mounted on first open and kept after that, so closing and reopening the panel,
|
||||
// or a rerun that briefly empties the review, doesn't draft a new batch of
|
||||
// prepared cases.
|
||||
|
|
@ -310,6 +345,7 @@ function toggleAddCheck() {
|
|||
}
|
||||
|
||||
async function onDeleteCheck(row: CheckRow) {
|
||||
if (!canChangeResult(row.id)) return;
|
||||
const source = resolveCaseSource();
|
||||
if (!source || !row.sourceRowId) {
|
||||
toast.showError(
|
||||
|
|
@ -474,7 +510,7 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
v-if="applicableSuggestionIds.length > 0 && !showingPreviousRun"
|
||||
variant="solid"
|
||||
size="small"
|
||||
:disabled="disabled || rerunning || inFlight || isApplyingSuggestions"
|
||||
:disabled="disabled || rerunning || isApplyingSuggestions"
|
||||
:loading="applyingAll"
|
||||
data-testid="agent-eval-checks-apply-all-suggestions"
|
||||
@click="onApplyAllSuggestions"
|
||||
|
|
@ -484,7 +520,7 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
<N8nButton
|
||||
variant="subtle"
|
||||
size="small"
|
||||
:disabled="disabled || rerunning"
|
||||
:disabled="disabled || rerunning || isApplyingSuggestions"
|
||||
:loading="rerunning || inFlight"
|
||||
data-testid="agent-eval-checks-run-all"
|
||||
@click="emit('rerun')"
|
||||
|
|
@ -508,7 +544,7 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
:what-to-check="row.whatToCheck"
|
||||
:fix-suggestion="row.fixSuggestion"
|
||||
:applying-suggestion="applyingSuggestionIds.includes(row.id)"
|
||||
:disabled="disabled || showingPreviousRun || isApplyingSuggestions"
|
||||
:disabled="!canChangeResult(row.id)"
|
||||
:running-check="row.status === 'waiting'"
|
||||
hide-revise
|
||||
view="complete"
|
||||
|
|
@ -522,6 +558,20 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
/>
|
||||
</div>
|
||||
|
||||
<N8nText
|
||||
v-if="savedForNextRunCount > 0"
|
||||
size="small"
|
||||
role="status"
|
||||
data-testid="agent-eval-checks-saved-next-run"
|
||||
>
|
||||
{{
|
||||
i18n.baseText('agents.builder.agentEvals.checks.savedForNextRun', {
|
||||
adjustToNumber: savedForNextRunCount,
|
||||
interpolate: { count: String(savedForNextRunCount) },
|
||||
})
|
||||
}}
|
||||
</N8nText>
|
||||
|
||||
<!-- Not tied to `addCheckSource`: a new run's review is empty until it loads, so the
|
||||
source is briefly null after an add. Unmounting here would draft the prepared
|
||||
cases again. The panel disables adding while its source is null. -->
|
||||
|
|
@ -532,9 +582,10 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
:agent-id="agentId"
|
||||
:case-source="addCheckSource"
|
||||
:disabled="disabled || showingPreviousRun"
|
||||
:busy="rerunning || inFlight"
|
||||
:busy="Boolean(rerunning)"
|
||||
:defer-run="inFlight"
|
||||
@close="addCheckOpen = false"
|
||||
@added="emit('rerun')"
|
||||
@added="onCheckAdded"
|
||||
/>
|
||||
|
||||
<!-- Gives way to the panel while it is open, and comes back when it is closed. -->
|
||||
|
|
@ -570,6 +621,7 @@ onBeforeUnmount(store.stopPollingRun);
|
|||
|
||||
.header {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing--sm);
|
||||
|
|
|
|||
|
|
@ -7,19 +7,60 @@
|
|||
* deliberately left unset: an eval result has no execution to fix from, and
|
||||
* without them the "Fix with assistant" callout is unreachable.
|
||||
*/
|
||||
import type { ToolCall } from '@/features/ai/shared/agentsChat/types';
|
||||
import { computed } from 'vue';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import { TOOL_CALL_STATE } from '@/features/ai/shared/agentsChat/constants';
|
||||
import type { EvalDisplayToolCall } from '../utils/agent-eval-tool-calls';
|
||||
|
||||
import AgentChatToolSteps from './AgentChatToolSteps.vue';
|
||||
|
||||
defineProps<{
|
||||
toolCalls: ToolCall[];
|
||||
const props = defineProps<{
|
||||
toolCalls: EvalDisplayToolCall[];
|
||||
projectId?: string;
|
||||
}>();
|
||||
|
||||
const i18n = useI18n();
|
||||
const displayCalls = computed(() =>
|
||||
props.toolCalls.map((call) => {
|
||||
if (
|
||||
!call.mocked ||
|
||||
!call.interceptedRequests?.length ||
|
||||
call.state !== TOOL_CALL_STATE.RUNNING
|
||||
) {
|
||||
return call;
|
||||
}
|
||||
return {
|
||||
...call,
|
||||
displaySummary: i18n.baseText('agents.builder.agentEvals.checks.mockedRequestRunning', {
|
||||
adjustToNumber: call.interceptedRequests.length,
|
||||
}),
|
||||
};
|
||||
}),
|
||||
);
|
||||
const additionalData = computed(() =>
|
||||
Object.fromEntries(
|
||||
props.toolCalls
|
||||
.filter((call) => call.mocked && call.interceptedRequests?.length)
|
||||
.map((call) => [
|
||||
call.toolCallId,
|
||||
[
|
||||
{
|
||||
label: i18n.baseText('agents.builder.agentEvals.checks.mockedRequests'),
|
||||
value: call.interceptedRequests,
|
||||
},
|
||||
],
|
||||
]),
|
||||
),
|
||||
);
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="$style.toolCalls" data-testid="agent-eval-tool-calls">
|
||||
<AgentChatToolSteps :tool-calls="toolCalls" :project-id="projectId" />
|
||||
<AgentChatToolSteps
|
||||
:tool-calls="displayCalls"
|
||||
:additional-data="additionalData"
|
||||
:project-id="projectId"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,7 @@
|
|||
<script setup lang="ts">
|
||||
/**
|
||||
* One eval case as a single row: a status avatar, the request text, and —
|
||||
* once it has run, or it needs correction — a chevron. In the complete view
|
||||
* the chevron expands the full input/output sample in place. In the small view
|
||||
* it is a right chevron that emits `open` instead, so the parent can open the
|
||||
* complete view on this case. A case with no output and nothing to correct has
|
||||
* nothing to open, so the chevron is hidden; an idle (never-run) case says so
|
||||
* in its place. A "needs work" or "couldn't finish" case still opens with no
|
||||
* output, to reach the correction form.
|
||||
* Shows one check and opens its full details, including running checks.
|
||||
* The complete view expands in place; the small view opens the full check view.
|
||||
*/
|
||||
import { computed, nextTick, ref, watch } from 'vue';
|
||||
import {
|
||||
|
|
@ -113,7 +107,8 @@ const showFixSuggestion = computed(
|
|||
);
|
||||
// A "couldn't finish" case often has no output at all — it must still expand
|
||||
// to reach the correction form, so this isn't gated on output alone.
|
||||
const canExpand = computed(() => props.output !== null || needsCorrection.value);
|
||||
const isRunning = computed(() => props.status === 'waiting' || props.runningCheck);
|
||||
const canExpand = computed(() => isRunning.value || props.output !== null || needsCorrection.value);
|
||||
|
||||
type StatusText = { labelKey: BaseTextKey; color?: TextColor; warning?: boolean };
|
||||
|
||||
|
|
@ -245,6 +240,8 @@ watch(
|
|||
type="button"
|
||||
:class="$style.expandToggle"
|
||||
:data-test-id="testId && `${testId}-toggle`"
|
||||
:aria-expanded="expanded"
|
||||
:aria-label="i18n.baseText('instanceAi.testAgentPreview.conversation')"
|
||||
>
|
||||
<N8nIcon :icon="expanded ? 'chevron-up' : 'chevron-down'" size="small" />
|
||||
</button>
|
||||
|
|
@ -261,6 +258,7 @@ watch(
|
|||
type="button"
|
||||
:class="$style.expandToggle"
|
||||
:data-test-id="testId && `${testId}-toggle`"
|
||||
:aria-label="i18n.baseText('instanceAi.testAgentPreview.conversation')"
|
||||
>
|
||||
<N8nIcon icon="chevron-right" size="small" />
|
||||
</button>
|
||||
|
|
@ -282,7 +280,7 @@ watch(
|
|||
variant="subtle"
|
||||
size="small"
|
||||
icon-only
|
||||
:disabled="disabled || runningCheck"
|
||||
:disabled="disabled || isRunning"
|
||||
:aria-label="i18n.baseText('instanceAi.testAgentPreview.editRule')"
|
||||
:data-test-id="testId && `${testId}-edit-rule`"
|
||||
@click="onStartEditWhatToCheck"
|
||||
|
|
@ -295,7 +293,7 @@ watch(
|
|||
variant="subtle"
|
||||
size="small"
|
||||
icon-only
|
||||
:disabled="disabled"
|
||||
:disabled="disabled || isRunning"
|
||||
:aria-label="i18n.baseText('instanceAi.testAgentPreview.deleteCheck.label')"
|
||||
:data-test-id="testId && `${testId}-delete-check`"
|
||||
@click="onDeleteCheck"
|
||||
|
|
@ -374,7 +372,7 @@ watch(
|
|||
:class="$style.fixSuggestion"
|
||||
:suggestion="fixSuggestion"
|
||||
:applying="applyingSuggestion"
|
||||
:disabled="disabled"
|
||||
:disabled="disabled || isRunning"
|
||||
:test-id="testId && `${testId}-suggestion-card`"
|
||||
@apply="emit('apply-suggestion')"
|
||||
@dismiss="suggestionDismissed = true"
|
||||
|
|
@ -387,7 +385,7 @@ watch(
|
|||
v-model="suggestion"
|
||||
type="textarea"
|
||||
:autosize="{ minRows: 2, maxRows: 6 }"
|
||||
:disabled="disabled"
|
||||
:disabled="disabled || isRunning"
|
||||
:placeholder="i18n.baseText('instanceAi.testAgentPreview.inputCorrectionPlaceholder')"
|
||||
:data-test-id="testId && `${testId}-suggestion`"
|
||||
@keydown.meta.enter="onSaveCheck"
|
||||
|
|
@ -422,7 +420,7 @@ watch(
|
|||
<N8nButton
|
||||
size="small"
|
||||
variant="subtle"
|
||||
:disabled="disabled"
|
||||
:disabled="disabled || isRunning"
|
||||
:loading="runningCheck"
|
||||
:data-test-id="testId && `${testId}-run-check`"
|
||||
@click="onRunCheck"
|
||||
|
|
|
|||
|
|
@ -41,7 +41,12 @@ const isFailing = computed(() => props.status === 'work' || props.status === 'fa
|
|||
|
||||
<AgentAnswerCard
|
||||
data-test-id="instance-ai-test-agent-preview-output"
|
||||
:source="previewOutput"
|
||||
:source="
|
||||
previewOutput ||
|
||||
(status === 'waiting'
|
||||
? i18n.baseText('agents.builder.agentEvals.checks.waitingForResponse')
|
||||
: '')
|
||||
"
|
||||
:status="status"
|
||||
/>
|
||||
<div v-if="previewOutput && !hideBanner && status !== 'waiting'" :class="$style.callout">
|
||||
|
|
|
|||
|
|
@ -53,13 +53,23 @@ describe('AgentEvalTryRow', () => {
|
|||
expect(await findByText('It is a P1.')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('hides the toggle and never renders a sample when output is null and nothing needs correcting', () => {
|
||||
const { queryByTestId } = renderComponent({
|
||||
props: { status: 'waiting', input: 'x', output: null, testId: 'row-1' },
|
||||
it('expands a running check before it has output and keeps run controls disabled', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, getByText } = renderComponent({
|
||||
props: {
|
||||
status: 'waiting',
|
||||
input: 'Send a message and wait for a reply',
|
||||
output: null,
|
||||
whatToCheck: 'Wait for a reply',
|
||||
testId: 'row-1',
|
||||
},
|
||||
});
|
||||
|
||||
expect(queryByTestId('row-1-toggle')).not.toBeInTheDocument();
|
||||
expect(queryByTestId('row-1-placeholder')).not.toBeInTheDocument();
|
||||
await user.click(getByTestId('row-1-toggle'));
|
||||
expect(getByTestId('row-1-placeholder')).toBeInTheDocument();
|
||||
expect(getByText('Wait for a reply')).toBeInTheDocument();
|
||||
expect(getByTestId('row-1-run-check')).toBeDisabled();
|
||||
expect(getByTestId('row-1-edit-rule')).toBeDisabled();
|
||||
expect(getByTestId('row-1-delete-check')).toBeDisabled();
|
||||
});
|
||||
|
||||
it.each(['work', 'fail'] as const)(
|
||||
|
|
|
|||
|
|
@ -92,3 +92,47 @@ describe('toDisplayToolCalls', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves live call identity and running state', () => {
|
||||
const calls = toDisplayToolCalls({
|
||||
calls: [
|
||||
{
|
||||
tool: 'wait_for_reply',
|
||||
toolCallId: 'call-1',
|
||||
status: 'running',
|
||||
input: { channel: 'support' },
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(calls[0]).toMatchObject({ toolCallId: 'call-1', state: 'running' });
|
||||
});
|
||||
|
||||
it('preserves recorded mock requests without treating them as completed output', () => {
|
||||
const request = {
|
||||
url: 'https://slack.com/api/chat.postMessage',
|
||||
method: 'POST',
|
||||
nodeType: 'n8n-nodes-base.slack',
|
||||
requestBody: { text: 'Mock update' },
|
||||
mockResponse: { ok: true },
|
||||
};
|
||||
const [call] = toDisplayToolCalls({
|
||||
calls: [
|
||||
{ tool: 'send_and_wait', status: 'running', mocked: true, interceptedRequests: [request] },
|
||||
],
|
||||
});
|
||||
expect(call).toMatchObject({ mocked: true, interceptedRequests: [request], state: 'running' });
|
||||
expect(call.output).toBeUndefined();
|
||||
});
|
||||
|
||||
it('filters malformed intercepted requests without discarding the tool call', () => {
|
||||
const [call] = toDisplayToolCalls({
|
||||
calls: [{ tool: 'lookup', mocked: 'yes', interceptedRequests: [null, {}, { url: 42 }] }],
|
||||
});
|
||||
expect(call).toMatchObject({ mocked: false, interceptedRequests: [] });
|
||||
});
|
||||
|
||||
it('renders a recorded error status without requiring a diagnostic string', () => {
|
||||
const [call] = toDisplayToolCalls({ calls: [{ tool: 'lookup', status: 'error' }] });
|
||||
expect(call.state).toBe(TOOL_CALL_STATE.ERROR);
|
||||
expect(call.output).toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import type { InstanceAiEvalInterceptedRequest } from '@n8n/api-types';
|
||||
import { isRecord } from '@n8n/utils/is-record';
|
||||
import { isUnknownArray } from '@n8n/utils/is-unknown-array';
|
||||
|
||||
|
|
@ -11,19 +12,37 @@ const CANCELED_ERROR = 'canceled';
|
|||
|
||||
/**
|
||||
* Only the fields the review view renders. Narrowing to the full record would
|
||||
* reject rows written before a field existed, for no gain — the extra fields
|
||||
* (`kind`, `mocked`, `interceptedRequests`) have no bearing on the disclosure.
|
||||
* reject rows written before a field existed.
|
||||
*/
|
||||
type PersistedToolCall = Pick<
|
||||
InstanceAiEvalAgentToolCallRecord,
|
||||
'tool' | 'input' | 'output' | 'error'
|
||||
'tool' | 'input' | 'output' | 'error' | 'toolCallId' | 'status' | 'mocked' | 'interceptedRequests'
|
||||
>;
|
||||
|
||||
export type EvalDisplayToolCall = ToolCall & {
|
||||
mocked?: boolean;
|
||||
interceptedRequests?: InstanceAiEvalInterceptedRequest[];
|
||||
};
|
||||
|
||||
function isInterceptedRequest(value: unknown): value is InstanceAiEvalInterceptedRequest {
|
||||
return (
|
||||
isRecord(value) &&
|
||||
typeof value.url === 'string' &&
|
||||
typeof value.method === 'string' &&
|
||||
typeof value.nodeType === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
function isPersistedToolCall(value: unknown): value is PersistedToolCall {
|
||||
return isRecord(value) && typeof value.tool === 'string';
|
||||
}
|
||||
|
||||
function resolveState(error: string | undefined): ToolCallState {
|
||||
function resolveState(
|
||||
error: string | undefined,
|
||||
status: PersistedToolCall['status'],
|
||||
): ToolCallState {
|
||||
if (status === 'running') return TOOL_CALL_STATE.RUNNING;
|
||||
if (status === 'error') return TOOL_CALL_STATE.ERROR;
|
||||
if (error === undefined) return TOOL_CALL_STATE.DONE;
|
||||
// A cancelled call isn't a failure, and rendering it as one would overstate
|
||||
// what went wrong in the run.
|
||||
|
|
@ -36,28 +55,29 @@ function resolveState(error: string | undefined): ToolCallState {
|
|||
* `AgentChatToolSteps` renders, so the chat component stays the single renderer of
|
||||
* a tool-call disclosure.
|
||||
*
|
||||
* The runner stores `{ calls: [...] }`, and its records carry neither a call id nor
|
||||
* a state — a chat tool call is observed live, an eval one is read back after the
|
||||
* fact — so both are synthesized here.
|
||||
* The runner stores `{ calls: [...] }`. Live progress carries the runtime call id
|
||||
* and status. Older completed records use a synthesized id and terminal state.
|
||||
*
|
||||
* Takes the blob as `unknown` rather than `JsonObject`: the column's declared type
|
||||
* says nothing useful about its elements, and narrowing from `JsonValue` can't
|
||||
* reach a record shape whose `input`/`output` are themselves opaque.
|
||||
*/
|
||||
export function toDisplayToolCalls(toolCalls: unknown): ToolCall[] {
|
||||
export function toDisplayToolCalls(toolCalls: unknown): EvalDisplayToolCall[] {
|
||||
if (!isRecord(toolCalls) || !isUnknownArray(toolCalls.calls)) return [];
|
||||
|
||||
return toolCalls.calls.filter(isPersistedToolCall).map((call, index) => ({
|
||||
tool: call.tool,
|
||||
// The persisted records carry no id, and this is only ever a render key.
|
||||
// Index-prefixed so repeated calls to the same tool stay distinct.
|
||||
toolCallId: `${index}-${call.tool}`,
|
||||
// Older records carry no id. The index keeps repeated calls distinct.
|
||||
toolCallId: typeof call.toolCallId === 'string' ? call.toolCallId : `${index}-${call.tool}`,
|
||||
input: call.input,
|
||||
output: call.output,
|
||||
state: resolveState(call.error),
|
||||
// `error` is deliberately not forwarded as `output`: the only errored
|
||||
// records the runner writes are ledger-only entries whose message is an
|
||||
// internal diagnostic about unattributed requests. Leaving `output` empty
|
||||
state: resolveState(call.error, call.status),
|
||||
mocked: call.mocked === true,
|
||||
interceptedRequests: isUnknownArray(call.interceptedRequests)
|
||||
? call.interceptedRequests.filter(isInterceptedRequest)
|
||||
: [],
|
||||
// `error` is deliberately not forwarded as `output`: it can be an
|
||||
// internal diagnostic about unattributed requests or interrupted execution. Leaving `output` empty
|
||||
// makes the renderer fall back to its generic tool-error copy, which is
|
||||
// what a reviewer should see.
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -609,12 +609,30 @@ describe('InstanceAiTestAgentExamplesPanel', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('is hidden while the run is still going', () => {
|
||||
const { queryByTestId } = renderComponent({
|
||||
it('ignores a queued apply-all click after a suggestion starts applying', async () => {
|
||||
const { getByTestId, emitted, rerender } = renderComponent({
|
||||
props: { caseRuns: [failed(1), failed(2)] },
|
||||
});
|
||||
const button = getByTestId(BUTTON);
|
||||
expect(button).toBeEnabled();
|
||||
await rerender({ applyingSuggestionIds: ['result-1'] });
|
||||
|
||||
button.dispatchEvent(new MouseEvent('click', { bubbles: true }));
|
||||
|
||||
expect(emitted('apply-suggestions')).toBeUndefined();
|
||||
expect(button).toBeDisabled();
|
||||
});
|
||||
|
||||
it('applies settled suggestions and offers the checks view while another check is running', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, emitted } = renderComponent({
|
||||
props: { caseRuns: [failed(1), failed(2), failed(3, { status: 'waiting' as const })] },
|
||||
});
|
||||
|
||||
expect(queryByTestId(BUTTON)).not.toBeInTheDocument();
|
||||
await user.click(getByTestId(BUTTON));
|
||||
expect(emitted('apply-suggestions')).toEqual([[['result-1', 'result-2']]]);
|
||||
await user.click(getByTestId('instance-ai-test-agent-examples-manage-checks'));
|
||||
expect(emitted('open-case')).toEqual([[null]]);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -677,8 +677,18 @@ describe('InstanceAiTestAgentPreviewPanel', () => {
|
|||
run: { status: 'running' } as never,
|
||||
results: [
|
||||
{ sourceRowId: '1', status: 'success', input: {}, output: { finalText: 'ok' } } as never,
|
||||
{
|
||||
id: 'running-result',
|
||||
sourceRowId: '2',
|
||||
status: 'running',
|
||||
input: {},
|
||||
output: null,
|
||||
toolCalls: {
|
||||
calls: [{ tool: 'wait_for_reply', toolCallId: 'call-1', status: 'running' }],
|
||||
},
|
||||
} as never,
|
||||
],
|
||||
resultsCount: 1,
|
||||
resultsCount: 2,
|
||||
ratingsByResultId: {},
|
||||
pendingByResultId: {},
|
||||
draftsByResultId: {},
|
||||
|
|
@ -688,7 +698,7 @@ describe('InstanceAiTestAgentPreviewPanel', () => {
|
|||
});
|
||||
|
||||
const user = userEvent.setup();
|
||||
const { getByTestId, findByTestId, queryByTestId, findByText } = renderComponent();
|
||||
const { getByTestId, findByTestId, queryByTestId, findByText, emitted } = renderComponent();
|
||||
await waitFor(() =>
|
||||
expect(getByTestId('instance-ai-test-agent-preview-check-harder')).toBeEnabled(),
|
||||
);
|
||||
|
|
@ -700,6 +710,9 @@ describe('InstanceAiTestAgentPreviewPanel', () => {
|
|||
expect(await findByText('Running 2 checks, 1 done')).toBeInTheDocument();
|
||||
expect(queryByTestId('instance-ai-test-agent-examples-try')).not.toBeInTheDocument();
|
||||
|
||||
await user.click(getByTestId('instance-ai-test-agent-examples-case-2-toggle'));
|
||||
expect(emitted('open-evals')).toContainEqual(['running-result']);
|
||||
|
||||
await user.click(getByTestId('instance-ai-test-agent-examples-stop'));
|
||||
|
||||
expect(cancelRun).toHaveBeenCalledWith('project-1', 'agent-1', 'dataset-2', 'suite-run');
|
||||
|
|
|
|||
|
|
@ -158,6 +158,7 @@ watch(applyingAny, (applying) => {
|
|||
});
|
||||
|
||||
function onApplyAllSuggestions() {
|
||||
if (applyingAny.value) return;
|
||||
applyingAll.value = true;
|
||||
emit('apply-suggestions', applicableSuggestionIds.value);
|
||||
}
|
||||
|
|
@ -385,7 +386,7 @@ function onCheckYourAgent() {
|
|||
</div>
|
||||
|
||||
<N8nButton
|
||||
v-if="runSettled && summaryExpanded && applicableSuggestionIds.length > 1"
|
||||
v-if="(!runSettled || summaryExpanded) && applicableSuggestionIds.length > 1"
|
||||
variant="solid"
|
||||
size="small"
|
||||
:disabled="applyingAny"
|
||||
|
|
@ -417,6 +418,15 @@ function onCheckYourAgent() {
|
|||
{{ i18n.baseText('agents.builder.agentEvals.run.cancel') }}
|
||||
</N8nButton>
|
||||
|
||||
<N8nButton
|
||||
variant="ghost"
|
||||
size="small"
|
||||
data-test-id="instance-ai-test-agent-examples-manage-checks"
|
||||
@click="emit('open-case', null)"
|
||||
>
|
||||
{{ i18n.baseText('agents.builder.agentEvals.checks.manageChecks') }}
|
||||
</N8nButton>
|
||||
|
||||
<N8nButton
|
||||
v-if="allPassed"
|
||||
variant="solid"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user