diff --git a/packages/cli/src/modules/agent-evals/agent-eval-case-generation.service.ts b/packages/cli/src/modules/agent-evals/agent-eval-case-generation.service.ts index a0e1090cf1b..8f0741cd0f3 100644 --- a/packages/cli/src/modules/agent-evals/agent-eval-case-generation.service.ts +++ b/packages/cli/src/modules/agent-evals/agent-eval-case-generation.service.ts @@ -1,6 +1,13 @@ import type { Agent } from '@n8n/agents'; import { getProviderPrefix } from '@n8n/ai-utilities/agent-config'; -import { AGENT_EVALS_FLAG, MANAGED_CREDENTIAL_TOKEN, type AgentJsonConfig } from '@n8n/api-types'; +import { + AGENT_EVALS_FLAG, + MANAGED_CREDENTIAL_TOKEN, + type AgentEvalDraftCase, + type AgentJsonConfig, + type GenerateDraftCasesOptions, + type GenerateDraftCasesResult, +} from '@n8n/api-types'; import { Logger } from '@n8n/backend-common'; import type { User } from '@n8n/db'; import { AgentEvalDatasetRepository } from '@n8n/db'; @@ -20,7 +27,6 @@ import { CASE_GENERATION_SYSTEM_PROMPT, deriveCapabilities, generatedCasesSchema, - type AgentEvalDraftCase, } from './case-generation/case-generation-prompt'; import { sampleDimensionTuples } from './case-generation/dimensions'; import { isSupportedAgentProvider } from '../agents/json-config/credential-field-mapping'; @@ -49,19 +55,6 @@ const CRITERIA_COLUMN = 'criteria'; // per-project name clash. const MAX_NAME_ATTEMPTS = 20; -export interface GenerateDraftCasesOptions { - /** How many cases to generate (clamped to [1, 20]). Defaults to 6. */ - count?: number; - /** Dataset/table name; defaults to one derived from the agent name. */ - datasetName?: string; -} - -export interface GenerateDraftCasesResult { - datasetId: string; - dataTableId: string; - cases: AgentEvalDraftCase[]; -} - /** * Generates a handful of realistic draft eval cases from an agent's config * (name / instructions / tools) and persists them as an editable dataset. diff --git a/packages/cli/src/modules/agent-evals/case-generation/case-generation-prompt.ts b/packages/cli/src/modules/agent-evals/case-generation/case-generation-prompt.ts index 49ccccadbe9..449e505012f 100644 --- a/packages/cli/src/modules/agent-evals/case-generation/case-generation-prompt.ts +++ b/packages/cli/src/modules/agent-evals/case-generation/case-generation-prompt.ts @@ -1,4 +1,8 @@ -import type { AgentJsonConfig, AgentJsonToolConfig } from '@n8n/api-types'; +import { + agentEvalDraftCaseSchema, + type AgentJsonConfig, + type AgentJsonToolConfig, +} from '@n8n/api-types'; import { z } from 'zod'; import { @@ -8,28 +12,14 @@ import { type DimensionTuple, } from './dimensions'; -/** A single draft case: what to send the agent + a plain-language check. */ -export interface AgentEvalDraftCase { - /** A realistic end-user message to send the agent. */ - input: string; - /** Plain-language description of what a good response should do. */ - whatToCheck: string; -} - -// Structural only: blank/whitespace fields are trimmed and dropped by the -// service, so a single bad case doesn't invalidate an otherwise-good batch. -const draftCaseSchema = z.object({ - input: z.string(), - whatToCheck: z.string(), -}); - /** - * Structured-output schema for the generation call. Wrapping the array in an - * object (rather than a bare array) is more reliable across providers' JSON - * modes. + * LLM-output envelope for the generation call. Kept local — it's a parsing + * detail, not an FE/BE contract. Wrapping the array in an object (rather than a + * bare array) is more reliable across providers' JSON modes; the per-case shape + * is the shared `agentEvalDraftCaseSchema`. */ export const generatedCasesSchema = z.object({ - cases: z.array(draftCaseSchema), + cases: z.array(agentEvalDraftCaseSchema), }); // Bound the prompt so token cost stays predictable regardless of how large the