From 48e7900db0e8a3324f5d8a6ec48162dfbceb449e Mon Sep 17 00:00:00 2001 From: Benjamin Schroth <68321970+schrothbn@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:29:42 +0200 Subject: [PATCH] test(ai-builder): Add eval cases for evaluation generation (#33719) --- ...i-agent-build-config-eval-placeholder.json | 33 +++++++++++++ ...-build-evals-preserve-production-path.json | 46 +++++++++++++++++++ ...ai-agent-build-wires-evals-when-asked.json | 36 +++++++++++++++ .../ai-agent-proactively-offers-evals.json | 28 +++++++++++ .../workflows/non-ai-build-no-eval-offer.json | 29 ++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-config-eval-placeholder.json create mode 100644 packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-evals-preserve-production-path.json create mode 100644 packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-wires-evals-when-asked.json create mode 100644 packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-proactively-offers-evals.json create mode 100644 packages/@n8n/instance-ai/evaluations/data/workflows/non-ai-build-no-eval-offer.json diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-config-eval-placeholder.json b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-config-eval-placeholder.json new file mode 100644 index 00000000000..6fbfa7cfe29 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-config-eval-placeholder.json @@ -0,0 +1,33 @@ +{ + "description": "PLACEHOLDER for a not-yet-built capability. n8n's evaluation feature also has a simplified, configuration-based form: an EvaluationConfig (name + startNodeName/endNodeName + canned metrics + a Data Table dataset) attached to the workflow via the evaluation-config API — NOT evaluation nodes on the canvas. Instance AI cannot build config-based evals yet; this case documents the target behaviour for when that tool lands. It is deliberately kept OUT of the `full` tier (datasets omits `full`) so it does not pollute the gating pass rate — expect it red until the capability ships. Run it explicitly with `--tier config-evals`. The differentiator from the node-based cases: NO evaluation nodes are added to the workflow graph. Once Instance AI can create an EvaluationConfig, add a positive assertion that the config was created (start/end nodes, metrics, dataset) — today the harness only sees the workflow JSON + transcript, so the concrete assertion is the ABSENCE of node-based eval wiring.", + "conversation": [ + { + "role": "user", + "text": "Build a workflow with a form where people submit a customer-support question (their name and their question). An AI agent should read the submitted question and write a helpful, friendly reply shown back on the form's completion screen." + }, + { + "role": "assistant", + "text": "Got it — I'll build that now." + }, + { + "role": "user", + "text": [ + "[This case is about the simplified, configuration-based evaluation feature — not the node-based one. After the agent reports the workflow is built, ask it to set up evaluations for the AI replies using n8n's built-in evaluation configuration, and be explicit that you do NOT want extra evaluation nodes added to the canvas — you want the simplified config-based evaluations attached to the workflow.", + "If it recommends a metric (for example 'correctness') and asks you to approve, approve it. If it asks you to choose metrics, pick correctness.", + "Let it finish once it has set things up (or explained it can't). Keep every message short and natural. Do not ask it to add nodes.]" + ] + } + ], + "messageBudget": 8, + "complexity": "medium", + "tags": ["behaviour", "build", "evals", "config-eval", "placeholder", "ai-agent", "form"], + "triggerType": "form", + "datasets": ["config-evals"], + "processExpectations": [ + "The agent set up a configuration-based evaluation for the workflow — a named eval config referencing the workflow's start/end nodes, one or more canned metrics (e.g. correctness), and a Data Table dataset — via the evaluation configuration feature, rather than by adding evaluation nodes to the workflow. (Placeholder: expected to fail until Instance AI can build config-based evals.)" + ], + "outcomeExpectations": [ + "No node-based evaluation wiring was added to the workflow graph: there is no `n8n-nodes-base.evaluationTrigger` and no `n8n-nodes-base.evaluation` node. The configuration-based evaluation lives outside the canvas, so the workflow's node graph is the original AI agent workflow, unchanged.", + "The original workflow still functions: the AI agent that answers the submitted support question and returns the reply on the form is intact." + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-evals-preserve-production-path.json b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-evals-preserve-production-path.json new file mode 100644 index 00000000000..39565b5c450 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-evals-preserve-production-path.json @@ -0,0 +1,46 @@ +{ + "description": "An AI workflow where the agent reads its input from an upstream lookup node (not directly from the trigger's $json) and the production path has a real side-effect (sending an email). When the user asks for evaluations after the build, wiring the eval nodes must route fixed test inputs into the agent while preserving the live path — the email send must stay on the normal branch, not the eval branch. Build-only (process + outcome); no executionScenarios because the eval branch reads an unmocked Data Table.", + "conversation": [ + { + "role": "user", + "text": "Build a webhook workflow that receives a customer ID, looks up that customer's recent orders from our orders API at https://api.example-shop.test/customers/{id}/orders, and then has an AI agent use that order history to draft a friendly re-engagement email. Send the finished email to the customer with Gmail." + }, + { + "role": "assistant", + "text": "Got it — I'll build that now." + }, + { + "role": "user", + "text": [ + "[After the agent finishes building the workflow and any normal post-build follow-ups, you want evaluations / test cases added for the AI-drafted email.", + "If the agent proactively offers to add test cases or evaluations, accept. If it does NOT offer, ask it plainly to add some test cases / evaluations so you can check the drafted emails stay good when you change the prompt or model. Don't use the words 'node' or 'config' — just ask for test cases / evaluations in plain terms.", + "If it recommends a metric (for example 'correctness' or 'helpfulness') and asks you to approve, approve it. If it asks you to choose metrics, pick correctness. Once it has set up the evaluation, let it finish. Keep every message short and natural.]" + ] + } + ], + "messageBudget": 12, + "complexity": "medium", + "tags": [ + "behaviour", + "build", + "evals", + "eval-wiring", + "ai-agent", + "webhook", + "http-request", + "gmail" + ], + "triggerType": "webhook", + "datasets": ["evals", "full"], + "processExpectations": [ + "When the user asked for evaluations / test cases, the agent set them up rather than refusing or dropping the request — it proceeded to configure the evaluation (e.g. proposing or confirming a metric such as correctness).", + "The agent created the evaluation's Data Table (with input columns derived from the agent's inputs, plus an expected-output column for the judged metric) and populated it with at least one test/sample row — it did not leave the dataset empty or defer it to 'later'." + ], + "outcomeExpectations": [ + "The final workflow contains n8n evaluation nodes added on top of the original workflow: an Evaluation Trigger (`n8n-nodes-base.evaluationTrigger`) and at least one Evaluation node (`n8n-nodes-base.evaluation`).", + "The evaluation is backed by a real dataset: the Evaluation Trigger references an n8n Data Table by id (its `dataTableId` resource-locator holds a real table id, not left empty or deferred).", + "The evaluation nodes are wired to the AI agent that drafts the email, and the agent's output flows into an Evaluation node that gates a dedicated eval branch (set-outputs → set-metrics) separate from the agent's normal downstream path.", + "Because the agent draws its input from the upstream orders-lookup rather than directly from a test-input field, the eval setup routed the fixed test inputs into the agent (for example via a small Set/adapter step feeding the agent, or by wiring the Evaluation Trigger into it) so the agent can run against the test dataset.", + "The live production path was preserved and its side-effect stayed off the eval branch: on the normal (non-eval) branch the workflow still looks up the customer's orders and sends the drafted email via Gmail, and the Gmail send was not moved onto or duplicated into the evaluation branch." + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-wires-evals-when-asked.json b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-wires-evals-when-asked.json new file mode 100644 index 00000000000..a0649c66e3f --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-build-wires-evals-when-asked.json @@ -0,0 +1,36 @@ +{ + "description": "When the user asks for evaluations / test cases after building a single-agent AI workflow (the agent reads the form field directly via $json), the agent should wire node-based eval nodes into the workflow. Measures eval-creation quality: given a request, does the agent actually produce a working node-based evaluation? Build-only (process + outcome); no executionScenarios because the eval branch reads an (unmocked) Data Table. Whether the agent PROACTIVELY offers evals is tested separately in ai-agent-proactively-offers-evals.", + "conversation": [ + { + "role": "user", + "text": "Build a workflow with a form where people submit a customer-support question (their name and their question). An AI agent should read the submitted question and write a helpful, friendly reply shown back on the form's completion screen." + }, + { + "role": "assistant", + "text": "Got it — I'll build that now." + }, + { + "role": "user", + "text": [ + "[After the agent finishes building the workflow and any normal post-build follow-ups (testing, error workflow, etc.), you want evaluations / test cases added for the AI replies.", + "If the agent proactively offers to add test cases or evaluations, accept. If it does NOT offer, ask it plainly to add some test cases / evaluations so you can check the AI's answers stay correct over time when you tweak the prompt or model. Do not use the words 'node' or 'config' — just ask for test cases / evaluations in a user's plain terms.", + "If it recommends a metric (for example 'correctness') and asks you to approve, approve it. If it asks you to choose metrics from a list, pick correctness. Once it has set up the evaluation, let it finish. Keep every message short and natural.]" + ] + } + ], + "messageBudget": 12, + "complexity": "medium", + "tags": ["behaviour", "build", "evals", "eval-wiring", "ai-agent", "form"], + "triggerType": "form", + "datasets": ["evals", "full"], + "processExpectations": [ + "When the user asked for evaluations / test cases, the agent set them up rather than refusing or dropping the request — it proceeded to configure the evaluation (e.g. proposing or confirming a metric such as correctness) and reported adding the eval setup.", + "The agent created the evaluation's Data Table (with input columns derived from the agent's inputs, plus an expected-output column for the judged metric) and populated it with at least one test/sample row — it did not leave the dataset empty or defer it to 'later'." + ], + "outcomeExpectations": [ + "The final workflow contains n8n evaluation nodes added on top of the original AI agent workflow: an Evaluation Trigger (`n8n-nodes-base.evaluationTrigger`) and at least one Evaluation node (`n8n-nodes-base.evaluation`).", + "The evaluation nodes are wired to the AI agent: the Evaluation Trigger feeds the agent's input, and the agent's output flows into an Evaluation node that gates a dedicated eval branch (set-outputs → set-metrics) separate from the agent's normal reply path.", + "The evaluation is backed by a real dataset: the Evaluation Trigger references an n8n Data Table by id (its `dataTableId` resource-locator holds a real table id, not left empty or deferred).", + "The original workflow still functions: the AI agent that answers the submitted support question and returns the reply on the form is intact, and its production path was preserved rather than replaced by the evaluation setup." + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-proactively-offers-evals.json b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-proactively-offers-evals.json new file mode 100644 index 00000000000..252d436e9dd --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/ai-agent-proactively-offers-evals.json @@ -0,0 +1,28 @@ +{ + "description": "GAP CASE (expected red today). After building an AI-agent workflow, does the agent PROACTIVELY offer to add evaluations / test cases — without the user asking? Observed today: it does not — the post-build flow offers verification, a live test, and an error workflow, but the evals `offer` precheck isn't wired into it. This case documents the target behaviour and measures the routing gap for TRUST-145. Kept OUT of the `full` tier (datasets omits `full`) so it doesn't gate the suite; run it with `--tier eval-offer`. The director never asks for evals — the offer must come unprompted.", + "conversation": [ + { + "role": "user", + "text": "Build a workflow with a form where people submit a customer-support question (their name and their question). An AI agent should read the submitted question and write a helpful, friendly reply shown back on the form's completion screen." + }, + { + "role": "assistant", + "text": "Got it — I'll build that now." + }, + { + "role": "user", + "text": [ + "[This case tests whether the agent, after building this AI workflow, proactively offers to add evaluations / test cases on its own. Do NOT ask for evals, test cases, or metrics yourself — that is the behaviour under test.", + "Answer any other post-build follow-ups briefly and naturally (a live test, an error workflow, etc.) — decline or defer them. If the agent offers to add evaluations / test cases, that is the behaviour we want; you may accept. If it never offers, do not prompt it — just let the conversation end. Keep messages short.]" + ] + } + ], + "messageBudget": 8, + "complexity": "medium", + "tags": ["behaviour", "evals", "eval-offer", "gap", "ai-agent", "form"], + "triggerType": "form", + "datasets": ["eval-offer"], + "processExpectations": [ + "After building the AI workflow, the agent proactively offered to add evaluations / test cases to guard the AI agent's replies — it raised this itself, without the user asking, rather than only offering verification / a live test / an error workflow and then stopping. (Gap: expected to fail until the evals offer is wired into the post-build flow.)" + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/non-ai-build-no-eval-offer.json b/packages/@n8n/instance-ai/evaluations/data/workflows/non-ai-build-no-eval-offer.json new file mode 100644 index 00000000000..3677450ef6b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/non-ai-build-no-eval-offer.json @@ -0,0 +1,29 @@ +{ + "description": "Negative guard: the eval offer must be scoped to AI workflows. This workflow has no AI/LLM nodes, so the agent must NOT offer to add evaluations. Guards the `offer` eligibility precheck (no-ai-nodes) against over-eager offering. Single-turn conversation (auto-approve mode) — the base workflow builds and any post-build follow-ups are auto-approved; the assertion is judged from the transcript and the resulting workflow.", + "conversation": [ + { + "role": "user", + "text": "Every weekday at 8am, pull yesterday's new signups from our Postgres database and post the total count to our Slack #growth channel." + } + ], + "complexity": "simple", + "tags": [ + "behaviour", + "build", + "evals", + "eval-offer", + "negative", + "schedule", + "slack", + "postgres" + ], + "triggerType": "schedule", + "datasets": ["behaviour", "full"], + "processExpectations": [ + "The agent did not offer to add evaluations or AI test cases for this workflow. Evaluations apply to workflows with AI/LLM nodes, and this workflow has none (a schedule → Postgres query → Slack count), so an eval offer would be inappropriate." + ], + "outcomeExpectations": [ + "The final workflow contains no n8n evaluation nodes: there is no `n8n-nodes-base.evaluationTrigger` and no `n8n-nodes-base.evaluation` node.", + "The workflow does what was asked: a schedule trigger runs on weekday mornings, yesterday's new signups are queried from Postgres, and the resulting count is posted to the Slack #growth channel." + ] +}