mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-19 16:55:02 +02:00
test(core): Grade node usage over a folder structure (no-changelog)
Two cases where the answer needs the node composition of a set of workflows, not their names or their graphs. - `finds-the-real-users-of-a-node-type`: inside one folder, three workflows use the Postgres node and none says so in its name, while two more carry Postgres in the NAME and never use it. A name filter returns exactly the wrong two. - `reads-a-folders-node-conventions-before-extending-it`: what a folder is built out of, before adding to it. Two workflows outside the folder use nodes that appear nowhere inside it, so an instance-wide summary gives the wrong convention. Both are folder-scoped rather than instance-wide, for a reason worth recording: the eval instance is shared and already holds unrelated workflows, so an instance-wide count is not a fact a case can assert. Scoping also makes them test the two capabilities composed — the folder bounds the set, the index says which members actually contain the node. The out-of-folder pair in the second case deliberately avoids Postgres: cases share one project and run concurrently, so one case's seeds can otherwise answer another case's question.
This commit is contained in:
Arvin Ansari
parent
fcba43ed45
commit
33ad2d3680
|
|
@ -0,0 +1,297 @@
|
|||
{
|
||||
"description": "Node usage composed with folder scope, over a structure whose names mislead. Inside one folder, three workflows use the Postgres node and none says so in its name; two more have Postgres in the NAME and do not use the node at all. A name filter returns exactly the wrong two; reading all five to check is the expensive alternative this measures against. Answering needs BOTH capabilities at once \u2014 the folder bounds the set, the dependency index says which members actually contain the node. Folder-scoped rather than instance-wide on purpose: the eval instance is shared, so an instance-wide count is not a fact a case can assert.",
|
||||
"seed": {
|
||||
"mode": "inline",
|
||||
"sessionBoundary": true,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"text": "This project has grown a lot over the last year."
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"text": "Understood."
|
||||
}
|
||||
],
|
||||
"workflows": [
|
||||
{
|
||||
"id": "wfLedgerSyncN01",
|
||||
"name": "Ledger Sync",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ee000001-0001-4d01-9a01-000000000001",
|
||||
"name": "Nightly Ledger",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ee000001-0002-4d01-9a01-000000000002",
|
||||
"name": "Write Ledger Rows",
|
||||
"type": "n8n-nodes-base.postgres",
|
||||
"typeVersion": 2.5,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"operation": "insert",
|
||||
"table": "ledger"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Nightly Ledger": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Write Ledger Rows",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Data Platform"
|
||||
},
|
||||
{
|
||||
"id": "wfChurnReportN01",
|
||||
"name": "Churn Report",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ee000002-0001-4d01-9a01-000000000001",
|
||||
"name": "Weekly Churn",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ee000002-0002-4d01-9a01-000000000002",
|
||||
"name": "Read Churn Rows",
|
||||
"type": "n8n-nodes-base.postgres",
|
||||
"typeVersion": 2.5,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"operation": "select",
|
||||
"table": "churn"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Weekly Churn": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Read Churn Rows",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Data Platform"
|
||||
},
|
||||
{
|
||||
"id": "wfSeatAuditN0001",
|
||||
"name": "Seat Audit",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ee000003-0001-4d01-9a01-000000000001",
|
||||
"name": "Audit Trigger",
|
||||
"type": "n8n-nodes-base.executeWorkflowTrigger",
|
||||
"typeVersion": 1.1,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {}
|
||||
},
|
||||
{
|
||||
"id": "ee000003-0002-4d01-9a01-000000000002",
|
||||
"name": "Read Seats",
|
||||
"type": "n8n-nodes-base.postgres",
|
||||
"typeVersion": 2.5,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"operation": "select",
|
||||
"table": "seats"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Audit Trigger": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Read Seats",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Data Platform"
|
||||
},
|
||||
{
|
||||
"id": "wfPgMigrDocsN01",
|
||||
"name": "Postgres Migration Notes",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ee000004-0001-4d01-9a01-000000000001",
|
||||
"name": "Docs Trigger",
|
||||
"type": "n8n-nodes-base.executeWorkflowTrigger",
|
||||
"typeVersion": 1.1,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {}
|
||||
},
|
||||
{
|
||||
"id": "ee000004-0002-4d01-9a01-000000000002",
|
||||
"name": "Format Notes",
|
||||
"type": "n8n-nodes-base.set",
|
||||
"typeVersion": 3.4,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Docs Trigger": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Format Notes",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Data Platform"
|
||||
},
|
||||
{
|
||||
"id": "wfPgHealthPingN1",
|
||||
"name": "Postgres Health Ping",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ee000005-0001-4d01-9a01-000000000001",
|
||||
"name": "Ping Schedule",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ee000005-0002-4d01-9a01-000000000002",
|
||||
"name": "Call Status Endpoint",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"url": "https://status.example.com/postgres",
|
||||
"options": {}
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Ping Schedule": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Call Status Endpoint",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Data Platform"
|
||||
}
|
||||
]
|
||||
},
|
||||
"conversation": [
|
||||
{
|
||||
"role": "user",
|
||||
"text": "Which workflows in my Data Platform folder actually use the Postgres node? I'm planning a database migration and I need the real list. Don't change anything.\n\n[Director note: you do not know the answer \u2014 that is why you are asking. If the agent asks you to confirm a list or name workflows, refuse and tell it to check the instance itself.]"
|
||||
}
|
||||
],
|
||||
"complexity": "medium",
|
||||
"tags": [
|
||||
"node-usage",
|
||||
"folders",
|
||||
"instance-context",
|
||||
"structure"
|
||||
],
|
||||
"messageBudget": 4,
|
||||
"contextAssertions": [
|
||||
{
|
||||
"text": "Seat Audit",
|
||||
"anchor": "turn-end",
|
||||
"note": "a real Postgres user whose name gives no hint \u2014 reachable only through the node index"
|
||||
}
|
||||
],
|
||||
"processExpectations": [
|
||||
"The agent answered with exactly three workflows from the Data Platform folder that use the Postgres node: Ledger Sync, Churn Report and Seat Audit.",
|
||||
"The agent did not list Postgres Migration Notes or Postgres Health Ping as Postgres-node users. Both are in the same folder and merely have Postgres in their name.",
|
||||
"The agent did not ask the user to supply or confirm the list."
|
||||
],
|
||||
"datasets": [
|
||||
"folders"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,375 @@
|
|||
{
|
||||
"description": "Convention discovery over a folder. The user asks what the folder is built out of before adding to it. Answering needs the node composition of six workflows, which without the dependency index means fetching and parsing all six. Folder scoping and node usage have to compose: two workflows outside the folder use nodes that appear nowhere inside it, so an instance-wide summary gives the wrong convention.",
|
||||
"seed": {
|
||||
"mode": "inline",
|
||||
"sessionBoundary": true,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"text": "The alerting work is all kept in one folder."
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"text": "Understood."
|
||||
}
|
||||
],
|
||||
"workflows": [
|
||||
{
|
||||
"id": "wfDiskAlertN0001",
|
||||
"name": "Disk Space Alert",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ff000001-0001-4d01-9a01-000000000001",
|
||||
"name": "Check Schedule",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000001-0002-4d01-9a01-000000000002",
|
||||
"name": "Query Metrics",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"url": "https://metrics.example.com/disk",
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000001-0003-4d01-9a01-000000000003",
|
||||
"name": "Post Alert",
|
||||
"type": "n8n-nodes-base.slack",
|
||||
"typeVersion": 2.2,
|
||||
"position": [
|
||||
440,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"resource": "message",
|
||||
"operation": "post"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Check Schedule": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Query Metrics",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Query Metrics": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Post Alert",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Alerting"
|
||||
},
|
||||
{
|
||||
"id": "wfLatencyAlertN1",
|
||||
"name": "Latency Alert",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ff000002-0001-4d01-9a01-000000000001",
|
||||
"name": "Latency Schedule",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000002-0002-4d01-9a01-000000000002",
|
||||
"name": "Query Latency",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"url": "https://metrics.example.com/latency",
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000002-0003-4d01-9a01-000000000003",
|
||||
"name": "Post Latency Alert",
|
||||
"type": "n8n-nodes-base.slack",
|
||||
"typeVersion": 2.2,
|
||||
"position": [
|
||||
440,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"resource": "message",
|
||||
"operation": "post"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Latency Schedule": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Query Latency",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Query Latency": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Post Latency Alert",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Alerting"
|
||||
},
|
||||
{
|
||||
"id": "wfErrorRateAlrN1",
|
||||
"name": "Error Rate Alert",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ff000003-0001-4d01-9a01-000000000001",
|
||||
"name": "Error Schedule",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [
|
||||
{
|
||||
"field": "days",
|
||||
"triggerAtHour": 7
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000003-0002-4d01-9a01-000000000002",
|
||||
"name": "Query Errors",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"url": "https://metrics.example.com/errors",
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000003-0003-4d01-9a01-000000000003",
|
||||
"name": "Post Error Alert",
|
||||
"type": "n8n-nodes-base.slack",
|
||||
"typeVersion": 2.2,
|
||||
"position": [
|
||||
440,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"resource": "message",
|
||||
"operation": "post"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Error Schedule": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Query Errors",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Query Errors": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Post Error Alert",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": "Alerting"
|
||||
},
|
||||
{
|
||||
"id": "wfBillingExpN01",
|
||||
"name": "Billing Export",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ff000004-0001-4d01-9a01-000000000001",
|
||||
"name": "Billing Hook",
|
||||
"type": "n8n-nodes-base.webhook",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"path": "billing"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000004-0002-4d01-9a01-000000000002",
|
||||
"name": "Append Billing Row",
|
||||
"type": "n8n-nodes-base.googleSheets",
|
||||
"typeVersion": 4.5,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"operation": "append"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Billing Hook": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Write Billing Rows",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wfCrmBackfillN01",
|
||||
"name": "CRM Backfill",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "ff000005-0001-4d01-9a01-000000000001",
|
||||
"name": "Backfill Hook",
|
||||
"type": "n8n-nodes-base.webhook",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"path": "crm"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ff000005-0002-4d01-9a01-000000000002",
|
||||
"name": "Append CRM Row",
|
||||
"type": "n8n-nodes-base.googleSheets",
|
||||
"typeVersion": 4.5,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"operation": "append"
|
||||
}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Backfill Hook": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Write CRM Rows",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"conversation": [
|
||||
{
|
||||
"role": "user",
|
||||
"text": "I want to add another alert to my Alerting folder and I want it to match what's already there. What are those workflows built out of \u2014 which nodes do they use? Just tell me, don't build it yet.\n\n[Director note: do not describe any workflow to the agent. If it asks what the existing alerts look like, tell it to go and look.]"
|
||||
}
|
||||
],
|
||||
"complexity": "medium",
|
||||
"tags": [
|
||||
"node-usage",
|
||||
"folders",
|
||||
"instance-context",
|
||||
"conventions"
|
||||
],
|
||||
"messageBudget": 4,
|
||||
"contextAssertions": [
|
||||
{
|
||||
"text": "n8n-nodes-base.slack",
|
||||
"anchor": "turn-end",
|
||||
"note": "the folder's house pattern; present only if the folder's composition was actually read"
|
||||
}
|
||||
],
|
||||
"processExpectations": [
|
||||
"The agent described the folder's existing pattern as a schedule trigger, an HTTP request for metrics, and a Slack message.",
|
||||
"The agent did not present Google Sheets or webhook triggers as part of the Alerting folder's pattern. Those are used only by Billing Export and CRM Backfill, which are outside the folder.",
|
||||
"The agent grounded its answer in what it read from the instance rather than asking the user to describe the existing alerts."
|
||||
],
|
||||
"datasets": [
|
||||
"folders"
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user