mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
feat(core): Count injected trigger output as simulated in AI Assistant verification (no-changelog) (#38626)
This commit is contained in:
parent
82b38382b7
commit
24951fa2f4
|
|
@ -1,5 +1,9 @@
|
|||
import type { WorkflowBuildOutcome } from '../../../../workflow-loop/workflow-loop-state';
|
||||
import { analyzeVerificationResult } from '../analyze-result';
|
||||
import {
|
||||
analyzeVerificationResult,
|
||||
INJECTED_TRIGGER_SIMULATION_REASON,
|
||||
WORKFLOW_PIN_SIMULATION_REASON,
|
||||
} from '../analyze-result';
|
||||
import type { ExecutionRunResult } from '../types';
|
||||
|
||||
function makeBuildOutcome(overrides: Partial<WorkflowBuildOutcome> = {}): WorkflowBuildOutcome {
|
||||
|
|
@ -326,6 +330,46 @@ describe('analyzeVerificationResult — workflow-pinned nodes', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('analyzeVerificationResult — injected trigger output', () => {
|
||||
const injectedRunResult = {
|
||||
executionId: 'exec-1',
|
||||
status: 'success',
|
||||
executedNodeNames: ['Webhook', 'Format'],
|
||||
lastNodeExecuted: 'Format',
|
||||
data: { Webhook: [{}], Format: [{}] },
|
||||
injectedTriggerNodeName: 'Webhook',
|
||||
} as unknown as ExecutionRunResult;
|
||||
|
||||
it('counts the injected trigger as simulated so the run is not a live trigger test', () => {
|
||||
const analysis = analyzeVerificationResult({
|
||||
result: injectedRunResult,
|
||||
buildOutcome: makeBuildOutcome(),
|
||||
simulatedNodes: [],
|
||||
stateBefore: undefined,
|
||||
runId: 'run-1',
|
||||
});
|
||||
|
||||
expect(analysis.reachedSimulatedNodes).toEqual([
|
||||
{ nodeName: 'Webhook', reason: INJECTED_TRIGGER_SIMULATION_REASON },
|
||||
]);
|
||||
expect(analysis.simulationNote).toContain('Webhook');
|
||||
});
|
||||
|
||||
it('does not duplicate a trigger that workflow pins already disclose', () => {
|
||||
const analysis = analyzeVerificationResult({
|
||||
result: { ...injectedRunResult, workflowPinnedNodeNames: ['Webhook'] },
|
||||
buildOutcome: makeBuildOutcome(),
|
||||
simulatedNodes: [],
|
||||
stateBefore: undefined,
|
||||
runId: 'run-1',
|
||||
});
|
||||
|
||||
expect(analysis.reachedSimulatedNodes).toEqual([
|
||||
{ nodeName: 'Webhook', reason: WORKFLOW_PIN_SIMULATION_REASON },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzeVerificationResult — trigger-scoped coverage', () => {
|
||||
const twoBranchOutcome = makeBuildOutcome({
|
||||
nodeSimulationPlan: [
|
||||
|
|
|
|||
|
|
@ -30,6 +30,26 @@ type ExecutionNodeError = NonNullable<ExecutionRunResult['nodeErrors']>[number];
|
|||
export const WORKFLOW_PIN_SIMULATION_REASON =
|
||||
'Output came from pinned data saved on the workflow — unpin it for a live test';
|
||||
|
||||
/** Disclosure for a trigger whose output the assistant injected instead of a real event. */
|
||||
export const INJECTED_TRIGGER_SIMULATION_REASON =
|
||||
'Trigger output was injected test input, not a real event — run a live test to prove the trigger';
|
||||
|
||||
/** Injected trigger output, deduplicated against the other simulation sources. */
|
||||
export function injectedTriggerSimulations(
|
||||
injectedTriggerNodeName: string | undefined,
|
||||
reachedNames: ReadonlySet<string>,
|
||||
alreadySimulatedNames: ReadonlySet<string>,
|
||||
): Array<{ nodeName: string; reason: string }> {
|
||||
if (
|
||||
injectedTriggerNodeName === undefined ||
|
||||
!reachedNames.has(injectedTriggerNodeName) ||
|
||||
alreadySimulatedNames.has(injectedTriggerNodeName)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
return [{ nodeName: injectedTriggerNodeName, reason: INJECTED_TRIGGER_SIMULATION_REASON }];
|
||||
}
|
||||
|
||||
const CREDENTIAL_FAILURE_KEYWORDS = [
|
||||
'credential',
|
||||
'unauthorized',
|
||||
|
|
@ -496,9 +516,16 @@ export function analyzeVerificationResult(args: {
|
|||
const workflowPinnedNodes = (result.workflowPinnedNodeNames ?? [])
|
||||
.filter((name) => reachedNames.has(name) && !plannedSimulatedNames.has(name))
|
||||
.map((name) => ({ nodeName: name, reason: WORKFLOW_PIN_SIMULATION_REASON }));
|
||||
// An injected trigger (inputData or verification pin) never fired for real,
|
||||
// so a run that starts from it is not a live test of the trigger either.
|
||||
const reachedSimulatedNodes = [
|
||||
...simulatedNodes.filter((n) => reachedNames.has(n.nodeName)),
|
||||
...workflowPinnedNodes,
|
||||
...injectedTriggerSimulations(
|
||||
result.injectedTriggerNodeName,
|
||||
reachedNames,
|
||||
new Set([...plannedSimulatedNames, ...workflowPinnedNodes.map((n) => n.nodeName)]),
|
||||
),
|
||||
];
|
||||
const nodesNotReached = (buildOutcome.nodeSimulationPlan ?? [])
|
||||
.map((verdict) => verdict.nodeName)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import {
|
||||
analyzeVerificationResult,
|
||||
buildSimulationNote,
|
||||
injectedTriggerSimulations,
|
||||
WORKFLOW_PIN_SIMULATION_REASON,
|
||||
type ChatModelRecoveryOptions,
|
||||
type VerificationAnalysis,
|
||||
|
|
@ -182,6 +183,11 @@ function mergeAnalyses(
|
|||
nodeName: name,
|
||||
reason: WORKFLOW_PIN_SIMULATION_REASON,
|
||||
})),
|
||||
...injectedTriggerSimulations(
|
||||
passes.find((pass) => pass.result.injectedTriggerNodeName)?.result.injectedTriggerNodeName,
|
||||
reachedNames,
|
||||
new Set([...plannedSimulatedNames, ...workflowPinnedNodeNames]),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -177,6 +177,12 @@ export interface ExecutionResult {
|
|||
* unsaved workflow, absent when the record could not be read.
|
||||
*/
|
||||
workflowVersionId?: string | null;
|
||||
/**
|
||||
* Set when the trigger did not fire from a real event: its output came from
|
||||
* injected `inputData` or verification pin data. Such a run proves nothing
|
||||
* about the trigger's ingress (auth, payload shape, response mode).
|
||||
*/
|
||||
injectedTriggerNodeName?: string;
|
||||
error?: string;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
|
|
|
|||
|
|
@ -4527,6 +4527,35 @@ describe('createExecutionAdapter run()', () => {
|
|||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('names the trigger whose output the caller injected', async () => {
|
||||
const { adapter } = createRunAdapterForTests(
|
||||
{
|
||||
id: 'wf-1',
|
||||
nodes: [
|
||||
{
|
||||
id: 'n1',
|
||||
name: 'Webhook',
|
||||
type: 'n8n-nodes-base.webhook',
|
||||
typeVersion: 2,
|
||||
position: [0, 0],
|
||||
},
|
||||
],
|
||||
connections: {},
|
||||
},
|
||||
{ execution: makeExecution({ status: 'success' }) },
|
||||
);
|
||||
|
||||
const injected = await adapter.run('wf-1', { body: { name: 'Ada' } });
|
||||
const pinned = await adapter.run('wf-1', undefined, {
|
||||
verificationPinData: { Webhook: [{ body: { name: 'Ada' } }] },
|
||||
});
|
||||
const live = await adapter.run('wf-1');
|
||||
|
||||
expect(injected.injectedTriggerNodeName).toBe('Webhook');
|
||||
expect(pinned.injectedTriggerNodeName).toBe('Webhook');
|
||||
expect(live).not.toHaveProperty('injectedTriggerNodeName');
|
||||
});
|
||||
|
||||
it('reports workflow-pinned nodes on the run result', async () => {
|
||||
const { adapter } = createRunAdapterForTests(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2048,9 +2048,18 @@ export class InstanceAiAdapterService {
|
|||
// Saved workflow pins fed this run (they ride every instance-ai run) —
|
||||
// report them so callers don't mistake pin-fed nodes for live ones.
|
||||
const workflowPinnedNodeNames = Object.keys(workflow.pinData ?? {});
|
||||
return workflowPinnedNodeNames.length > 0
|
||||
? { ...result, workflowPinnedNodeNames }
|
||||
: result;
|
||||
// The trigger did not fire when the assistant supplied its output.
|
||||
const injectedTriggerNodeName =
|
||||
triggerNode &&
|
||||
(pinDataPlan.mockDataSources.includes('trigger_input') ||
|
||||
Object.hasOwn(pinDataPlan.verificationPinData, triggerNode.name))
|
||||
? triggerNode.name
|
||||
: undefined;
|
||||
return {
|
||||
...result,
|
||||
...(workflowPinnedNodeNames.length > 0 ? { workflowPinnedNodeNames } : {}),
|
||||
...(injectedTriggerNodeName ? { injectedTriggerNodeName } : {}),
|
||||
};
|
||||
} catch (error) {
|
||||
// A failure to launch (or any other unsettled error) is still an
|
||||
// errored builder run — track it before rethrowing so it isn't
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user