github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-07-28 03:24:59 +02:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity Graph

fix(ai-builder): Trace eval backend threads and add model input to exec evals CI (no-changelog) (#33458)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
José Braulio González Valido 2026-07-02 15:17:47 +01:00 committed by GitHub
parent 41681795b9
commit af009e64a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 96 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
.github/workflows/test-evals-instance-ai.yml vendored
View File

@ -53,6 +53,11 @@ on:
required: false
type: string
default: ''
model:
description: 'Model override for all Instance AI agents in the lane backends, provider/model format. Empty = backend default.'
required: false
type: string
default: ''
workflow_dispatch:
inputs:
branch:
@ -79,6 +84,10 @@ on:
description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)'
required: false
default: ''
model:
description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). For experiments: anthropic/claude-sonnet-4-6, anthropic/claude-sonnet-5.'
required: false
default: ''
jobs:
run-evals:
@ -124,9 +133,14 @@ jobs:
N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
# LangSmith creds for backend thread traces, routed to the same
# dedicated project as the eval CLI to keep the 'instance-ai' corpus clean.
LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
# Prefer head_ref: inputs.branch may be a merge ref (refs/pull/N/merge).
SANDBOX_NAME_PREFIX: evals-ci-${{ github.head_ref || inputs.branch || github.ref_name }}
SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
INSTANCE_AI_MODEL: ${{ inputs.model }}
run: |
# Build provider-specific env args
SANDBOX_ARGS=()
@ -151,6 +165,12 @@ jobs:
NETWORK_ARGS+=(--network n8n-eval-net)
fi
# Model override for A/B experiments; empty = backend default (instance-ai.config.ts)
MODEL_ARGS=()
if [ -n "$INSTANCE_AI_MODEL" ]; then
MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL="$INSTANCE_AI_MODEL")
fi
IFS=',' read -ra PORTS <<< "$LANE_PORTS"
for i in "${!PORTS[@]}"; do
port="${PORTS[$i]}"
@ -163,6 +183,11 @@ jobs:
-e N8N_AI_ASSISTANT_BASE_URL="" \
-e N8N_INSTANCE_AI_SANDBOX_ENABLED=true \
"${SANDBOX_ARGS[@]}" \
"${MODEL_ARGS[@]}" \
-e LANGSMITH_TRACING=true \
-e LANGSMITH_API_KEY="$LANGSMITH_API_KEY" \
-e LANGSMITH_ENDPOINT="$LANGSMITH_ENDPOINT" \
-e LANGSMITH_PROJECT=instance-ai-evals \
-e N8N_LICENSE_ACTIVATION_KEY="$N8N_LICENSE_ACTIVATION_KEY" \
-e N8N_LICENSE_CERT="$N8N_LICENSE_CERT" \
-e N8N_ENCRYPTION_KEY="$N8N_ENCRYPTION_KEY" \
@ -206,13 +231,14 @@ jobs:
}'
done
# Belt-and-suspenders: env vars set sandbox config but persisted admin
# Belt-and-suspenders: env vars set sandbox/model config but persisted
# settings can override. Per-lane assertion catches env-injection hiccups
# or unexpected DB-side state. A single misconfigured lane would
# silently route some builds through tool mode and pollute results.
- name: Assert sandbox is enabled on every lane
# or unexpected DB-side state. A single misconfigured lane would silently
# route some builds through tool mode or the wrong model and pollute results.
- name: Assert sandbox and model config on every lane
env:
SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
INSTANCE_AI_MODEL: ${{ inputs.model }}
run: |
IFS=',' read -ra PORTS <<< "$LANE_PORTS"
bad=0
@ -232,9 +258,22 @@ jobs:
else
echo " lane $lane: sandboxEnabled=true sandboxProvider=$SANDBOX_PROVIDER ok"
fi
# /preferences returns the effective model name (user pref || config),
# i.e. the name part of provider/model.
if [ -n "$INSTANCE_AI_MODEL" ]; then
effective=$(curl -sf -b "/tmp/cookies-$port.txt" \
"http://localhost:$port/rest/instance-ai/preferences" \
| jq -r '.data.modelName')
if [ "$effective" != "${INSTANCE_AI_MODEL##*/}" ]; then
echo "::error::lane $lane (port $port): expected model '${INSTANCE_AI_MODEL##*/}', got '$effective'"
bad=$((bad+1))
else
echo " lane $lane: model=$effective ok"
fi
fi
done
if [ "$bad" -gt 0 ]; then
echo "::error::$bad lane(s) misconfigured - eval would mix sandbox + tool-mode builds"
echo "::error::$bad lane(s) misconfigured - eval results would mix configurations"
exit 1
fi
@ -291,6 +330,7 @@ jobs:
N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
EVALS_LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
run: |
# Layer 2 — defense in depth: explicitly mask each secret's value.
# ::add-mask:: is a single-line workflow command. Multi-line secrets
@ -298,7 +338,7 @@ jobs:
# a time, otherwise only the first line is registered.
for v in "$EVALS_ANTHROPIC_KEY" "$DAYTONA_API_KEY" \
"$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" \
"$N8N_ENCRYPTION_KEY"; do
"$N8N_ENCRYPTION_KEY" "$EVALS_LANGSMITH_API_KEY"; do
[ -z "$v" ] && continue
while IFS= read -r line; do
[ -n "$line" ] && echo "::add-mask::$line"

40
packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts
View File

@ -346,6 +346,8 @@ describe('createInstanceAiTraceContext', () => {
const originalLangSmithApiKey = process.env.LANGSMITH_API_KEY;
const originalLangSmithTracing = process.env.LANGSMITH_TRACING;
const originalLangChainTracingV2 = process.env.LANGCHAIN_TRACING_V2;
const originalLangSmithProject = process.env.LANGSMITH_PROJECT;
const originalLangChainProject = process.env.LANGCHAIN_PROJECT;
const originalTraceInternal = process.env.N8N_INSTANCE_AI_TRACE_INTERNAL;
const originalDiagnosticsEnabled = process.env.N8N_DIAGNOSTICS_ENABLED;
@ -355,6 +357,8 @@ describe('createInstanceAiTraceContext', () => {
process.env.LANGSMITH_API_KEY = 'test-key';
delete process.env.LANGSMITH_TRACING;
delete process.env.LANGCHAIN_TRACING_V2;
delete process.env.LANGSMITH_PROJECT;
delete process.env.LANGCHAIN_PROJECT;
delete process.env.N8N_INSTANCE_AI_TRACE_INTERNAL;
delete process.env.N8N_DIAGNOSTICS_ENABLED;
});
@ -371,6 +375,16 @@ describe('createInstanceAiTraceContext', () => {
} else {
process.env.LANGCHAIN_TRACING_V2 = originalLangChainTracingV2;
}
if (originalLangSmithProject === undefined) {
delete process.env.LANGSMITH_PROJECT;
} else {
process.env.LANGSMITH_PROJECT = originalLangSmithProject;
}
if (originalLangChainProject === undefined) {
delete process.env.LANGCHAIN_PROJECT;
} else {
process.env.LANGCHAIN_PROJECT = originalLangChainProject;
}
if (originalTraceInternal === undefined) {
delete process.env.N8N_INSTANCE_AI_TRACE_INTERNAL;
} else {
@ -476,6 +490,32 @@ describe('createInstanceAiTraceContext', () => {
expect(process.env.LANGCHAIN_TRACING_V2).toBeUndefined();
});
it('defaults trace runs to the instance-ai project', async () => {
const tracing = await createInstanceAiTraceContext({
threadId: 'thread-project-default',
messageId: 'message-project-default',
runId: 'run-project-default',
userId: 'user-project-default',
input: { message: 'hello' },
});
expect(tracing?.rootRun.projectName).toBe('instance-ai');
});
it('routes trace runs to the project from LANGSMITH_PROJECT when set', async () => {
process.env.LANGSMITH_PROJECT = 'instance-ai-evals';
const tracing = await createInstanceAiTraceContext({
threadId: 'thread-project-env',
messageId: 'message-project-env',
runId: 'run-project-env',
userId: 'user-project-env',
input: { message: 'hello' },
});
expect(tracing?.rootRun.projectName).toBe('instance-ai-evals');
});
it('uses the current foreground actor run in native telemetry metadata', async () => {
const tracing = await createInstanceAiTraceContext({
threadId: 'thread-1',

14
packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts
View File

@ -652,6 +652,11 @@ function readBooleanEnvFlag(value: string | undefined): boolean | undefined {
return undefined;
}
// LANGSMITH_PROJECT lets deployments (e.g. eval CI) route product traces off the default project.
function resolveDefaultProjectName(): string {
return process.env.LANGSMITH_PROJECT ?? process.env.LANGCHAIN_PROJECT ?? DEFAULT_PROJECT_NAME;
}
function isLangSmithTracingEnabled(proxyAvailable = false): boolean {
if (readBooleanEnvFlag(process.env.N8N_DIAGNOSTICS_ENABLED) === false) {
return false;
@ -1637,7 +1642,7 @@ export async function createInstanceAiTraceContext(
return undefined;
}
const projectName = options.projectName ?? DEFAULT_PROJECT_NAME;
const projectName = options.projectName ?? resolveDefaultProjectName();
const baseMetadata = await buildBaseMetadata(options);
const createTraceRuns = async () => {
@ -1699,7 +1704,8 @@ export async function continueInstanceAiTraceContext(
}
const baseMetadata = await buildBaseMetadata(options);
const projectName = existingContext?.projectName ?? options.projectName ?? DEFAULT_PROJECT_NAME;
const projectName =
existingContext?.projectName ?? options.projectName ?? resolveDefaultProjectName();
const continuedMetadata =
existingContext && existingContext.rootRun.traceId !== 'stub'
? {
@ -1773,7 +1779,7 @@ export async function createDetachedSubAgentTraceContext(
return undefined;
}
const projectName = options.projectName ?? DEFAULT_PROJECT_NAME;
const projectName = options.projectName ?? resolveDefaultProjectName();
const baseMetadata = await buildBaseMetadata(options);
const createDetachedRuns = async () => {
@ -1832,7 +1838,7 @@ export async function createInternalOperationTraceContext(
return undefined;
}
const projectName = options.projectName ?? DEFAULT_PROJECT_NAME;
const projectName = options.projectName ?? resolveDefaultProjectName();
const baseMetadata = await buildBaseMetadata({
...options,
messageId: options.messageId ?? `internal:${options.operationName}:${options.runId}`,