From af009e64a161db7364f96744ca702f391af0ef60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Braulio=20Gonz=C3=A1lez=20Valido?= Date: Thu, 2 Jul 2026 15:17:47 +0100 Subject: [PATCH] fix(ai-builder): Trace eval backend threads and add model input to exec evals CI (no-changelog) (#33458) Co-authored-by: Claude Fable 5 --- .github/workflows/test-evals-instance-ai.yml | 52 ++++++++++++++++--- .../__tests__/langsmith-tracing.test.ts | 40 ++++++++++++++ .../src/tracing/langsmith-tracing.ts | 14 +++-- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-evals-instance-ai.yml b/.github/workflows/test-evals-instance-ai.yml index 023fb045e80..4834ebaeaef 100644 --- a/.github/workflows/test-evals-instance-ai.yml +++ b/.github/workflows/test-evals-instance-ai.yml @@ -53,6 +53,11 @@ on: required: false type: string default: '' + model: + description: 'Model override for all Instance AI agents in the lane backends, provider/model format. Empty = backend default.' + required: false + type: string + default: '' workflow_dispatch: inputs: branch: @@ -79,6 +84,10 @@ on: description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)' required: false default: '' + model: + description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). For experiments: anthropic/claude-sonnet-4-6, anthropic/claude-sonnet-5.' + required: false + default: '' jobs: run-evals: @@ -124,9 +133,14 @@ jobs: N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + # LangSmith creds for backend thread traces, routed to the same + # dedicated project as the eval CLI to keep the 'instance-ai' corpus clean. + LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} + LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }} # Prefer head_ref: inputs.branch may be a merge ref (refs/pull/N/merge). SANDBOX_NAME_PREFIX: evals-ci-${{ github.head_ref || inputs.branch || github.ref_name }} SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} + INSTANCE_AI_MODEL: ${{ inputs.model }} run: | # Build provider-specific env args SANDBOX_ARGS=() @@ -151,6 +165,12 @@ jobs: NETWORK_ARGS+=(--network n8n-eval-net) fi + # Model override for A/B experiments; empty = backend default (instance-ai.config.ts) + MODEL_ARGS=() + if [ -n "$INSTANCE_AI_MODEL" ]; then + MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL="$INSTANCE_AI_MODEL") + fi + IFS=',' read -ra PORTS <<< "$LANE_PORTS" for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" @@ -163,6 +183,11 @@ jobs: -e N8N_AI_ASSISTANT_BASE_URL="" \ -e N8N_INSTANCE_AI_SANDBOX_ENABLED=true \ "${SANDBOX_ARGS[@]}" \ + "${MODEL_ARGS[@]}" \ + -e LANGSMITH_TRACING=true \ + -e LANGSMITH_API_KEY="$LANGSMITH_API_KEY" \ + -e LANGSMITH_ENDPOINT="$LANGSMITH_ENDPOINT" \ + -e LANGSMITH_PROJECT=instance-ai-evals \ -e N8N_LICENSE_ACTIVATION_KEY="$N8N_LICENSE_ACTIVATION_KEY" \ -e N8N_LICENSE_CERT="$N8N_LICENSE_CERT" \ -e N8N_ENCRYPTION_KEY="$N8N_ENCRYPTION_KEY" \ @@ -206,13 +231,14 @@ jobs: }' done - # Belt-and-suspenders: env vars set sandbox config but persisted admin + # Belt-and-suspenders: env vars set sandbox/model config but persisted # settings can override. Per-lane assertion catches env-injection hiccups - # or unexpected DB-side state. A single misconfigured lane would - # silently route some builds through tool mode and pollute results. - - name: Assert sandbox is enabled on every lane + # or unexpected DB-side state. A single misconfigured lane would silently + # route some builds through tool mode or the wrong model and pollute results. + - name: Assert sandbox and model config on every lane env: SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} + INSTANCE_AI_MODEL: ${{ inputs.model }} run: | IFS=',' read -ra PORTS <<< "$LANE_PORTS" bad=0 @@ -232,9 +258,22 @@ jobs: else echo " lane $lane: sandboxEnabled=true sandboxProvider=$SANDBOX_PROVIDER ok" fi + # /preferences returns the effective model name (user pref || config), + # i.e. the name part of provider/model. + if [ -n "$INSTANCE_AI_MODEL" ]; then + effective=$(curl -sf -b "/tmp/cookies-$port.txt" \ + "http://localhost:$port/rest/instance-ai/preferences" \ + | jq -r '.data.modelName') + if [ "$effective" != "${INSTANCE_AI_MODEL##*/}" ]; then + echo "::error::lane $lane (port $port): expected model '${INSTANCE_AI_MODEL##*/}', got '$effective'" + bad=$((bad+1)) + else + echo " lane $lane: model=$effective ok" + fi + fi done if [ "$bad" -gt 0 ]; then - echo "::error::$bad lane(s) misconfigured - eval would mix sandbox + tool-mode builds" + echo "::error::$bad lane(s) misconfigured - eval results would mix configurations" exit 1 fi @@ -291,6 +330,7 @@ jobs: N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} + EVALS_LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} run: | # Layer 2 — defense in depth: explicitly mask each secret's value. # ::add-mask:: is a single-line workflow command. Multi-line secrets @@ -298,7 +338,7 @@ jobs: # a time, otherwise only the first line is registered. for v in "$EVALS_ANTHROPIC_KEY" "$DAYTONA_API_KEY" \ "$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" \ - "$N8N_ENCRYPTION_KEY"; do + "$N8N_ENCRYPTION_KEY" "$EVALS_LANGSMITH_API_KEY"; do [ -z "$v" ] && continue while IFS= read -r line; do [ -n "$line" ] && echo "::add-mask::$line" diff --git a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts index c6991b932ae..bea156100c8 100644 --- a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts +++ b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts @@ -346,6 +346,8 @@ describe('createInstanceAiTraceContext', () => { const originalLangSmithApiKey = process.env.LANGSMITH_API_KEY; const originalLangSmithTracing = process.env.LANGSMITH_TRACING; const originalLangChainTracingV2 = process.env.LANGCHAIN_TRACING_V2; + const originalLangSmithProject = process.env.LANGSMITH_PROJECT; + const originalLangChainProject = process.env.LANGCHAIN_PROJECT; const originalTraceInternal = process.env.N8N_INSTANCE_AI_TRACE_INTERNAL; const originalDiagnosticsEnabled = process.env.N8N_DIAGNOSTICS_ENABLED; @@ -355,6 +357,8 @@ describe('createInstanceAiTraceContext', () => { process.env.LANGSMITH_API_KEY = 'test-key'; delete process.env.LANGSMITH_TRACING; delete process.env.LANGCHAIN_TRACING_V2; + delete process.env.LANGSMITH_PROJECT; + delete process.env.LANGCHAIN_PROJECT; delete process.env.N8N_INSTANCE_AI_TRACE_INTERNAL; delete process.env.N8N_DIAGNOSTICS_ENABLED; }); @@ -371,6 +375,16 @@ describe('createInstanceAiTraceContext', () => { } else { process.env.LANGCHAIN_TRACING_V2 = originalLangChainTracingV2; } + if (originalLangSmithProject === undefined) { + delete process.env.LANGSMITH_PROJECT; + } else { + process.env.LANGSMITH_PROJECT = originalLangSmithProject; + } + if (originalLangChainProject === undefined) { + delete process.env.LANGCHAIN_PROJECT; + } else { + process.env.LANGCHAIN_PROJECT = originalLangChainProject; + } if (originalTraceInternal === undefined) { delete process.env.N8N_INSTANCE_AI_TRACE_INTERNAL; } else { @@ -476,6 +490,32 @@ describe('createInstanceAiTraceContext', () => { expect(process.env.LANGCHAIN_TRACING_V2).toBeUndefined(); }); + it('defaults trace runs to the instance-ai project', async () => { + const tracing = await createInstanceAiTraceContext({ + threadId: 'thread-project-default', + messageId: 'message-project-default', + runId: 'run-project-default', + userId: 'user-project-default', + input: { message: 'hello' }, + }); + + expect(tracing?.rootRun.projectName).toBe('instance-ai'); + }); + + it('routes trace runs to the project from LANGSMITH_PROJECT when set', async () => { + process.env.LANGSMITH_PROJECT = 'instance-ai-evals'; + + const tracing = await createInstanceAiTraceContext({ + threadId: 'thread-project-env', + messageId: 'message-project-env', + runId: 'run-project-env', + userId: 'user-project-env', + input: { message: 'hello' }, + }); + + expect(tracing?.rootRun.projectName).toBe('instance-ai-evals'); + }); + it('uses the current foreground actor run in native telemetry metadata', async () => { const tracing = await createInstanceAiTraceContext({ threadId: 'thread-1', diff --git a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts index d980d4a024c..366f28148cc 100644 --- a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts +++ b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts @@ -652,6 +652,11 @@ function readBooleanEnvFlag(value: string | undefined): boolean | undefined { return undefined; } +// LANGSMITH_PROJECT lets deployments (e.g. eval CI) route product traces off the default project. +function resolveDefaultProjectName(): string { + return process.env.LANGSMITH_PROJECT ?? process.env.LANGCHAIN_PROJECT ?? DEFAULT_PROJECT_NAME; +} + function isLangSmithTracingEnabled(proxyAvailable = false): boolean { if (readBooleanEnvFlag(process.env.N8N_DIAGNOSTICS_ENABLED) === false) { return false; @@ -1637,7 +1642,7 @@ export async function createInstanceAiTraceContext( return undefined; } - const projectName = options.projectName ?? DEFAULT_PROJECT_NAME; + const projectName = options.projectName ?? resolveDefaultProjectName(); const baseMetadata = await buildBaseMetadata(options); const createTraceRuns = async () => { @@ -1699,7 +1704,8 @@ export async function continueInstanceAiTraceContext( } const baseMetadata = await buildBaseMetadata(options); - const projectName = existingContext?.projectName ?? options.projectName ?? DEFAULT_PROJECT_NAME; + const projectName = + existingContext?.projectName ?? options.projectName ?? resolveDefaultProjectName(); const continuedMetadata = existingContext && existingContext.rootRun.traceId !== 'stub' ? { @@ -1773,7 +1779,7 @@ export async function createDetachedSubAgentTraceContext( return undefined; } - const projectName = options.projectName ?? DEFAULT_PROJECT_NAME; + const projectName = options.projectName ?? resolveDefaultProjectName(); const baseMetadata = await buildBaseMetadata(options); const createDetachedRuns = async () => { @@ -1832,7 +1838,7 @@ export async function createInternalOperationTraceContext( return undefined; } - const projectName = options.projectName ?? DEFAULT_PROJECT_NAME; + const projectName = options.projectName ?? resolveDefaultProjectName(); const baseMetadata = await buildBaseMetadata({ ...options, messageId: options.messageId ?? `internal:${options.operationName}:${options.runId}`,