name: 'Instance AI Evals: Experiments' run-name: "Instance AI Evals: ${{ inputs.branch || 'master' }} (tier=${{ inputs.tier || 'all' }}, iterations=${{ inputs.iterations }}, model=${{ inputs.model || 'default' }}, experiment=${{ inputs['experiment-name'] || '-' }})" on: workflow_call: inputs: branch: description: 'GitHub branch to test' required: false type: string default: 'master' filter: description: 'Filter test cases by filename substring (e.g. "contact-form")' required: false type: string default: '' tier: description: 'Test-case dataset to run (e.g. "pr", "full"). Empty = no filter.' required: false type: string default: '' suite: description: 'LangTracer suite slug or id to pull test cases from.' required: false type: string default: 'baseline' sandbox-provider: description: 'Sandbox provider (n8n-sandbox or daytona)' required: false type: string default: 'n8n-sandbox' iterations: description: 'Iterations per test case' required: false type: string default: '3' experiment-name: description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)' required: false type: string default: '' pr-number: description: 'PR number to post results back to. Empty = derive from the pull_request event.' required: false type: string default: '' cache-sha: description: 'SHA for the docker image cache key. Empty = github.sha. Set to the tested commit on dispatch re-runs so the image matches the code under test.' required: false type: string default: '' revision-sha: description: 'Commit SHA under test, for LangSmith revision tagging. Empty = github.sha.' required: false type: string default: '' head-ref: description: 'Branch name under test, for LangSmith branch tagging. Empty = derive from context.' required: false type: string default: '' model: description: 'Model override for all Instance AI agents in the lane backends, provider/model format. Empty = backend default.' required: false type: string default: '' workflow_dispatch: inputs: branch: description: 'GitHub branch to test' required: false default: 'master' filter: description: 'Filter test cases by filename substring (e.g. "contact-form")' required: false default: '' tier: description: 'Test-case dataset to run (e.g. "pr", "full"). Empty = no filter.' required: false default: '' suite: description: 'LangTracer suite slug or id to pull test cases from.' required: false default: 'baseline' sandbox-provider: description: 'Sandbox provider (n8n-sandbox or daytona)' required: false default: 'n8n-sandbox' iterations: description: 'Iterations per test case (use 10 for a baseline)' required: false default: '3' experiment-name: description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)' required: false default: '' model: description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). For experiments: anthropic/claude-sonnet-4-6, anthropic/claude-sonnet-5.' required: false default: '' jobs: run-evals: name: 'Run Evals' # 8vcpu/32GB: full-suite N=10 baselines need ~150 min of eval time and # did not fit the 4vcpu tier at any timeout. runs-on: blacksmith-8vcpu-ubuntu-2204 # Long ceiling only for high-N baseline captures; regular runs keep a # tight guard so a wedged run fails fast. Plain comparison (not fromJSON): # it coerces the string input to a number, and a malformed dispatch value # (NaN) falls back to the 90-minute guard instead of erroring the job. timeout-minutes: ${{ (inputs.iterations || '3') >= 5 && 240 || 90 }} env: # Each port hosts an independent n8n container. The eval CLI's # work-stealing allocator dispatches builds across them, capped per-lane. # 10 lanes x 2.5 GB caps = 25 GB, leaving ~7 GB for sandbox + CLI + OS. LANE_PORTS: '5678,5679,5680,5681,5682,5683,5684,5685,5686,5687' permissions: contents: read pull-requests: write steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.branch || github.ref }} fetch-depth: 1 - name: Setup Environment uses: ./.github/actions/setup-nodejs with: build-command: 'pnpm build' # Cache populated by prepare-docker; on a miss (e.g. fresh PR-open runs # that outpace prepare-docker) the action itself falls back to a rebuild # via build-n8n-docker. - name: Load n8n Docker image uses: ./.github/actions/load-n8n-docker with: cache-sha: ${{ inputs.cache-sha }} # Host OOMs and disk exhaustion leave no application trace — sample host # + container telemetry every 60s and ship it with the results artifact. # # `docker stats` names a hot lane but not the process inside it, and `comm` # can't (Node reports `MainThread`). Sorted in the shell — `docker top` parses # the ps output it asks for and the runner's ps rejects `--sort`. - name: Start host telemetry sampler run: | mkdir -p eval-diag { echo "=== disk at job start ==="; df -h /; docker system df; } > eval-diag/df-at-start.log # shellcheck disable=SC2016 # deferred on purpose: the sampler expands these nohup bash -c 'while true; do { date -u +%FT%TZ free -m | head -2 df -h / | tail -1 docker stats --no-stream --format "{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}" ps -eo rss=,pid=,comm= --sort=-rss | head -5 for c in $(docker ps --format "{{.Names}}" --filter name=n8n-eval-); do docker top "$c" -o pid,pcpu,args 2>&1 | tail -n +2 | sort -k2 -rn | head -2 | sed "s|^|$c |" | cut -c1-150 done echo } >> eval-diag/host-samples.log 2>&1 sleep 60 done' > /dev/null 2>&1 & echo $! > eval-diag/sampler.pid - name: Start sandbox service if: ${{ inputs.sandbox-provider == 'n8n-sandbox' }} run: pnpm --filter n8n-containers services --services sandbox --network n8n-eval-net --name n8n-svc-sandbox - name: Start n8n containers env: EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} INSTANCE_AI_BRAVE_SEARCH_API_KEY: ${{ secrets.INSTANCE_AI_BRAVE_SEARCH_API_KEY }} N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} # LangSmith creds for backend thread traces, routed to the same # dedicated project as the eval CLI to keep the 'instance-ai' corpus clean. LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }} # Prefer head_ref: inputs.branch may be a merge ref (refs/pull/N/merge). SANDBOX_NAME_PREFIX: evals-ci-${{ github.head_ref || inputs.branch || github.ref_name }} SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} INSTANCE_AI_MODEL: ${{ inputs.model }} run: | # Build provider-specific env args SANDBOX_ARGS=() if [ "$SANDBOX_PROVIDER" = "daytona" ]; then SANDBOX_ARGS+=( -e N8N_INSTANCE_AI_SANDBOX_PROVIDER=daytona -e N8N_INSTANCE_AI_SANDBOX_NAME_PREFIX="$SANDBOX_NAME_PREFIX" -e DAYTONA_API_URL=https://app.daytona.io/api -e DAYTONA_API_KEY="$DAYTONA_API_KEY" ) else SANDBOX_ARGS+=( -e N8N_INSTANCE_AI_SANDBOX_PROVIDER=n8n-sandbox -e N8N_SANDBOX_SERVICE_URL=http://sandbox-api:8080 -e N8N_SANDBOX_SERVICE_API_KEY=n8n-sandbox-ci-key ) fi # Use the eval network when sandbox service is running NETWORK_ARGS=() if [ "$SANDBOX_PROVIDER" == "n8n-sandbox" ]; then NETWORK_ARGS+=(--network n8n-eval-net) fi # Model override for A/B experiments; empty = backend default (instance-ai.config.ts) MODEL_ARGS=() if [ -n "$INSTANCE_AI_MODEL" ]; then MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL="$INSTANCE_AI_MODEL") fi IFS=',' read -ra PORTS <<< "$LANE_PORTS" for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" # Bounded and self-healing: a lane that exhausts its capped heap is # restarted by docker instead of staying dead; pruning + log caps # keep per-lane disk and memory flat over a multi-hour run. # # Arms Node's diagnostic report on SIGUSR2 (requested in the log-capture # step). Keep flag and signal in sync — unarmed, SIGUSR2 kills the process. # --report-exclude-env is REQUIRED: the report embeds # `environmentVariables` by default, and this container's env holds the # Anthropic, LangSmith and licence secrets. `::add-mask::` only redacts # log output, never an uploaded artifact, so without this the report # would carry them into the run's artifacts verbatim. docker run -d --name "n8n-eval-$((i+1))" \ "${NETWORK_ARGS[@]}" \ --memory 2.5g --memory-swap 2.5g \ --restart on-failure \ --log-opt max-size=50m --log-opt max-file=2 \ -e NODE_OPTIONS="--max-old-space-size=2048 --report-on-signal --report-signal=SIGUSR2 --report-directory=/tmp --report-exclude-env" \ -e EXECUTIONS_DATA_PRUNE=true \ -e EXECUTIONS_DATA_MAX_AGE=1 \ -e E2E_TESTS=true \ -e N8N_ENABLED_MODULES=instance-ai \ -e N8N_AI_ENABLED=true \ -e N8N_INSTANCE_AI_MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" \ -e INSTANCE_AI_BRAVE_SEARCH_API_KEY="$INSTANCE_AI_BRAVE_SEARCH_API_KEY" \ -e N8N_AI_ASSISTANT_BASE_URL="" \ -e N8N_INSTANCE_AI_SANDBOX_ENABLED=true \ "${SANDBOX_ARGS[@]}" \ "${MODEL_ARGS[@]}" \ -e LANGSMITH_TRACING=true \ -e LANGSMITH_API_KEY="$LANGSMITH_API_KEY" \ -e LANGSMITH_ENDPOINT="$LANGSMITH_ENDPOINT" \ -e LANGSMITH_PROJECT=instance-ai-evals \ -e N8N_LICENSE_ACTIVATION_KEY="$N8N_LICENSE_ACTIVATION_KEY" \ -e N8N_LICENSE_CERT="$N8N_LICENSE_CERT" \ -e N8N_ENCRYPTION_KEY="$N8N_ENCRYPTION_KEY" \ -p "$port:5678" \ n8nio/n8n:local done # 120s budget per port: containers booting in parallel on a shared # runner contend for CPU/disk during n8n's startup (DB migrations, # license init), so each takes longer than a solo boot. for port in "${PORTS[@]}"; do ready=false for i in $(seq 1 120); do if curl -s "http://localhost:$port/healthz/readiness" -o /dev/null -w "%{http_code}" | grep -q 200; then echo "n8n on port $port ready after ${i}s" ready=true break fi sleep 1 done if [ "$ready" != "true" ]; then echo "::error::n8n on port $port failed to start within 120s" for n in $(docker ps -aq --filter "name=n8n-eval-"); do echo "Logs for $n:" docker logs "$n" --tail 30 || true done exit 1 fi done - name: Create test users run: | IFS=',' read -ra PORTS <<< "$LANE_PORTS" for port in "${PORTS[@]}"; do curl -sf -X POST "http://localhost:$port/rest/e2e/reset" \ -H "Content-Type: application/json" \ -d '{ "owner":{"email":"nathan@n8n.io","password":"PlaywrightTest123","firstName":"Eval","lastName":"Owner"}, "admin":{"email":"admin@n8n.io","password":"PlaywrightTest123","firstName":"Admin","lastName":"User"}, "members":[], "chat":{"email":"chat@n8n.io","password":"PlaywrightTest123","firstName":"Chat","lastName":"User"} }' done # Belt-and-suspenders: env vars set sandbox/model config but persisted # settings can override. Per-lane assertion catches env-injection hiccups # or unexpected DB-side state. A single misconfigured lane would silently # route some builds through tool mode or the wrong model and pollute results. - name: Assert sandbox and model config on every lane env: SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }} INSTANCE_AI_MODEL: ${{ inputs.model }} run: | IFS=',' read -ra PORTS <<< "$LANE_PORTS" bad=0 for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" lane="$((i+1))" curl -sf -X POST "http://localhost:$port/rest/login" \ -H "Content-Type: application/json" \ -d '{"emailOrLdapLoginId":"nathan@n8n.io","password":"PlaywrightTest123"}' \ -c "/tmp/cookies-$port.txt" -o /dev/null cfg=$(curl -sf -b "/tmp/cookies-$port.txt" \ "http://localhost:$port/rest/instance-ai/settings" \ | jq -r '.data | "\(.sandboxEnabled) \(.sandboxProvider)"') if [ "$cfg" != "true $SANDBOX_PROVIDER" ]; then echo "::error::lane $lane (port $port): expected 'true $SANDBOX_PROVIDER', got '$cfg'" bad=$((bad+1)) else echo " lane $lane: sandboxEnabled=true sandboxProvider=$SANDBOX_PROVIDER ok" fi # /preferences returns the effective model name (user pref || config), # i.e. the name part of provider/model. if [ -n "$INSTANCE_AI_MODEL" ]; then effective=$(curl -sf -b "/tmp/cookies-$port.txt" \ "http://localhost:$port/rest/instance-ai/preferences" \ | jq -r '.data.modelName') if [ "$effective" != "${INSTANCE_AI_MODEL##*/}" ]; then echo "::error::lane $lane (port $port): expected model '${INSTANCE_AI_MODEL##*/}', got '$effective'" bad=$((bad+1)) else echo " lane $lane: model=$effective ok" fi fi done if [ "$bad" -gt 0 ]; then echo "::error::$bad lane(s) misconfigured - eval results would mix configurations" exit 1 fi # Disk pressure develops during the eval (per-lane SQLite growth, # verifier snapshots, logs), not at job start — so the runway check # lives here, after the image load and lane startup, where "is there # enough free disk for the next few hours" is a meaningful question. # A run-A-style on-runner fallback build leaves tens of GB of builder # cache that is safe to reclaim once every lane is up. - name: Ensure disk runway for the eval run shell: bash run: | free_gb=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') if [ "${free_gb:-0}" -lt 30 ]; then echo "Only ${free_gb}GB free — reclaiming builder cache before the run accretes data" # Best-effort: a prune failure must not skip the eval run itself. docker builder prune -f || echo "::warning::builder prune failed — continuing with ${free_gb}GB free" df -h / || true else echo "Disk runway OK: ${free_gb}GB free" fi - name: Run Instance AI Evals continue-on-error: true working-directory: packages/@n8n/instance-ai env: N8N_INSTANCE_AI_MODEL_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} LANGSMITH_TRACING: 'true' LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }} LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} LANGSMITH_REVISION_ID: ${{ inputs.revision-sha || github.sha }} LANGSMITH_BRANCH: ${{ inputs.head-ref || github.event.pull_request.head.ref || github.head_ref || github.ref_name }} # Dedicated project so eval build traces don't pollute the default 'instance-ai' corpus. LANGSMITH_PROJECT: instance-ai-evals FILTER: ${{ inputs.filter }} TIER: ${{ inputs.tier }} SUITE: ${{ inputs.suite }} # LangTracer is the test-case source of truth (TRUST-247); the CLI # pulls the suite per run via its export API. LANGTRACER_URL: ${{ secrets.EVALS_LANGTRACER_URL }} LANGTRACER_API_KEY: ${{ secrets.LANGTRACER_API_KEY }} ITERATIONS: ${{ inputs.iterations }} EXPERIMENT_NAME: ${{ inputs.experiment-name }} EVAL_PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }} run: | IFS=',' read -ra PORTS <<< "$LANE_PORTS" URLS=() for port in "${PORTS[@]}"; do URLS+=("http://localhost:$port") done BASE_URLS=$(IFS=,; printf '%s' "${URLS[*]}") ARGS=(--base-url "$BASE_URLS" --concurrency 32 --verbose --iterations "${ITERATIONS:-3}") # LangTracer is the only CI case source (no disk fallback by design). ARGS+=(--source langtracer --suite "${SUITE:-baseline}") # Pin the LangSmith cohort: langtracer mode otherwise derives a # suite-scoped dataset/baseline prefix, which would fork the KPI # history and orphan the baseline comparison. ARGS+=(--dataset instance-ai-workflow-evals --baseline-prefix instance-ai-baseline-) [ -n "$FILTER" ] && ARGS+=(--filter "$FILTER") [ -n "$TIER" ] && ARGS+=(--tier "$TIER") [ -n "$EXPERIMENT_NAME" ] && ARGS+=(--experiment-name "$EXPERIMENT_NAME") pnpm eval:instance-ai "${ARGS[@]}" # Captures sandbox/builder diagnostic signals that surface during the # eval (after migrations finish). Two layers of secret-leak defense: # # 1. Filter to specific diagnostic patterns — never tail raw output. # The grep allowlist scopes the log surface to lines we care # about for debugging (sandbox lifecycle, builder, errors). # # 2. Re-register secrets via ::add-mask:: so any line that does # match the allowlist has the secret values replaced with *** # before reaching the GH Actions log. GitHub auto-masks # ${{ secrets.X }} references, but the masking is fragile # against transformed or split values; explicit registration # reinforces it. # # Runs even on eval failure so we have the post-mortem regardless. - name: Capture n8n container logs (debug) if: ${{ always() }} env: EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }} N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }} N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} EVALS_LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} run: | # Layer 2 — defense in depth: explicitly mask each secret's value. # ::add-mask:: is a single-line workflow command. Multi-line secrets # (e.g. N8N_LICENSE_CERT is PEM-encoded) must be masked one line at # a time, otherwise only the first line is registered. for v in "$EVALS_ANTHROPIC_KEY" "$DAYTONA_API_KEY" \ "$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" \ "$N8N_ENCRYPTION_KEY" "$EVALS_LANGSMITH_API_KEY"; do [ -z "$v" ] && continue while IFS= read -r line; do [ -n "$line" ] && echo "::add-mask::$line" done <<< "$v" done # A spin leaves no log line, so snapshot the processes and ask for a JS # stack. Report is collected at the end of the step, once Node can write it. HOT_LANES="" while read -r name cpu; do [ -n "$name" ] || continue pct="${cpu%\%}" pct="${pct%.*}" # `docker stats` can emit a non-numeric CPUPerc (e.g. mid restart). With # `-e`/pipefail an arithmetic compare on that aborts the whole step — and # this step exists precisely to report on adverse conditions. case "$pct" in ''|*[!0-9]*) continue ;; esac [ "$pct" -ge 70 ] || continue HOT_LANES="$HOT_LANES $name" echo "" echo "=== $name at $cpu — per-process snapshot ===" docker top "$name" -o pid,ppid,pcpu,etime,args 2>&1 | cut -c1-200 || true docker kill -s SIGUSR2 "$name" >/dev/null 2>&1 || true done <<< "$(docker stats --no-stream --format '{{.Name}} {{.CPUPerc}}' 2>/dev/null | grep '^n8n-eval-' || true)" # Layer 1 — accuracy filter: only surface diagnostic signals. # `tail -1000` after the filter so we get the LATEST matching lines # (post-eval failure signal), not the earliest startup-time ones. # -t keeps timestamps; the inspect line surfaces OOM kills/restarts, # which leave no log line of their own. SIGNALS='sandbox|builder|sandbox-service|daytona|instance.?ai|error|warn|reject|exception|fail' for c in $(docker ps -aq --filter "name=n8n-eval-"); do name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||') echo "" echo "============================================================" echo "=== $name (filtered diagnostic signals, last 1000 lines) ===" echo "============================================================" docker inspect --format 'state: status={{.State.Status}} oomkilled={{.State.OOMKilled}} exitcode={{.State.ExitCode}} restarts={{.RestartCount}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' "$c" || true docker logs -t "$c" 2>&1 \ | grep -ivE 'migration' \ | grep -iE "$SIGNALS" \ | tail -1000 \ || true done # Sandbox service container logs (when using n8n-sandbox provider) for c in $(docker ps -aq --filter "label=com.docker.compose.project=n8n-svc-sandbox"); do name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||') echo "" echo "============================================================" echo "=== $name (last 100 lines) ===" echo "============================================================" docker logs "$c" 2>&1 | tail -100 || true done for name in $HOT_LANES; do docker exec "$name" sh -c 'cat /tmp/report.*.json 2>/dev/null' \ > "eval-diag/node-report-$name.json" 2>/dev/null || true if [ -s "eval-diag/node-report-$name.json" ]; then echo "captured Node diagnostic report for $name" else rm -f "eval-diag/node-report-$name.json" echo "no Node diagnostic report from $name (loop never freed up)" fi done - name: Stop n8n containers if: ${{ always() }} run: | [ -f eval-diag/sampler.pid ] && kill "$(cat eval-diag/sampler.pid)" 2>/dev/null || true mapfile -t ids < <(docker ps -aq --filter "name=n8n-eval-") if [ "${#ids[@]}" -gt 0 ]; then docker stop "${ids[@]}" 2>/dev/null || true docker rm "${ids[@]}" 2>/dev/null || true fi # Sandbox service cleanup pnpm --filter n8n-containers services:clean 2>/dev/null || true docker network rm n8n-eval-net 2>/dev/null || true - name: Post eval results to PR if: ${{ always() && (inputs.pr-number || github.event.pull_request.number) }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }} run: | # The eval CLI writes the full PR comment as eval-pr-comment.md # (see comparison/format.ts:formatComparisonMarkdown). It includes # the alert, aggregate, comparison sections, per-test-case results # collapsed, and failure details collapsed. CI just relays it. COMMENT_FILE="packages/@n8n/instance-ai/eval-pr-comment.md" if [ ! -f "$COMMENT_FILE" ]; then echo "No PR comment file found (eval likely cancelled before writing results)" exit 0 fi cp "$COMMENT_FILE" /tmp/eval-comment.md # Find and update existing eval comment, or create new one COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \ --jq '.[] | select(.body | startswith("### Instance AI Workflow Eval")) | .id' | tail -1) if [ -n "$COMMENT_ID" ]; then gh api "repos/$REPO/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md else gh pr comment "$PR_NUMBER" --body-file /tmp/eval-comment.md fi - name: Upload Results if: ${{ always() }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: instance-ai-workflow-eval-results path: | packages/@n8n/instance-ai/eval-results.json packages/@n8n/instance-ai/.data/workflow-eval-report.html eval-diag/ retention-days: 14 # The eval step runs with continue-on-error so the logs/artifacts above # always land — but a run that never wrote eval-results.json (bad secret, # unresolvable LangTracer suite, CLI crash before any results) must still # fail the job rather than end green with zero evals. Per-case build # failures DO write results and stay green by design. Mirrors the same # guard in test-evals-mcp.yml. - name: Fail when no results were produced if: ${{ always() }} working-directory: packages/@n8n/instance-ai run: | if [ ! -f eval-results.json ]; then echo "::error::Eval run produced no eval-results.json — it failed before writing any results. Check the 'Run Instance AI Evals' step logs and the uploaded artifacts." exit 1 fi