n8n/.github/workflows/test-evals-mcp.yml

name: 'Test: MCP Workflow Evals'

# Scores workflows built through the instance MCP server (packages/cli/src/modules/mcp)
# with the Instance AI verifier, using the fused `--build-via-mcp` lane model:
#
#   * N independent n8n containers ("lanes") run on one runner.
#   * A single `eval:instance-ai --build-via-mcp` process drives `claude -p`
#     against each lane's own MCP server to build a workflow, then verifies it on
#     that same lane (mock-execution + verifier). The work-stealing allocator
#     spreads builds across lanes, capped per-lane.
#
# One process → one LangSmith experiment (no manifest hop, no shard/merge). MCP
# builds AST-parse SDK code server-side (no sandbox needed); the builder is the
# `claude` CLI, and verification runs on the execution engine.
on:
  workflow_call:
    inputs:
      branch:
        description: 'Git ref to test. Defaults to the caller ref when empty.'
        required: false
        type: string
        default: ''
      tier:
        description: 'Test-case dataset to build + eval (e.g. "mcp")'
        required: false
        type: string
        default: 'mcp'
      suite:
        description: 'LangTracer suite slug or id to pull test cases from.'
        required: false
        type: string
        default: 'n8n-workflows'
      filter:
        description: 'Only build + eval test cases whose slug matches (comma-separated; empty = whole tier)'
        required: false
        type: string
        default: ''
      lanes:
        description: 'Number of parallel n8n lanes (containers) on the runner'
        required: false
        type: string
        default: '6'
      iterations:
        description: 'Builds per test case + verifier iterations (1-5; 5 = high-confidence).'
        required: false
        type: string
        default: '3'
      eval-concurrency:
        description: 'Concurrent scenario executions in the verifier (empty = lanes * 2). Raise it to push throughput, but too high starves the shared Anthropic budget and causes verifier/MCP timeouts.'
        required: false
        type: string
        default: ''
      build-model:
        description: 'Anthropic model id for the claude MCP build step (injected as ANTHROPIC_MODEL)'
        required: false
        type: string
        default: 'claude-opus-4-8'
      experiment-name:
        description: 'LangSmith experiment name (mcp-baseline refreshes the baseline)'
        required: false
        type: string
        default: ''
    # Declared explicitly (not `secrets: inherit`) so the caller passes only what
    # this workflow needs. All optional — a missing secret resolves to "" (e.g.
    # LangSmith recording is simply skipped without a key).
    secrets:
      EVALS_ANTHROPIC_KEY:
        required: false
      N8N_LICENSE_ACTIVATION_KEY:
        required: false
      N8N_LICENSE_CERT:
        required: false
      N8N_ENCRYPTION_KEY:
        required: false
      EVALS_LANGSMITH_ENDPOINT:
        required: false
      EVALS_LANGSMITH_API_KEY:
        required: false
      # The eval CLI pulls the test-case suite from LangTracer.
      EVALS_LANGTRACER_URL:
        required: false
      LANGTRACER_API_KEY:
        required: false
  workflow_dispatch:
    inputs:
      branch:
        description: 'Git ref to test. Defaults to the dispatched ref when empty.'
        required: false
        default: ''
      tier:
        description: 'Test-case dataset to build + eval (e.g. "mcp")'
        required: false
        default: 'mcp'
      suite:
        description: 'LangTracer suite slug or id to pull test cases from.'
        required: false
        default: 'n8n-workflows'
      filter:
        description: 'Only build + eval test cases whose slug matches (comma-separated; empty = whole tier)'
        required: false
        default: ''
      lanes:
        description: 'Number of parallel n8n lanes (containers) on the runner'
        required: false
        default: '6'
      iterations:
        description: 'Builds per test case + verifier iterations (1-5; 5 = high-confidence).'
        required: false
        type: choice
        options:
          - '1'
          - '3'
          - '5'
        default: '3'
      eval-concurrency:
        description: 'Concurrent scenario executions in the verifier (empty = lanes * 2). Raise it to push throughput, but too high starves the shared Anthropic budget and causes verifier/MCP timeouts.'
        required: false
        default: ''
      build-model:
        description: 'Anthropic model id for the claude MCP build step (injected as ANTHROPIC_MODEL)'
        required: false
        default: 'claude-opus-4-8'
      experiment-name:
        description: 'LangSmith experiment name (set to mcp-baseline to refresh the baseline)'
        required: false
        default: ''

jobs:
  run-evals:
    name: 'Run MCP Evals'
    runs-on: blacksmith-4vcpu-ubuntu-2204
    # Generous cap: a full-tier baseline run (--iterations 5, the high-confidence
    # maximum both dispatch forms offer) is ~350+ build+verify units and, with the
    # heaviest cases, runs a few hours. Lowering it per-dispatch isn't possible,
    # so size for the baseline case.
    timeout-minutes: 240
    permissions:
      contents: read

    steps:
      - name: Checkout
        uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1
        with:
          ref: ${{ inputs.branch || github.ref }}
          fetch-depth: 1
          # This job uploads artifacts and needs no token post-checkout — drop the
          # persisted credential so it can't leak (zizmor: artipacked).
          persist-credentials: false

      - name: Setup Environment
        uses: ./.github/actions/setup-nodejs
        with:
          build-command: 'pnpm build'

      # The eval CLI drives the standalone `claude` CLI as a subprocess per lane
      # (see cli/mcp-builder.ts). It reaches each lane's MCP server over that
      # lane's published port on the host.
      - name: Install Claude Code CLI
        run: |
          npm install -g @anthropic-ai/claude-code
          command -v claude || { echo "::error::claude CLI not on PATH after install"; exit 1; }
          # Force-trust MCP so `claude -p` runs headless (mirrors the official
          # claude-code-action's enableAllProjectMcpServers). The eval CLI stages a
          # per-lane --mcp-config; this only handles the trust prompt.
          mkdir -p "$HOME/.claude"
          echo '{ "enableAllProjectMcpServers": true }' > "$HOME/.claude/settings.json"          

      # Cache populated by prepare-docker; on a miss the action falls back to a
      # rebuild via build-n8n-docker (manual/scheduled runs usually rebuild).
      - name: Load n8n Docker image
        uses: ./.github/actions/load-n8n-docker

      - name: Start n8n lanes
        id: lanes
        env:
          LANES: ${{ inputs.lanes || '6' }}
          EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
          N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
        run: |
          # Contiguous ports from 5678; the range 5678..5688 avoids Node fetch()'s
          # blocked-port list, so up to 11 lanes are safe.
          PORTS=()
          for i in $(seq 0 $((LANES - 1))); do
            PORTS+=($((5678 + i)))
          done
          # instance-ai serves the verifier; mcp + mcp-registry are default modules,
          # so MCP is available (the eval CLI enables the *setting* per lane after
          # reset). No sandbox — the builder is `claude`, not the in-n8n agent.
          for i in "${!PORTS[@]}"; do
            port="${PORTS[$i]}"
            docker run -d --name "n8n-eval-mcp-$((i + 1))" \
              -e E2E_TESTS=true \
              -e N8N_ENABLED_MODULES=instance-ai \
              -e N8N_AI_ENABLED=true \
              -e N8N_INSTANCE_AI_MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" \
              -e N8N_AI_ASSISTANT_BASE_URL="" \
              -e N8N_LICENSE_ACTIVATION_KEY="$N8N_LICENSE_ACTIVATION_KEY" \
              -e N8N_LICENSE_CERT="$N8N_LICENSE_CERT" \
              -e N8N_ENCRYPTION_KEY="$N8N_ENCRYPTION_KEY" \
              -p "$port:5678" \
              n8nio/n8n:local
          done

          # 120s budget per port: containers booting in parallel on a shared 4vcpu
          # runner contend for CPU/disk during startup (DB migrations, license init).
          for port in "${PORTS[@]}"; do
            ready=false
            for i in $(seq 1 120); do
              if curl -s "http://localhost:$port/healthz/readiness" -o /dev/null -w "%{http_code}" | grep -q 200; then
                echo "n8n on port $port ready after ${i}s"
                ready=true
                break
              fi
              sleep 1
            done
            if [ "$ready" != "true" ]; then
              echo "::error::n8n on port $port failed to start within 120s"
              for n in $(docker ps -aq --filter "name=n8n-eval-mcp-"); do
                echo "Logs for $n:"; docker logs "$n" --tail 30 || true
              done
              exit 1
            fi
          done

          # Emit the base-url CSV the eval CLI consumes as its lanes.
          BASE_URLS=""
          for port in "${PORTS[@]}"; do
            BASE_URLS="${BASE_URLS:+$BASE_URLS,}http://localhost:$port"
          done
          echo "base_urls=$BASE_URLS" >> "$GITHUB_OUTPUT"          

      - name: Create test users
        env:
          LANES: ${{ inputs.lanes || '6' }}
        run: |
          for i in $(seq 0 $((LANES - 1))); do
            port=$((5678 + i))
            curl -sf -X POST "http://localhost:$port/rest/e2e/reset" \
              -H "Content-Type: application/json" \
              -d '{
                "owner":{"email":"nathan@n8n.io","password":"PlaywrightTest123","firstName":"Eval","lastName":"Owner"},
                "admin":{"email":"admin@n8n.io","password":"PlaywrightTest123","firstName":"Admin","lastName":"User"},
                "members":[],
                "chat":{"email":"chat@n8n.io","password":"PlaywrightTest123","firstName":"Chat","lastName":"User"}
              }'
          done          

      # Single fused run: build via MCP + verify on each lane. The eval CLI enables
      # MCP, mints an API key, and stages a `claude` MCP config per lane itself.
      - name: Run MCP Evals
        continue-on-error: true
        working-directory: packages/@n8n/instance-ai
        env:
          # `claude` authenticates with the Anthropic direct API key; the verifier's
          # own LLM checks use the same key (two budgets).
          ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          # Build model rides claude's native env var (no CLI flag): the eval CLI
          # reads it, pins a default when unset, and records it as build_model.
          ANTHROPIC_MODEL: ${{ inputs.build-model }}
          N8N_INSTANCE_AI_MODEL_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          LANGSMITH_TRACING: 'true'
          LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
          LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
          LANGSMITH_REVISION_ID: ${{ github.sha }}
          LANGSMITH_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
          BASE_URLS: ${{ steps.lanes.outputs.base_urls }}
          LANES: ${{ inputs.lanes || '6' }}
          TIER: ${{ inputs.tier }}
          SUITE: ${{ inputs.suite }}
          LANGTRACER_URL: ${{ secrets.EVALS_LANGTRACER_URL }}
          LANGTRACER_API_KEY: ${{ secrets.LANGTRACER_API_KEY }}
          FILTER: ${{ inputs.filter }}
          ITERATIONS: ${{ inputs.iterations }}
          EVAL_CONCURRENCY: ${{ inputs.eval-concurrency }}
          EXPERIMENT_NAME: ${{ inputs.experiment-name }}
        run: |
          # Default concurrency to lanes * 2. Each lane already caps builds at 4,
          # but running all lanes flat-out (lanes * 4) floods the shared Anthropic
          # budget — builds, verifier checks, and AI-node mocks all contend — which
          # surfaces as verifier/MCP timeouts. lanes * 2 keeps that in check.
          CONCURRENCY="${EVAL_CONCURRENCY:-$((LANES * 2))}"
          echo "Lanes: $LANES | eval concurrency: $CONCURRENCY"

          # --dataset + --baseline-prefix keep the MCP cohort isolated from the
          # Instance AI dataset/baseline in LangSmith (both halves required).
          # --timeout-ms 25min: MCP-tier multi-agent cases with large mocked
          # payloads legitimately run past the 15-min default (trading-bot at
          # 32 klines rows). Scoped here, NOT the harness default — under the
          # experiments suite's higher concurrency, generous budgets let
          # starved scenarios hold lane slots and amplify contention.
          ARGS=(
            --base-url "$BASE_URLS"
            --build-via-mcp
            --tier "${TIER:-mcp}"
            --iterations "${ITERATIONS:-3}"
            --concurrency "$CONCURRENCY"
            --timeout-ms 1500000
            --dataset mcp-workflow-evals
            --baseline-prefix mcp-baseline-
            --verbose
          )
          # LangTracer is the only CI case source (no disk fallback by design).
          ARGS+=(--source langtracer --suite "${SUITE:-n8n-workflows}")
          [ -n "$FILTER" ] && ARGS+=(--filter "$FILTER")
          [ -n "$EXPERIMENT_NAME" ] && ARGS+=(--experiment-name "$EXPERIMENT_NAME")
          pnpm eval:instance-ai "${ARGS[@]}"          

      # No PR to comment on (manual/scheduled), so surface the rendered result
      # comment in the job summary instead. Always runs so failures are visible.
      - name: Write eval summary
        if: ${{ always() }}
        working-directory: packages/@n8n/instance-ai
        run: |
          if [ -f eval-pr-comment.md ]; then
            cat eval-pr-comment.md >> "$GITHUB_STEP_SUMMARY"
          else
            {
              echo "### MCP Workflow Eval"
              echo ""
              echo "No eval results produced (build or eval failed before writing results). Check the job logs and the uploaded build logs."
            } >> "$GITHUB_STEP_SUMMARY"
          fi          

      # Runs even on failure for post-mortem. Two layers of secret-leak defense:
      # (1) filter to diagnostic patterns, never tail raw output; (2) re-register
      # each secret via ::add-mask:: (multi-line certs masked line-by-line).
      - name: Capture n8n container logs (debug)
        if: ${{ always() }}
        env:
          EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
          N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
        run: |
          for v in "$EVALS_ANTHROPIC_KEY" "$N8N_LICENSE_ACTIVATION_KEY" \
                   "$N8N_LICENSE_CERT" "$N8N_ENCRYPTION_KEY"; do
            [ -z "$v" ] && continue
            while IFS= read -r line; do
              [ -n "$line" ] && echo "::add-mask::$line"
            done <<< "$v"
          done

          SIGNALS='mcp|builder|instance.?ai|error|warn|reject|exception|fail'
          for c in $(docker ps -aq --filter "name=n8n-eval-mcp-"); do
            name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||')
            echo ""
            echo "============================================================"
            echo "=== $name (filtered diagnostic signals, last 100 lines) ==="
            echo "============================================================"
            docker logs "$c" 2>&1 \
              | grep -ivE 'migration' \
              | grep -iE "$SIGNALS" \
              | tail -100 \
              || true
          done          

      - name: Stop n8n lanes
        if: ${{ always() }}
        run: |
          mapfile -t ids < <(docker ps -aq --filter "name=n8n-eval-mcp-")
          if [ "${#ids[@]}" -gt 0 ]; then
            docker stop "${ids[@]}" 2>/dev/null || true
            docker rm "${ids[@]}" 2>/dev/null || true
          fi          

      - name: Upload Results
        if: ${{ always() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: mcp-workflow-eval-results
          path: |
            packages/@n8n/instance-ai/eval-results.json
            packages/@n8n/instance-ai/eval-pr-comment.md
            packages/@n8n/instance-ai/.data/workflow-eval-report.html
            packages/@n8n/instance-ai/mcp-build-logs            
          if-no-files-found: ignore
          retention-days: 14

      # The eval step runs with continue-on-error so the summary/logs/artifacts
      # above always land — but a run that never wrote eval-results.json (bad
      # EVALS_ANTHROPIC_KEY, MCP setup breakage, CLI crash before any results)
      # must still fail the job rather than end green on a summary note.
      # Per-case build failures DO write results and stay green by design.
      - name: Fail when no results were produced
        if: ${{ always() }}
        working-directory: packages/@n8n/instance-ai
        run: |
          if [ ! -f eval-results.json ]; then
            echo "::error::Eval run produced no eval-results.json — it failed before writing any results. Check the 'Run MCP Evals' step logs and the uploaded artifacts."
            exit 1
          fi