n8n/.github/workflows/test-evals-instance-ai.yml

name: 'Instance AI Evals: Experiments'

on:
  workflow_call:
    inputs:
      branch:
        description: 'GitHub branch to test'
        required: false
        type: string
        default: 'master'
      filter:
        description: 'Filter test cases by filename substring (e.g. "contact-form")'
        required: false
        type: string
        default: ''
      tier:
        description: 'Test-case dataset to run (e.g. "pr", "full"). Empty = no filter.'
        required: false
        type: string
        default: ''
      suite:
        description: 'LangTracer suite slug or id to pull test cases from.'
        required: false
        type: string
        default: 'baseline'
      sandbox-provider:
        description: 'Sandbox provider (n8n-sandbox or daytona)'
        required: false
        type: string
        default: 'n8n-sandbox'
      iterations:
        description: 'Iterations per test case'
        required: false
        type: string
        default: '3'
      experiment-name:
        description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)'
        required: false
        type: string
        default: ''
      pr-number:
        description: 'PR number to post results back to. Empty = derive from the pull_request event.'
        required: false
        type: string
        default: ''
      cache-sha:
        description: 'SHA for the docker image cache key. Empty = github.sha. Set to the tested commit on dispatch re-runs so the image matches the code under test.'
        required: false
        type: string
        default: ''
      revision-sha:
        description: 'Commit SHA under test, for LangSmith revision tagging. Empty = github.sha.'
        required: false
        type: string
        default: ''
      head-ref:
        description: 'Branch name under test, for LangSmith branch tagging. Empty = derive from context.'
        required: false
        type: string
        default: ''
      model:
        description: 'Model override for all Instance AI agents in the lane backends, provider/model format. Empty = backend default.'
        required: false
        type: string
        default: ''
  workflow_dispatch:
    inputs:
      branch:
        description: 'GitHub branch to test'
        required: false
        default: 'master'
      filter:
        description: 'Filter test cases by filename substring (e.g. "contact-form")'
        required: false
        default: ''
      tier:
        description: 'Test-case dataset to run (e.g. "pr", "full"). Empty = no filter.'
        required: false
        default: ''
      suite:
        description: 'LangTracer suite slug or id to pull test cases from.'
        required: false
        default: 'baseline'
      sandbox-provider:
        description: 'Sandbox provider (n8n-sandbox or daytona)'
        required: false
        default: 'n8n-sandbox'
      iterations:
        description: 'Iterations per test case (use 10 for a baseline)'
        required: false
        default: '3'
      experiment-name:
        description: 'LangSmith experiment name (instance-ai-baseline refreshes the baseline)'
        required: false
        default: ''
      model:
        description: 'Model for all Instance AI agents (provider/model). Empty = backend default (anthropic/claude-opus-4-8). For experiments: anthropic/claude-sonnet-4-6, anthropic/claude-sonnet-5.'
        required: false
        default: ''

jobs:
  run-evals:
    name: 'Run Evals'
    # 8vcpu/32GB: full-suite N=10 baselines need ~150 min of eval time and
    # did not fit the 4vcpu tier at any timeout.
    runs-on: blacksmith-8vcpu-ubuntu-2204
    # Long ceiling only for high-N baseline captures; regular runs keep a
    # tight guard so a wedged run fails fast. Plain comparison (not fromJSON):
    # it coerces the string input to a number, and a malformed dispatch value
    # (NaN) falls back to the 90-minute guard instead of erroring the job.
    timeout-minutes: ${{ (inputs.iterations || '3') >= 5 && 240 || 90 }}
    env:
      # Each port hosts an independent n8n container. The eval CLI's
      # work-stealing allocator dispatches builds across them, capped per-lane.
      # 10 lanes x 2.5 GB caps = 25 GB, leaving ~7 GB for sandbox + CLI + OS.
      LANE_PORTS: '5678,5679,5680,5681,5682,5683,5684,5685,5686,5687'
    permissions:
      contents: read
      pull-requests: write

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.branch || github.ref }}
          fetch-depth: 1

      - name: Setup Environment
        uses: ./.github/actions/setup-nodejs
        with:
          build-command: 'pnpm build'

      # Cache populated by prepare-docker; on a miss (e.g. fresh PR-open runs
      # that outpace prepare-docker) the action itself falls back to a rebuild
      # via build-n8n-docker.
      - name: Load n8n Docker image
        uses: ./.github/actions/load-n8n-docker
        with:
          cache-sha: ${{ inputs.cache-sha }}

      # Host OOMs and disk exhaustion leave no application trace — sample host
      # + container telemetry every 60s and ship it with the results artifact.
      - name: Start host telemetry sampler
        run: |
          mkdir -p eval-diag
          { echo "=== disk at job start ==="; df -h /; docker system df; } > eval-diag/df-at-start.log
          nohup bash -c 'while true; do
            {
              date -u +%FT%TZ
              free -m | head -2
              df -h / | tail -1
              docker stats --no-stream --format "{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}"
              ps -eo rss=,pid=,comm= --sort=-rss | head -5
              echo
            } >> eval-diag/host-samples.log 2>&1
            sleep 60
          done' > /dev/null 2>&1 &
          echo $! > eval-diag/sampler.pid          

      - name: Start sandbox service
        if: ${{ inputs.sandbox-provider == 'n8n-sandbox' }}
        run: pnpm --filter n8n-containers services --services sandbox --network n8n-eval-net --name n8n-svc-sandbox

      - name: Start n8n containers
        env:
          EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
          N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
          # LangSmith creds for backend thread traces, routed to the same
          # dedicated project as the eval CLI to keep the 'instance-ai' corpus clean.
          LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
          LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
          # Prefer head_ref: inputs.branch may be a merge ref (refs/pull/N/merge).
          SANDBOX_NAME_PREFIX: evals-ci-${{ github.head_ref || inputs.branch || github.ref_name }}
          SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
          INSTANCE_AI_MODEL: ${{ inputs.model }}
        run: |
          # Build provider-specific env args
          SANDBOX_ARGS=()
          if [ "$SANDBOX_PROVIDER" = "daytona" ]; then
            SANDBOX_ARGS+=(
              -e N8N_INSTANCE_AI_SANDBOX_PROVIDER=daytona
              -e N8N_INSTANCE_AI_SANDBOX_NAME_PREFIX="$SANDBOX_NAME_PREFIX"
              -e DAYTONA_API_URL=https://app.daytona.io/api
              -e DAYTONA_API_KEY="$DAYTONA_API_KEY"
            )
          else
            SANDBOX_ARGS+=(
              -e N8N_INSTANCE_AI_SANDBOX_PROVIDER=n8n-sandbox
              -e N8N_SANDBOX_SERVICE_URL=http://sandbox-api:8080
              -e N8N_SANDBOX_SERVICE_API_KEY=n8n-sandbox-ci-key
            )
          fi

          # Use the eval network when sandbox service is running
          NETWORK_ARGS=()
          if [ "$SANDBOX_PROVIDER" == "n8n-sandbox" ]; then
            NETWORK_ARGS+=(--network n8n-eval-net)
          fi

          # Model override for A/B experiments; empty = backend default (instance-ai.config.ts)
          MODEL_ARGS=()
          if [ -n "$INSTANCE_AI_MODEL" ]; then
            MODEL_ARGS+=(-e N8N_INSTANCE_AI_MODEL="$INSTANCE_AI_MODEL")
          fi

          IFS=',' read -ra PORTS <<< "$LANE_PORTS"
          for i in "${!PORTS[@]}"; do
            port="${PORTS[$i]}"
            # Bounded and self-healing: a lane that exhausts its capped heap is
            # restarted by docker instead of staying dead; pruning + log caps
            # keep per-lane disk and memory flat over a multi-hour run.
            docker run -d --name "n8n-eval-$((i+1))" \
              "${NETWORK_ARGS[@]}" \
              --memory 2.5g --memory-swap 2.5g \
              --restart on-failure \
              --log-opt max-size=50m --log-opt max-file=2 \
              -e NODE_OPTIONS=--max-old-space-size=2048 \
              -e EXECUTIONS_DATA_PRUNE=true \
              -e EXECUTIONS_DATA_MAX_AGE=1 \
              -e E2E_TESTS=true \
              -e N8N_ENABLED_MODULES=instance-ai \
              -e N8N_AI_ENABLED=true \
              -e N8N_INSTANCE_AI_MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" \
              -e N8N_AI_ASSISTANT_BASE_URL="" \
              -e N8N_INSTANCE_AI_SANDBOX_ENABLED=true \
              "${SANDBOX_ARGS[@]}" \
              "${MODEL_ARGS[@]}" \
              -e LANGSMITH_TRACING=true \
              -e LANGSMITH_API_KEY="$LANGSMITH_API_KEY" \
              -e LANGSMITH_ENDPOINT="$LANGSMITH_ENDPOINT" \
              -e LANGSMITH_PROJECT=instance-ai-evals \
              -e N8N_LICENSE_ACTIVATION_KEY="$N8N_LICENSE_ACTIVATION_KEY" \
              -e N8N_LICENSE_CERT="$N8N_LICENSE_CERT" \
              -e N8N_ENCRYPTION_KEY="$N8N_ENCRYPTION_KEY" \
              -p "$port:5678" \
              n8nio/n8n:local
          done
          # 120s budget per port: containers booting in parallel on a shared
          # runner contend for CPU/disk during n8n's startup (DB migrations,
          # license init), so each takes longer than a solo boot.
          for port in "${PORTS[@]}"; do
            ready=false
            for i in $(seq 1 120); do
              if curl -s "http://localhost:$port/healthz/readiness" -o /dev/null -w "%{http_code}" | grep -q 200; then
                echo "n8n on port $port ready after ${i}s"
                ready=true
                break
              fi
              sleep 1
            done
            if [ "$ready" != "true" ]; then
              echo "::error::n8n on port $port failed to start within 120s"
              for n in $(docker ps -aq --filter "name=n8n-eval-"); do
                echo "Logs for $n:"
                docker logs "$n" --tail 30 || true
              done
              exit 1
            fi
          done          

      - name: Create test users
        run: |
          IFS=',' read -ra PORTS <<< "$LANE_PORTS"
          for port in "${PORTS[@]}"; do
            curl -sf -X POST "http://localhost:$port/rest/e2e/reset" \
              -H "Content-Type: application/json" \
              -d '{
                "owner":{"email":"nathan@n8n.io","password":"PlaywrightTest123","firstName":"Eval","lastName":"Owner"},
                "admin":{"email":"admin@n8n.io","password":"PlaywrightTest123","firstName":"Admin","lastName":"User"},
                "members":[],
                "chat":{"email":"chat@n8n.io","password":"PlaywrightTest123","firstName":"Chat","lastName":"User"}
              }'
          done          

      # Belt-and-suspenders: env vars set sandbox/model config but persisted
      # settings can override. Per-lane assertion catches env-injection hiccups
      # or unexpected DB-side state. A single misconfigured lane would silently
      # route some builds through tool mode or the wrong model and pollute results.
      - name: Assert sandbox and model config on every lane
        env:
          SANDBOX_PROVIDER: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
          INSTANCE_AI_MODEL: ${{ inputs.model }}
        run: |
          IFS=',' read -ra PORTS <<< "$LANE_PORTS"
          bad=0
          for i in "${!PORTS[@]}"; do
            port="${PORTS[$i]}"
            lane="$((i+1))"
            curl -sf -X POST "http://localhost:$port/rest/login" \
              -H "Content-Type: application/json" \
              -d '{"emailOrLdapLoginId":"nathan@n8n.io","password":"PlaywrightTest123"}' \
              -c "/tmp/cookies-$port.txt" -o /dev/null
            cfg=$(curl -sf -b "/tmp/cookies-$port.txt" \
              "http://localhost:$port/rest/instance-ai/settings" \
              | jq -r '.data | "\(.sandboxEnabled) \(.sandboxProvider)"')
            if [ "$cfg" != "true $SANDBOX_PROVIDER" ]; then
              echo "::error::lane $lane (port $port): expected 'true $SANDBOX_PROVIDER', got '$cfg'"
              bad=$((bad+1))
            else
              echo "  lane $lane: sandboxEnabled=true sandboxProvider=$SANDBOX_PROVIDER ok"
            fi
            # /preferences returns the effective model name (user pref || config),
            # i.e. the name part of provider/model.
            if [ -n "$INSTANCE_AI_MODEL" ]; then
              effective=$(curl -sf -b "/tmp/cookies-$port.txt" \
                "http://localhost:$port/rest/instance-ai/preferences" \
                | jq -r '.data.modelName')
              if [ "$effective" != "${INSTANCE_AI_MODEL##*/}" ]; then
                echo "::error::lane $lane (port $port): expected model '${INSTANCE_AI_MODEL##*/}', got '$effective'"
                bad=$((bad+1))
              else
                echo "  lane $lane: model=$effective ok"
              fi
            fi
          done
          if [ "$bad" -gt 0 ]; then
            echo "::error::$bad lane(s) misconfigured - eval results would mix configurations"
            exit 1
          fi          

      # Disk pressure develops during the eval (per-lane SQLite growth,
      # verifier snapshots, logs), not at job start — so the runway check
      # lives here, after the image load and lane startup, where "is there
      # enough free disk for the next few hours" is a meaningful question.
      # A run-A-style on-runner fallback build leaves tens of GB of builder
      # cache that is safe to reclaim once every lane is up.
      - name: Ensure disk runway for the eval run
        shell: bash
        run: |
          free_gb=$(df -BG --output=avail / | tail -1 | tr -dc '0-9')
          if [ "${free_gb:-0}" -lt 30 ]; then
            echo "Only ${free_gb}GB free — reclaiming builder cache before the run accretes data"
            # Best-effort: a prune failure must not skip the eval run itself.
            docker builder prune -f || echo "::warning::builder prune failed — continuing with ${free_gb}GB free"
            df -h / || true
          else
            echo "Disk runway OK: ${free_gb}GB free"
          fi          

      - name: Run Instance AI Evals
        continue-on-error: true
        working-directory: packages/@n8n/instance-ai
        env:
          N8N_INSTANCE_AI_MODEL_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          LANGSMITH_TRACING: 'true'
          LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
          LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
          LANGSMITH_REVISION_ID: ${{ inputs.revision-sha || github.sha }}
          LANGSMITH_BRANCH: ${{ inputs.head-ref || github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
          # Dedicated project so eval build traces don't pollute the default 'instance-ai' corpus.
          LANGSMITH_PROJECT: instance-ai-evals
          FILTER: ${{ inputs.filter }}
          TIER: ${{ inputs.tier }}
          SUITE: ${{ inputs.suite }}
          # LangTracer is the test-case source of truth (TRUST-247); the CLI
          # pulls the suite per run via its export API.
          LANGTRACER_URL: ${{ secrets.EVALS_LANGTRACER_URL }}
          LANGTRACER_API_KEY: ${{ secrets.LANGTRACER_API_KEY }}
          ITERATIONS: ${{ inputs.iterations }}
          EXPERIMENT_NAME: ${{ inputs.experiment-name }}
          EVAL_PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
        run: |
          IFS=',' read -ra PORTS <<< "$LANE_PORTS"
          URLS=()
          for port in "${PORTS[@]}"; do
            URLS+=("http://localhost:$port")
          done
          BASE_URLS=$(IFS=,; printf '%s' "${URLS[*]}")
          ARGS=(--base-url "$BASE_URLS" --concurrency 32 --verbose --iterations "${ITERATIONS:-3}")
          # LangTracer is the only CI case source (no disk fallback by design).
          ARGS+=(--source langtracer --suite "${SUITE:-baseline}")
          # Pin the LangSmith cohort: langtracer mode otherwise derives a
          # suite-scoped dataset/baseline prefix, which would fork the KPI
          # history and orphan the baseline comparison.
          ARGS+=(--dataset instance-ai-workflow-evals --baseline-prefix instance-ai-baseline-)
          [ -n "$FILTER" ] && ARGS+=(--filter "$FILTER")
          [ -n "$TIER" ] && ARGS+=(--tier "$TIER")
          [ -n "$EXPERIMENT_NAME" ] && ARGS+=(--experiment-name "$EXPERIMENT_NAME")
          pnpm eval:instance-ai "${ARGS[@]}"          

      # Captures sandbox/builder diagnostic signals that surface during the
      # eval (after migrations finish). Two layers of secret-leak defense:
      #
      #   1. Filter to specific diagnostic patterns — never tail raw output.
      #      The grep allowlist scopes the log surface to lines we care
      #      about for debugging (sandbox lifecycle, builder, errors).
      #
      #   2. Re-register secrets via ::add-mask:: so any line that does
      #      match the allowlist has the secret values replaced with ***
      #      before reaching the GH Actions log. GitHub auto-masks
      #      ${{ secrets.X }} references, but the masking is fragile
      #      against transformed or split values; explicit registration
      #      reinforces it.
      #
      # Runs even on eval failure so we have the post-mortem regardless.
      - name: Capture n8n container logs (debug)
        if: ${{ always() }}
        env:
          EVALS_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
          DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }}
          N8N_LICENSE_ACTIVATION_KEY: ${{ secrets.N8N_LICENSE_ACTIVATION_KEY }}
          N8N_LICENSE_CERT: ${{ secrets.N8N_LICENSE_CERT }}
          N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
          EVALS_LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
        run: |
          # Layer 2 — defense in depth: explicitly mask each secret's value.
          # ::add-mask:: is a single-line workflow command. Multi-line secrets
          # (e.g. N8N_LICENSE_CERT is PEM-encoded) must be masked one line at
          # a time, otherwise only the first line is registered.
          for v in "$EVALS_ANTHROPIC_KEY" "$DAYTONA_API_KEY" \
                   "$N8N_LICENSE_ACTIVATION_KEY" "$N8N_LICENSE_CERT" \
                   "$N8N_ENCRYPTION_KEY" "$EVALS_LANGSMITH_API_KEY"; do
            [ -z "$v" ] && continue
            while IFS= read -r line; do
              [ -n "$line" ] && echo "::add-mask::$line"
            done <<< "$v"
          done

          # Layer 1 — accuracy filter: only surface diagnostic signals.
          # `tail -1000` after the filter so we get the LATEST matching lines
          # (post-eval failure signal), not the earliest startup-time ones.
          # -t keeps timestamps; the inspect line surfaces OOM kills/restarts,
          # which leave no log line of their own.
          SIGNALS='sandbox|builder|sandbox-service|daytona|instance.?ai|error|warn|reject|exception|fail'
          for c in $(docker ps -aq --filter "name=n8n-eval-"); do
            name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||')
            echo ""
            echo "============================================================"
            echo "=== $name (filtered diagnostic signals, last 1000 lines) ==="
            echo "============================================================"
            docker inspect --format 'state: status={{.State.Status}} oomkilled={{.State.OOMKilled}} exitcode={{.State.ExitCode}} restarts={{.RestartCount}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' "$c" || true
            docker logs -t "$c" 2>&1 \
              | grep -ivE 'migration' \
              | grep -iE "$SIGNALS" \
              | tail -1000 \
              || true
          done

          # Sandbox service container logs (when using n8n-sandbox provider)
          for c in $(docker ps -aq --filter "label=com.docker.compose.project=n8n-svc-sandbox"); do
            name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||')
            echo ""
            echo "============================================================"
            echo "=== $name (last 100 lines) ==="
            echo "============================================================"
            docker logs "$c" 2>&1 | tail -100 || true
          done          

      - name: Stop n8n containers
        if: ${{ always() }}
        run: |
          [ -f eval-diag/sampler.pid ] && kill "$(cat eval-diag/sampler.pid)" 2>/dev/null || true
          mapfile -t ids < <(docker ps -aq --filter "name=n8n-eval-")
          if [ "${#ids[@]}" -gt 0 ]; then
            docker stop "${ids[@]}" 2>/dev/null || true
            docker rm "${ids[@]}" 2>/dev/null || true
          fi
          # Sandbox service cleanup
          pnpm --filter n8n-containers services:clean 2>/dev/null || true
          docker network rm n8n-eval-net 2>/dev/null || true          

      - name: Post eval results to PR
        if: ${{ always() && (inputs.pr-number || github.event.pull_request.number) }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: ${{ github.repository }}
          PR_NUMBER: ${{ inputs.pr-number || github.event.pull_request.number }}
        run: |
          # The eval CLI writes the full PR comment as eval-pr-comment.md
          # (see comparison/format.ts:formatComparisonMarkdown). It includes
          # the alert, aggregate, comparison sections, per-test-case results
          # collapsed, and failure details collapsed. CI just relays it.
          COMMENT_FILE="packages/@n8n/instance-ai/eval-pr-comment.md"
          if [ ! -f "$COMMENT_FILE" ]; then
            echo "No PR comment file found (eval likely cancelled before writing results)"
            exit 0
          fi
          cp "$COMMENT_FILE" /tmp/eval-comment.md

          # Find and update existing eval comment, or create new one
          COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \
            --jq '.[] | select(.body | startswith("### Instance AI Workflow Eval")) | .id' | tail -1)

          if [ -n "$COMMENT_ID" ]; then
            gh api "repos/$REPO/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/eval-comment.md
          else
            gh pr comment "$PR_NUMBER" --body-file /tmp/eval-comment.md
          fi          

      - name: Upload Results
        if: ${{ always() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: instance-ai-workflow-eval-results
          path: |
            packages/@n8n/instance-ai/eval-results.json
            packages/@n8n/instance-ai/.data/workflow-eval-report.html
            eval-diag/            
          retention-days: 14

      # The eval step runs with continue-on-error so the logs/artifacts above
      # always land — but a run that never wrote eval-results.json (bad secret,
      # unresolvable LangTracer suite, CLI crash before any results) must still
      # fail the job rather than end green with zero evals. Per-case build
      # failures DO write results and stay green by design. Mirrors the same
      # guard in test-evals-mcp.yml.
      - name: Fail when no results were produced
        if: ${{ always() }}
        working-directory: packages/@n8n/instance-ai
        run: |
          if [ ! -f eval-results.json ]; then
            echo "::error::Eval run produced no eval-results.json — it failed before writing any results. Check the 'Run Instance AI Evals' step logs and the uploaded artifacts."
            exit 1
          fi