n8n/.github/workflows/ci-evals.yml

name: Run Workflow Builder Evals

on:
  push:
    branches:
      - master
    paths:
      - 'packages/@n8n/ai-workflow-builder.ee/**'
      - '.github/workflows/ci-evals.yml'
  workflow_dispatch:
    inputs:
      branch:
        description: 'GitHub branch to test.'
        required: false
        default: 'master'
      dataset:
        description: 'LangSmith dataset to use.'
        required: false
        default: 'workflow-builder-canvas-prompts'

jobs:
  evals:
    name: Run Evaluations
    runs-on: blacksmith-2vcpu-ubuntu-2204
    env:
      N8N_AI_ANTHROPIC_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
      LANGSMITH_TRACING: true
      LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
      LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
      LANGSMITH_DATASET_NAME: ${{ github.event.inputs.dataset || 'workflow-builder-canvas-prompts' }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ github.event.inputs.branch || github.ref }}

      - name: Setup and Build
        uses: ./.github/actions/setup-nodejs-blacksmith

      - name: Export Node Types
        run: |
          ./packages/cli/bin/n8n export:nodes --output ./packages/@n8n/ai-workflow-builder.ee/evaluations/nodes.json          

      - name: Run Evaluations
        working-directory: packages/@n8n/ai-workflow-builder.ee/evaluations
        run: |
          pnpm eval:langsmith --repetitions 3