mirror of
https://github.com/n8n-io/n8n.git
synced 2026-07-29 03:55:15 +02:00
59 lines
2.1 KiB
YAML
59 lines
2.1 KiB
YAML
name: 'CI: Instance AI Evals'
|
|
|
|
# Auto-runs on PR open/reopen/ready (non-draft, path-filtered). Pushes don't
|
|
# re-trigger (no `synchronize`); use workflow_dispatch for manual runs.
|
|
on:
|
|
pull_request:
|
|
types: [opened, reopened, ready_for_review]
|
|
paths:
|
|
- 'packages/@n8n/instance-ai/src/**'
|
|
- 'packages/@n8n/instance-ai/evaluations/**'
|
|
- 'packages/cli/src/modules/instance-ai/**'
|
|
- 'packages/core/src/execution-engine/eval-mock-helpers.ts'
|
|
workflow_dispatch:
|
|
inputs:
|
|
branch:
|
|
description: 'GitHub branch to test'
|
|
required: false
|
|
default: 'master'
|
|
tier:
|
|
description: 'Test-case dataset to run (e.g. `pr`, `full`)'
|
|
required: false
|
|
default: 'full'
|
|
sandbox-provider:
|
|
description: 'Sandbox provider (n8n-sandbox or daytona)'
|
|
required: false
|
|
default: 'n8n-sandbox'
|
|
iterations:
|
|
description: 'Iterations per test case (use 10 for a baseline)'
|
|
required: false
|
|
default: '3'
|
|
experiment-name:
|
|
description: 'LangSmith experiment name (set to instance-ai-baseline to refresh the baseline)'
|
|
required: false
|
|
default: ''
|
|
|
|
concurrency:
|
|
group: instance-ai-evals-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
run-evals:
|
|
name: Instance AI Workflow Evals
|
|
# Skip drafts; allow workflow_dispatch (no pull_request payload).
|
|
if: >-
|
|
github.repository == 'n8n-io/n8n' &&
|
|
(github.event_name != 'pull_request' || github.event.pull_request.draft == false)
|
|
uses: ./.github/workflows/test-evals-instance-ai.yml
|
|
with:
|
|
# On PR events github.sha is the merge commit, so the eval tests the
|
|
# merged state (like e2e) and the checkout matches the key under which
|
|
# prepare-docker publishes the docker image cache.
|
|
branch: ${{ inputs.branch || github.sha }}
|
|
# PR events default to `pr`; dispatch defaults to `full`.
|
|
tier: ${{ inputs.tier || 'pr' }}
|
|
sandbox-provider: ${{ inputs.sandbox-provider || 'n8n-sandbox' }}
|
|
iterations: ${{ inputs.iterations }}
|
|
experiment-name: ${{ inputs.experiment-name }}
|
|
secrets: inherit
|