From e7b353cabcfd95fd1175bd252eda65d754cf4d79 Mon Sep 17 00:00:00 2001 From: Declan Carroll Date: Fri, 8 May 2026 09:08:39 +0100 Subject: [PATCH] ci: Shard weekly E2E coverage run across cached docker image (no-changelog) (#29337) Co-authored-by: Claude Opus 4.7 (1M context) --- .github/WORKFLOWS.md | 4 +- .github/workflows/ci-pull-requests.yml | 1 - .../workflows/test-e2e-coverage-weekly.yml | 71 ++++++++------- .github/workflows/test-e2e-reusable.yml | 15 ++-- .../testing/playwright/currents.config.ts | 8 +- packages/testing/playwright/package.json | 1 + .../testing/playwright/playwright-projects.ts | 8 ++ .../playwright/scripts/coverage-workflow.md | 9 +- .../scripts/generate-coverage-report.js | 90 +++---------------- 9 files changed, 74 insertions(+), 133 deletions(-) diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index ad23af2b601..8b93297f4a1 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -487,7 +487,7 @@ Team ownership mappings in `CODEOWNERS`: | `ubuntu-latest` | 2 | Simple jobs, fork PR E2E | | `blacksmith-2vcpu-ubuntu-2204` | 2 | Standard builds, E2E shards | | `blacksmith-4vcpu-ubuntu-2204` | 4 | Unit tests, typecheck, lint | -| `blacksmith-8vcpu-ubuntu-2204` | 8 | E2E coverage (weekly) | +| `blacksmith-8vcpu-ubuntu-2204` | 8 | Heavy parallel workloads | | `blacksmith-4vcpu-ubuntu-2204-arm` | 4 | ARM64 Docker builds | ### Selection Guidelines @@ -500,7 +500,7 @@ Team ownership mappings in `CODEOWNERS`: **`blacksmith-4vcpu-ubuntu-2204`** - Unit tests (parallelized), linting (parallel file processing), typechecking (CPU-intensive), E2E test shards -**`blacksmith-8vcpu-ubuntu-2204`** - Heavy parallel workloads, full E2E coverage runs +**`blacksmith-8vcpu-ubuntu-2204`** - Heavy parallel workloads ### Runner Provider Toggle diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index 3d567a5b9a4..1e940c138c1 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -230,7 +230,6 @@ jobs: test-command: ${{ github.event.pull_request.head.repo.fork == true && 'pnpm --filter=n8n-playwright test:container:sqlite:e2e --grep-invert=@licensed' || 'pnpm --filter=n8n-playwright test:container:multi-main:e2e' }} workers: '1' pre-generated-matrix: ${{ needs.install-and-build.outputs.matrix }} - upload-failure-artifacts: ${{ github.event.pull_request.head.repo.fork == true }} secrets: inherit # Boots the editor-ui against the Vite dev server and fails on any console diff --git a/.github/workflows/test-e2e-coverage-weekly.yml b/.github/workflows/test-e2e-coverage-weekly.yml index bb0e161fbc5..665285cee24 100644 --- a/.github/workflows/test-e2e-coverage-weekly.yml +++ b/.github/workflows/test-e2e-coverage-weekly.yml @@ -5,48 +5,57 @@ on: - cron: '0 2 * * 1' # Every Monday at 2 AM workflow_dispatch: # Allow manual triggering -env: - NODE_OPTIONS: --max-old-space-size=16384 - PLAYWRIGHT_WORKERS: 4 - PLAYWRIGHT_BROWSERS_PATH: packages/testing/playwright/.playwright-browsers - jobs: - coverage: - runs-on: blacksmith-8vcpu-ubuntu-2204 - name: Coverage Tests + prepare-docker: + name: Prepare Docker (coverage) + uses: ./.github/workflows/prepare-docker-reusable.yml + with: + build-variant: coverage + runner: blacksmith-8vcpu-ubuntu-2204 + secrets: inherit + e2e: + name: E2E (coverage) + needs: prepare-docker + uses: ./.github/workflows/test-e2e-reusable.yml + with: + test-mode: docker-artifact + test-command: pnpm --filter=n8n-playwright test:container:coverage + workers: '1' + runner: blacksmith-4vcpu-ubuntu-2204 + timeout-minutes: 45 + pre-generated-matrix: '[{"shard":1,"images":""},{"shard":2,"images":""},{"shard":3,"images":""},{"shard":4,"images":""}]' + secrets: inherit + + aggregate: + name: Aggregate Coverage + needs: e2e + if: always() && needs.e2e.result != 'skipped' && needs.e2e.result != 'cancelled' + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Environment uses: ./.github/actions/setup-nodejs - env: - INCLUDE_TEST_CONTROLLER: 'true' - - name: Build Docker Image with Coverage - run: pnpm build:docker:coverage - env: - INCLUDE_TEST_CONTROLLER: 'true' + - name: Download shard artifacts + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: e2e-shard-* + path: /tmp/shards/ - - name: Install Browsers - run: pnpm turbo run install-browsers --filter=n8n-playwright - - - name: Run Container Coverage Tests - id: coverage-tests + - name: Collect coverage JSON + shell: bash run: | - pnpm --filter n8n-playwright test:container:sqlite \ - --workers=${{ env.PLAYWRIGHT_WORKERS }} - env: - BUILD_WITH_COVERAGE: 'true' - CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }} - CURRENTS_PROJECT_ID: 'LRxcNt' - QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} - QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} - QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} + mkdir -p packages/testing/playwright/.nyc_output/coverage + found=$(find /tmp/shards -path '*/.nyc_output/coverage/*.json' 2>/dev/null | wc -l) + echo "Found $found coverage JSON files across shards" + find /tmp/shards -path '*/.nyc_output/coverage/*.json' \ + -exec cp {} packages/testing/playwright/.nyc_output/coverage/ \; + ls -la packages/testing/playwright/.nyc_output/coverage/ || true - name: Generate Coverage Report - if: always() && steps.coverage-tests.outcome != 'skipped' run: pnpm --filter n8n-playwright coverage:report - name: Upload Coverage Report Artifact @@ -68,7 +77,7 @@ jobs: fail_ci_if_error: false - name: Analyse Coverage Gaps - if: always() && steps.coverage-tests.outcome != 'skipped' + if: always() env: CODECOV_API_TOKEN: ${{ secrets.CODECOV_API_TOKEN }} run: | @@ -76,7 +85,7 @@ jobs: --md --top=15 --out-json=coverage-gaps.json >> "$GITHUB_STEP_SUMMARY" - name: Upload Coverage Gap Report - if: always() && steps.coverage-tests.outcome != 'skipped' + if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: coverage-gap-report diff --git a/.github/workflows/test-e2e-reusable.yml b/.github/workflows/test-e2e-reusable.yml index 3c43190e704..b0d5a211564 100644 --- a/.github/workflows/test-e2e-reusable.yml +++ b/.github/workflows/test-e2e-reusable.yml @@ -32,11 +32,6 @@ on: required: false default: 30 type: number - upload-failure-artifacts: - description: 'Upload test failure artifacts (screenshots, traces, videos). Enable for community PRs without Currents access.' - required: false - default: false - type: boolean currents-project-id: description: 'Currents project ID for reporting' required: false @@ -121,15 +116,17 @@ jobs: N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }} N8N_TEST_ENV: ${{ inputs.n8n-env }} - - name: Upload Failure Artifacts - if: ${{ failure() && inputs.upload-failure-artifacts }} + - name: Upload Shard Artifacts + if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: playwright-report-shard-${{ matrix.shard }} + name: e2e-shard-${{ matrix.shard }} path: | packages/testing/playwright/test-results/ packages/testing/playwright/playwright-report/ - retention-days: 7 + packages/testing/playwright/.nyc_output/ + retention-days: 1 + if-no-files-found: ignore - name: Cancel Currents run if workflow is cancelled if: ${{ cancelled() }} diff --git a/packages/testing/playwright/currents.config.ts b/packages/testing/playwright/currents.config.ts index 9110c1dbc4a..7af6ca941d9 100644 --- a/packages/testing/playwright/currents.config.ts +++ b/packages/testing/playwright/currents.config.ts @@ -3,11 +3,9 @@ import type { CurrentsConfig } from '@currents/playwright'; const config: CurrentsConfig = { recordKey: process.env.CURRENTS_RECORD_KEY ?? '', projectId: process.env.CURRENTS_PROJECT_ID ?? 'LRxcNt', - ...(process.env.BUILD_WITH_COVERAGE === 'true' && { - coverage: { - projects: true, - }, - }), + coverage: { + projects: ['coverage'], + }, }; // eslint-disable-next-line import-x/no-default-export diff --git a/packages/testing/playwright/package.json b/packages/testing/playwright/package.json index ccb837c5fac..d4d15f775f4 100644 --- a/packages/testing/playwright/package.json +++ b/packages/testing/playwright/package.json @@ -18,6 +18,7 @@ "test:container:queue": "playwright test --project='queue:*'", "test:container:multi-main": "playwright test --project='multi-main:*'", "test:container:multi-main:e2e": "playwright test --project='multi-main:e2e'", + "test:container:coverage": "playwright test --project=coverage", "test:workflows:setup": "tsx ./tests/cli-workflows/setup-workflow-tests.ts", "test:workflows": "playwright test --project=cli-workflows", "test:workflows:schema": "SCHEMA=true playwright test --project=cli-workflows", diff --git a/packages/testing/playwright/playwright-projects.ts b/packages/testing/playwright/playwright-projects.ts index bcf2f59f0cb..3226fca26ae 100644 --- a/packages/testing/playwright/playwright-projects.ts +++ b/packages/testing/playwright/playwright-projects.ts @@ -202,6 +202,14 @@ export function getProjects(): Project[] { ); } + projects.push({ + name: 'coverage', + testDir: './tests/e2e', + timeout: 60000, + fullyParallel: true, + use: { containerConfig: {} }, + }); + for (const { name, config } of CI_BENCHMARK_PROFILES) { projects.push({ name: `benchmark-${name}:infrastructure`, diff --git a/packages/testing/playwright/scripts/coverage-workflow.md b/packages/testing/playwright/scripts/coverage-workflow.md index 017eeb6ced4..d2468876586 100644 --- a/packages/testing/playwright/scripts/coverage-workflow.md +++ b/packages/testing/playwright/scripts/coverage-workflow.md @@ -87,10 +87,11 @@ The HTML report will show you: If you see "No coverage files found": 1. Build with coverage: `BUILD_WITH_COVERAGE=true pnpm build` or `pnpm build:docker:coverage` -2. Run tests with coverage enabled: `BUILD_WITH_COVERAGE=true pnpm test:container:sqlite` +2. Run tests against the coverage project: `pnpm test:container:coverage` 3. Check that coverage files exist in `.nyc_output/{projectName}/` directories + - For CI coverage runs: `.nyc_output/coverage/` - For local mode: `.nyc_output/e2e/` - - For container mode: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc. + - For ad-hoc container runs: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc. ### Low Coverage Percentage @@ -129,9 +130,7 @@ For automated coverage reporting: run: pnpm build:docker:coverage - name: Run Container Coverage Tests - run: pnpm --filter n8n-playwright test:container:sqlite - env: - BUILD_WITH_COVERAGE: 'true' + run: pnpm --filter n8n-playwright test:container:coverage - name: Generate Coverage Report run: pnpm --filter n8n-playwright coverage:report diff --git a/packages/testing/playwright/scripts/generate-coverage-report.js b/packages/testing/playwright/scripts/generate-coverage-report.js index 95bd992f967..253cf57f212 100755 --- a/packages/testing/playwright/scripts/generate-coverage-report.js +++ b/packages/testing/playwright/scripts/generate-coverage-report.js @@ -11,103 +11,33 @@ const fs = require('fs'); const NYC_OUTPUT_DIR = path.join(__dirname, '..', '.nyc_output'); const COVERAGE_DIR = path.join(__dirname, '..', 'coverage'); const NYC_CONFIG = path.join(__dirname, '..', 'nyc.config.ts'); - -// Coverage directories to look for - Currents writes to .nyc_output/{projectName}/ -// Project names come from playwright-projects.ts -const COVERAGE_PROJECT_PATTERNS = [ - 'e2e', // Local mode project - 'sqlite:e2e', // Container mode projects - 'postgres:e2e', - 'queue:e2e', - 'multi-main:e2e', -]; - -/** - * Find all coverage directories that exist and contain JSON files - */ -function findCoverageDirectories() { - const foundDirs = []; - - for (const projectName of COVERAGE_PROJECT_PATTERNS) { - const projectDir = path.join(NYC_OUTPUT_DIR, projectName); - if (fs.existsSync(projectDir)) { - const files = fs.readdirSync(projectDir); - const jsonFiles = files.filter((f) => f.endsWith('.json')); - if (jsonFiles.length > 0) { - foundDirs.push({ dir: projectDir, projectName, fileCount: jsonFiles.length }); - } - } - } - - return foundDirs; -} +const COVERAGE_INPUT_DIR = path.join(NYC_OUTPUT_DIR, 'coverage'); function main() { console.log('🔍 Generating Coverage Report'); console.log('==============================\n'); - // Find all coverage directories - const coverageDirs = findCoverageDirectories(); + const jsonFiles = fs.existsSync(COVERAGE_INPUT_DIR) + ? fs.readdirSync(COVERAGE_INPUT_DIR).filter((f) => f.endsWith('.json')) + : []; - if (coverageDirs.length === 0) { - console.error('❌ No coverage data found in .nyc_output/'); - console.log('\nSearched for coverage in these project directories:'); - COVERAGE_PROJECT_PATTERNS.forEach((p) => console.log(` - .nyc_output/${p}/`)); + if (jsonFiles.length === 0) { + console.error('❌ No coverage data found in .nyc_output/coverage/'); console.log('\nTo generate coverage data:'); console.log( '1. Build editor-ui with coverage: BUILD_WITH_COVERAGE=true pnpm --filter n8n-editor-ui build', ); - console.log( - '2. Run Playwright tests with coverage: BUILD_WITH_COVERAGE=true pnpm test:container:sqlite', - ); + console.log('2. Run Playwright tests with coverage: pnpm test:container:coverage'); process.exit(1); } - console.log('Found coverage data in:'); - coverageDirs.forEach(({ projectName, fileCount }) => - console.log(` - ${projectName}: ${fileCount} files`), - ); - console.log(''); + console.log(`Found ${jsonFiles.length} coverage files in .nyc_output/coverage/\n`); try { - // Merge coverage files from all found project directories - // nyc merge only accepts one input directory, so we need to: - // 1. Copy all JSON files to a single temp directory - // 2. Run nyc merge on that directory - console.log('Merging coverage files from all projects...'); const mergedFile = path.join(NYC_OUTPUT_DIR, 'out.json'); - const tempMergeDir = path.join(NYC_OUTPUT_DIR, '_merge_temp'); + console.log('Merging coverage files...'); + execSync(`npx nyc merge "${COVERAGE_INPUT_DIR}" "${mergedFile}"`, { stdio: 'inherit' }); - // Create temp directory for merging - if (fs.existsSync(tempMergeDir)) { - fs.rmSync(tempMergeDir, { recursive: true }); - } - fs.mkdirSync(tempMergeDir, { recursive: true }); - - // Copy all JSON files from all project directories to the temp directory - let fileIndex = 0; - for (const { dir, projectName } of coverageDirs) { - const files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); - for (const file of files) { - const srcPath = path.join(dir, file); - // Use unique names to avoid collisions between projects - const destPath = path.join( - tempMergeDir, - `${projectName.replace(/:/g, '_')}_${fileIndex++}_${file}`, - ); - fs.copyFileSync(srcPath, destPath); - } - } - - console.log(`Copied ${fileIndex} coverage files to temp directory`); - - // Now merge the single temp directory - execSync(`npx nyc merge "${tempMergeDir}" "${mergedFile}"`, { stdio: 'inherit' }); - - // Clean up temp directory - fs.rmSync(tempMergeDir, { recursive: true }); - - // Generate reports (HTML for viewing, LCOV for Codecov) console.log('Generating coverage reports...'); execSync( `npx nyc report --reporter=html --reporter=lcov --report-dir=${COVERAGE_DIR} --temp-dir=${NYC_OUTPUT_DIR} --config=${NYC_CONFIG} --exclude-after-remap=false`,