ci: Shard weekly E2E coverage run across cached docker image (no-changelog) (#29337)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.github/WORKFLOWS.md

@@ -487,7 +487,7 @@ Team ownership mappings in `CODEOWNERS`:
 | `ubuntu-latest`                     | 2    | Simple jobs, fork PR E2E    |
 | `blacksmith-2vcpu-ubuntu-2204`      | 2    | Standard builds, E2E shards |
 | `blacksmith-4vcpu-ubuntu-2204`      | 4    | Unit tests, typecheck, lint |
-| `blacksmith-8vcpu-ubuntu-2204`      | 8    | E2E coverage (weekly)       |
+| `blacksmith-8vcpu-ubuntu-2204`      | 8    | Heavy parallel workloads    |
 | `blacksmith-4vcpu-ubuntu-2204-arm`  | 4    | ARM64 Docker builds         |
 
 ### Selection Guidelines
@@ -500,7 +500,7 @@ Team ownership mappings in `CODEOWNERS`:
 
 **`blacksmith-4vcpu-ubuntu-2204`** - Unit tests (parallelized), linting (parallel file processing), typechecking (CPU-intensive), E2E test shards
 
-**`blacksmith-8vcpu-ubuntu-2204`** - Heavy parallel workloads, full E2E coverage runs
+**`blacksmith-8vcpu-ubuntu-2204`** - Heavy parallel workloads
 
 ### Runner Provider Toggle
 

.github/workflows/ci-pull-requests.yml

@@ -230,7 +230,6 @@ jobs:
       test-command: ${{ github.event.pull_request.head.repo.fork == true && 'pnpm --filter=n8n-playwright test:container:sqlite:e2e --grep-invert=@licensed' || 'pnpm --filter=n8n-playwright test:container:multi-main:e2e' }}
       workers: '1'
       pre-generated-matrix: ${{ needs.install-and-build.outputs.matrix }}
-      upload-failure-artifacts: ${{ github.event.pull_request.head.repo.fork == true }}
     secrets: inherit
 
   # Boots the editor-ui against the Vite dev server and fails on any console

.github/workflows/test-e2e-coverage-weekly.yml

@@ -5,48 +5,57 @@ on:
     - cron: '0 2 * * 1' # Every Monday at 2 AM
   workflow_dispatch: # Allow manual triggering
 
-env:
-  NODE_OPTIONS: --max-old-space-size=16384
-  PLAYWRIGHT_WORKERS: 4
-  PLAYWRIGHT_BROWSERS_PATH: packages/testing/playwright/.playwright-browsers
-
 jobs:
-  coverage:
-    runs-on: blacksmith-8vcpu-ubuntu-2204
-    name: Coverage Tests
+  prepare-docker:
+    name: Prepare Docker (coverage)
+    uses: ./.github/workflows/prepare-docker-reusable.yml
+    with:
+      build-variant: coverage
+      runner: blacksmith-8vcpu-ubuntu-2204
+    secrets: inherit
 
+  e2e:
+    name: E2E (coverage)
+    needs: prepare-docker
+    uses: ./.github/workflows/test-e2e-reusable.yml
+    with:
+      test-mode: docker-artifact
+      test-command: pnpm --filter=n8n-playwright test:container:coverage
+      workers: '1'
+      runner: blacksmith-4vcpu-ubuntu-2204
+      timeout-minutes: 45
+      pre-generated-matrix: '[{"shard":1,"images":""},{"shard":2,"images":""},{"shard":3,"images":""},{"shard":4,"images":""}]'
+    secrets: inherit
+
+  aggregate:
+    name: Aggregate Coverage
+    needs: e2e
+    if: always() && needs.e2e.result != 'skipped' && needs.e2e.result != 'cancelled'
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Setup Environment
         uses: ./.github/actions/setup-nodejs
-        env:
-          INCLUDE_TEST_CONTROLLER: 'true'
 
-      - name: Build Docker Image with Coverage
-        run: pnpm build:docker:coverage
-        env:
-          INCLUDE_TEST_CONTROLLER: 'true'
+      - name: Download shard artifacts
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          pattern: e2e-shard-*
+          path: /tmp/shards/
 
-      - name: Install Browsers
-        run: pnpm turbo run install-browsers --filter=n8n-playwright
-
-      - name: Run Container Coverage Tests
-        id: coverage-tests
+      - name: Collect coverage JSON
+        shell: bash
         run: |
-          pnpm --filter n8n-playwright test:container:sqlite \
-            --workers=${{ env.PLAYWRIGHT_WORKERS }}
-        env:
-          BUILD_WITH_COVERAGE: 'true'
-          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
-          CURRENTS_PROJECT_ID: 'LRxcNt'
-          QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
-          QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
-          QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
+          mkdir -p packages/testing/playwright/.nyc_output/coverage
+          found=$(find /tmp/shards -path '*/.nyc_output/coverage/*.json' 2>/dev/null | wc -l)
+          echo "Found $found coverage JSON files across shards"
+          find /tmp/shards -path '*/.nyc_output/coverage/*.json' \
+            -exec cp {} packages/testing/playwright/.nyc_output/coverage/ \;
+          ls -la packages/testing/playwright/.nyc_output/coverage/ || true
 
       - name: Generate Coverage Report
-        if: always() && steps.coverage-tests.outcome != 'skipped'
         run: pnpm --filter n8n-playwright coverage:report
 
       - name: Upload Coverage Report Artifact
@@ -68,7 +77,7 @@ jobs:
           fail_ci_if_error: false
 
       - name: Analyse Coverage Gaps
-        if: always() && steps.coverage-tests.outcome != 'skipped'
+        if: always()
         env:
           CODECOV_API_TOKEN: ${{ secrets.CODECOV_API_TOKEN }}
         run: |
@@ -76,7 +85,7 @@ jobs:
             --md --top=15 --out-json=coverage-gaps.json >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload Coverage Gap Report
-        if: always() && steps.coverage-tests.outcome != 'skipped'
+        if: always()
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: coverage-gap-report

.github/workflows/test-e2e-reusable.yml

@@ -32,11 +32,6 @@ on:
         required: false
         default: 30
         type: number
-      upload-failure-artifacts:
-        description: 'Upload test failure artifacts (screenshots, traces, videos). Enable for community PRs without Currents access.'
-        required: false
-        default: false
-        type: boolean
       currents-project-id:
         description: 'Currents project ID for reporting'
         required: false
@@ -121,15 +116,17 @@ jobs:
           N8N_ENCRYPTION_KEY: ${{ secrets.N8N_ENCRYPTION_KEY }}
           N8N_TEST_ENV: ${{ inputs.n8n-env }}
 
-      - name: Upload Failure Artifacts
-        if: ${{ failure() && inputs.upload-failure-artifacts }}
+      - name: Upload Shard Artifacts
+        if: always()
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
-          name: playwright-report-shard-${{ matrix.shard }}
+          name: e2e-shard-${{ matrix.shard }}
           path: |
             packages/testing/playwright/test-results/
             packages/testing/playwright/playwright-report/
-          retention-days: 7
+            packages/testing/playwright/.nyc_output/
+          retention-days: 1
+          if-no-files-found: ignore
 
       - name: Cancel Currents run if workflow is cancelled
         if: ${{ cancelled() }}

packages/testing/playwright/currents.config.ts

@@ -3,11 +3,9 @@ import type { CurrentsConfig } from '@currents/playwright';
 const config: CurrentsConfig = {
 	recordKey: process.env.CURRENTS_RECORD_KEY ?? '',
 	projectId: process.env.CURRENTS_PROJECT_ID ?? 'LRxcNt',
-	...(process.env.BUILD_WITH_COVERAGE === 'true' && {
-		coverage: {
-			projects: true,
-		},
-	}),
+	coverage: {
+		projects: ['coverage'],
+	},
 };
 
 // eslint-disable-next-line import-x/no-default-export

packages/testing/playwright/package.json

@@ -18,6 +18,7 @@
     "test:container:queue": "playwright test --project='queue:*'",
     "test:container:multi-main": "playwright test --project='multi-main:*'",
     "test:container:multi-main:e2e": "playwright test --project='multi-main:e2e'",
+    "test:container:coverage": "playwright test --project=coverage",
     "test:workflows:setup": "tsx ./tests/cli-workflows/setup-workflow-tests.ts",
     "test:workflows": "playwright test --project=cli-workflows",
     "test:workflows:schema": "SCHEMA=true playwright test --project=cli-workflows",

packages/testing/playwright/playwright-projects.ts

@@ -202,6 +202,14 @@ export function getProjects(): Project[] {
 			);
 		}
 
+		projects.push({
+			name: 'coverage',
+			testDir: './tests/e2e',
+			timeout: 60000,
+			fullyParallel: true,
+			use: { containerConfig: {} },
+		});
+
 		for (const { name, config } of CI_BENCHMARK_PROFILES) {
 			projects.push({
 				name: `benchmark-${name}:infrastructure`,

packages/testing/playwright/scripts/coverage-workflow.md

@@ -87,10 +87,11 @@ The HTML report will show you:
 If you see "No coverage files found":
 
 1. Build with coverage: `BUILD_WITH_COVERAGE=true pnpm build` or `pnpm build:docker:coverage`
-2. Run tests with coverage enabled: `BUILD_WITH_COVERAGE=true pnpm test:container:sqlite`
+2. Run tests against the coverage project: `pnpm test:container:coverage`
 3. Check that coverage files exist in `.nyc_output/{projectName}/` directories
+   - For CI coverage runs: `.nyc_output/coverage/`
    - For local mode: `.nyc_output/e2e/`
-   - For container mode: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc.
+   - For ad-hoc container runs: `.nyc_output/sqlite:e2e/`, `.nyc_output/postgres:e2e/`, etc.
 
 ### Low Coverage Percentage
 
@@ -129,9 +130,7 @@ For automated coverage reporting:
   run: pnpm build:docker:coverage
 
 - name: Run Container Coverage Tests
-  run: pnpm --filter n8n-playwright test:container:sqlite
-  env:
-    BUILD_WITH_COVERAGE: 'true'
+  run: pnpm --filter n8n-playwright test:container:coverage
 
 - name: Generate Coverage Report
   run: pnpm --filter n8n-playwright coverage:report

packages/testing/playwright/scripts/generate-coverage-report.js

@@ -11,103 +11,33 @@ const fs = require('fs');
 const NYC_OUTPUT_DIR = path.join(__dirname, '..', '.nyc_output');
 const COVERAGE_DIR = path.join(__dirname, '..', 'coverage');
 const NYC_CONFIG = path.join(__dirname, '..', 'nyc.config.ts');
-
-// Coverage directories to look for - Currents writes to .nyc_output/{projectName}/
-// Project names come from playwright-projects.ts
-const COVERAGE_PROJECT_PATTERNS = [
-	'e2e', // Local mode project
-	'sqlite:e2e', // Container mode projects
-	'postgres:e2e',
-	'queue:e2e',
-	'multi-main:e2e',
-];
-
-/**
- * Find all coverage directories that exist and contain JSON files
- */
-function findCoverageDirectories() {
-	const foundDirs = [];
-
-	for (const projectName of COVERAGE_PROJECT_PATTERNS) {
-		const projectDir = path.join(NYC_OUTPUT_DIR, projectName);
-		if (fs.existsSync(projectDir)) {
-			const files = fs.readdirSync(projectDir);
-			const jsonFiles = files.filter((f) => f.endsWith('.json'));
-			if (jsonFiles.length > 0) {
-				foundDirs.push({ dir: projectDir, projectName, fileCount: jsonFiles.length });
-			}
-		}
-	}
-
-	return foundDirs;
-}
+const COVERAGE_INPUT_DIR = path.join(NYC_OUTPUT_DIR, 'coverage');
 
 function main() {
 	console.log('🔍 Generating Coverage Report');
 	console.log('==============================\n');
 
-	// Find all coverage directories
-	const coverageDirs = findCoverageDirectories();
+	const jsonFiles = fs.existsSync(COVERAGE_INPUT_DIR)
+		? fs.readdirSync(COVERAGE_INPUT_DIR).filter((f) => f.endsWith('.json'))
+		: [];
 
-	if (coverageDirs.length === 0) {
-		console.error('❌ No coverage data found in .nyc_output/');
-		console.log('\nSearched for coverage in these project directories:');
-		COVERAGE_PROJECT_PATTERNS.forEach((p) => console.log(`  - .nyc_output/${p}/`));
+	if (jsonFiles.length === 0) {
+		console.error('❌ No coverage data found in .nyc_output/coverage/');
 		console.log('\nTo generate coverage data:');
 		console.log(
 			'1. Build editor-ui with coverage: BUILD_WITH_COVERAGE=true pnpm --filter n8n-editor-ui build',
 		);
-		console.log(
-			'2. Run Playwright tests with coverage: BUILD_WITH_COVERAGE=true pnpm test:container:sqlite',
-		);
+		console.log('2. Run Playwright tests with coverage: pnpm test:container:coverage');
 		process.exit(1);
 	}
 
-	console.log('Found coverage data in:');
-	coverageDirs.forEach(({ projectName, fileCount }) =>
-		console.log(`  - ${projectName}: ${fileCount} files`),
-	);
-	console.log('');
+	console.log(`Found ${jsonFiles.length} coverage files in .nyc_output/coverage/\n`);
 
 	try {
-		// Merge coverage files from all found project directories
-		// nyc merge only accepts one input directory, so we need to:
-		// 1. Copy all JSON files to a single temp directory
-		// 2. Run nyc merge on that directory
-		console.log('Merging coverage files from all projects...');
 		const mergedFile = path.join(NYC_OUTPUT_DIR, 'out.json');
-		const tempMergeDir = path.join(NYC_OUTPUT_DIR, '_merge_temp');
+		console.log('Merging coverage files...');
+		execSync(`npx nyc merge "${COVERAGE_INPUT_DIR}" "${mergedFile}"`, { stdio: 'inherit' });
 
-		// Create temp directory for merging
-		if (fs.existsSync(tempMergeDir)) {
-			fs.rmSync(tempMergeDir, { recursive: true });
-		}
-		fs.mkdirSync(tempMergeDir, { recursive: true });
-
-		// Copy all JSON files from all project directories to the temp directory
-		let fileIndex = 0;
-		for (const { dir, projectName } of coverageDirs) {
-			const files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
-			for (const file of files) {
-				const srcPath = path.join(dir, file);
-				// Use unique names to avoid collisions between projects
-				const destPath = path.join(
-					tempMergeDir,
-					`${projectName.replace(/:/g, '_')}_${fileIndex++}_${file}`,
-				);
-				fs.copyFileSync(srcPath, destPath);
-			}
-		}
-
-		console.log(`Copied ${fileIndex} coverage files to temp directory`);
-
-		// Now merge the single temp directory
-		execSync(`npx nyc merge "${tempMergeDir}" "${mergedFile}"`, { stdio: 'inherit' });
-
-		// Clean up temp directory
-		fs.rmSync(tempMergeDir, { recursive: true });
-
-		// Generate reports (HTML for viewing, LCOV for Codecov)
 		console.log('Generating coverage reports...');
 		execSync(
 			`npx nyc report --reporter=html --reporter=lcov --report-dir=${COVERAGE_DIR} --temp-dir=${NYC_OUTPUT_DIR} --config=${NYC_CONFIG} --exclude-after-remap=false`,