fix: Extract inline run-report attachments before aggregating sizing (#32799)

Co-authored-by: n8n-cat-bot[bot] <n8n-cat-bot[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

packages/testing/playwright/scripts/sizing-matrix-aggregate.test.ts

@@ -0,0 +1,129 @@
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { extractInlineRunReports } from './sizing-matrix-aggregate';
+import { RunReportBuilder } from '../utils/benchmark/run-report';
+
+function makeReport(spec: string) {
+	return new RunReportBuilder(
+		{ spec, dimensions: { commitSha: 'abc123' } },
+		{ totalMs: 60_000, wallClockMs: 60_000 },
+		{ execPerSec: 10, p50Ms: 5, p99Ms: 20 },
+	).build();
+}
+
+/** Minimal Playwright JSON report wrapping run-report.json inline attachments. */
+function testResultsWithReports(specs: string[]) {
+	return {
+		config: {},
+		errors: [],
+		stats: {},
+		suites: [
+			{
+				title: 'benchmark',
+				file: 'bench.spec.ts',
+				column: 0,
+				line: 0,
+				specs: specs.map((spec) => ({
+					title: spec,
+					ok: true,
+					tags: [],
+					id: spec,
+					file: 'bench.spec.ts',
+					line: 0,
+					column: 0,
+					tests: [
+						{
+							results: [
+								{
+									attachments: [
+										{
+											name: 'run-report.json',
+											contentType: 'application/json',
+											path: '',
+											body: Buffer.from(JSON.stringify(makeReport(spec))).toString('base64'),
+										},
+										{ name: 'trace', contentType: 'application/zip', path: 'trace.zip' },
+									],
+								},
+							],
+						},
+					],
+				})),
+			},
+		],
+	};
+}
+
+describe('extractInlineRunReports', () => {
+	const dirs: string[] = [];
+
+	afterEach(() => {
+		// vitest cleans the OS tmpdir; nothing persistent to remove between tests.
+		dirs.length = 0;
+	});
+
+	function setupLane(name: string, specs: string[]): string {
+		const root = mkdtempSync(join(tmpdir(), 'sizing-extract-'));
+		dirs.push(root);
+		const laneDir = join(root, name);
+		mkdirSync(laneDir, { recursive: true });
+		writeFileSync(
+			join(laneDir, 'test-results.json'),
+			JSON.stringify(testResultsWithReports(specs)),
+		);
+		return root;
+	}
+
+	it('decodes inline run-report.json attachments into loose files', () => {
+		const root = setupLane('benchmark-webhook-shard-1', [
+			'webhook/webhook-single-instance.spec.ts',
+		]);
+
+		const extracted = extractInlineRunReports(root);
+
+		expect(extracted).toBe(1);
+		const laneDir = join(root, 'benchmark-webhook-shard-1');
+		const looseFiles = readdirSync(laneDir).filter((f) => f.includes('run-report'));
+		expect(looseFiles).toHaveLength(1);
+
+		const written = JSON.parse(readFileSync(join(laneDir, looseFiles[0]), 'utf8'));
+		expect(written.schemaVersion).toBe(1);
+		expect(written.scenario.spec).toBe('webhook/webhook-single-instance.spec.ts');
+	});
+
+	it('extracts one loose file per spec in a multi-spec shard', () => {
+		const root = setupLane('benchmark-kafka-shard-1', [
+			'kafka/single-instance-ceiling.spec.ts',
+			'kafka/burst-drain-capacity.spec.ts',
+		]);
+
+		const extracted = extractInlineRunReports(root);
+
+		expect(extracted).toBe(2);
+		const laneDir = join(root, 'benchmark-kafka-shard-1');
+		const looseFiles = readdirSync(laneDir).filter((f) => f.includes('run-report'));
+		expect(looseFiles).toHaveLength(2);
+	});
+
+	it('returns 0 when no test-results.json contains run-report attachments', () => {
+		const root = mkdtempSync(join(tmpdir(), 'sizing-extract-'));
+		dirs.push(root);
+		writeFileSync(
+			join(root, 'test-results.json'),
+			JSON.stringify({ suites: [{ specs: [{ tests: [{ results: [{ attachments: [] }] }] }] }] }),
+		);
+
+		expect(extractInlineRunReports(root)).toBe(0);
+	});
+
+	it('tolerates malformed test-results.json without throwing', () => {
+		const root = mkdtempSync(join(tmpdir(), 'sizing-extract-'));
+		dirs.push(root);
+		writeFileSync(join(root, 'test-results.json'), '{ not valid json');
+
+		expect(extractInlineRunReports(root)).toBe(0);
+	});
+});

packages/testing/playwright/scripts/sizing-matrix-aggregate.ts

@@ -6,18 +6,26 @@
  *
  *   CURRENTS_API_KEY=… ... --currents-run <runId>
  *
+ * In `--input` mode the report can be either a loose `run-report.json` file or
+ * an inline base64 attachment inside a Playwright `test-results.json` (how the
+ * benchmark lanes emit it). Inline attachments are decoded to loose files first
+ * so the filesystem scan picks them up — no Currents/secret coupling needed.
+ *
  * Hardware defaults to the Blacksmith CI runner (8 vCPU / 16 GB). Override
  * via `--hardware-runner/--hardware-vcpu/--hardware-ram-gb` or
  * `SIZING_MATRIX_RUNNER/VCPU/RAM_GB` env when running off-CI, or the matrix
  * will mis-attribute the source.
  */
 
+import type { JSONReport, JSONReportSuite } from '@playwright/test/reporter';
 import { readdirSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { dirname, join, resolve } from 'node:path';
+import { basename, dirname, join, relative, resolve } from 'node:path';
 
 const CURRENTS_API = 'https://api.currents.dev/v1';
 
+const RUN_REPORT_ATTACHMENT = 'run-report.json';
+
 // Repo root anchored to this file's location, so version/sha auto-detection
 // works regardless of the cwd the script is invoked from.
 const REPO_ROOT = resolve(__dirname, '../../../..');
@@ -217,7 +225,7 @@ function sanitiseFilename(spec: string): string {
 	return spec.replace(/[^a-z0-9._-]+/gi, '-').slice(0, 120);
 }
 
-function findRunReports(root: string): string[] {
+function findFiles(root: string, match: (path: string) => boolean): string[] {
 	const found: string[] = [];
 	const stack = [root];
 	while (stack.length) {
@@ -225,7 +233,7 @@ function findRunReports(root: string): string[] {
 		if (!current) continue;
 		const stat = statSync(current);
 		if (stat.isFile()) {
-			if (current.endsWith('.json') && current.includes('run-report')) found.push(current);
+			if (match(current)) found.push(current);
 			continue;
 		}
 		if (!stat.isDirectory()) continue;
@@ -236,6 +244,60 @@ function findRunReports(root: string): string[] {
 	return found.sort();
 }
 
+function findRunReports(root: string): string[] {
+	return findFiles(root, (p) => p.endsWith('.json') && p.includes('run-report'));
+}
+
+/** Decodes every `run-report.json` attachment in a Playwright JSON report. */
+function collectRunReportBodies(suites: JSONReportSuite[]): string[] {
+	const bodies: string[] = [];
+	const walk = (suite: JSONReportSuite): void => {
+		for (const spec of suite.specs ?? []) {
+			for (const test of spec.tests ?? []) {
+				for (const result of test.results ?? []) {
+					for (const attachment of result.attachments ?? []) {
+						if (attachment.name === RUN_REPORT_ATTACHMENT && attachment.body) {
+							bodies.push(Buffer.from(attachment.body, 'base64').toString('utf8'));
+						}
+					}
+				}
+			}
+		}
+		for (const child of suite.suites ?? []) walk(child);
+	};
+	for (const suite of suites) walk(suite);
+	return bodies;
+}
+
+/**
+ * Each benchmark lane uploads its Playwright `test-results.json`, which carries
+ * the run-report as an inline base64 attachment (from
+ * `testInfo.attach('run-report.json', { body })`) rather than as a loose file.
+ * The filesystem scan in `findRunReports` only sees loose files, so decode each
+ * inline attachment to a loose `run-report.*.json` next to its source report
+ * first. Returns the number of reports written. Idempotent across re-runs.
+ */
+export function extractInlineRunReports(root: string): number {
+	let extracted = 0;
+	for (const file of findFiles(root, (p) => basename(p) === 'test-results.json')) {
+		let report: JSONReport;
+		try {
+			report = JSON.parse(readFileSync(file, 'utf8')) as JSONReport;
+		} catch (error) {
+			console.warn(`[sizing-matrix] Failed to parse ${file}: ${(error as Error).message}`);
+			continue;
+		}
+		const bodies = collectRunReportBodies(report.suites ?? []);
+		const laneDir = dirname(file);
+		const label = sanitiseFilename(relative(root, laneDir) || 'lane');
+		bodies.forEach((body, index) => {
+			writeFileSync(join(laneDir, `${label}.run-report.${index}.json`), body);
+			extracted++;
+		});
+	}
+	return extracted;
+}
+
 function loadReport(path: string): { path: string; report: RunReport } | undefined {
 	try {
 		const report = JSON.parse(readFileSync(path, 'utf8')) as RunReport;
@@ -262,6 +324,12 @@ async function main(): Promise<void> {
 		reportPaths = fetched.reportPaths;
 		if (fetched.commitSha) commitSha = fetched.commitSha;
 	} else if (args.input) {
+		const extracted = extractInlineRunReports(args.input);
+		if (extracted > 0) {
+			console.log(
+				`[sizing-matrix] Extracted ${extracted} inline run-report.json attachment(s) from test-results.json`,
+			);
+		}
 		reportPaths = findRunReports(args.input);
 	} else {
 		throw new Error('Either --input or --currents-run is required');
@@ -316,7 +384,9 @@ async function main(): Promise<void> {
 	}
 }
 
-main().catch((error) => {
-	console.error(`[sizing-matrix] ${(error as Error).message}`);
-	process.exit(1);
-});
+if (require.main === module) {
+	main().catch((error) => {
+		console.error(`[sizing-matrix] ${(error as Error).message}`);
+		process.exit(1);
+	});
+}