From c8a3a3ce3d1b366f3d68f37f536d56bd25f4eb41 Mon Sep 17 00:00:00 2001 From: "n8n-cat-bot[bot]" <283985454+n8n-cat-bot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:33:43 +0100 Subject: [PATCH] fix: Extract inline run-report attachments before aggregating sizing (#32799) Co-authored-by: n8n-cat-bot[bot] Co-authored-by: Claude Opus 4.8 --- .../scripts/sizing-matrix-aggregate.test.ts | 129 ++++++++++++++++++ .../scripts/sizing-matrix-aggregate.ts | 84 +++++++++++- 2 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 packages/testing/playwright/scripts/sizing-matrix-aggregate.test.ts diff --git a/packages/testing/playwright/scripts/sizing-matrix-aggregate.test.ts b/packages/testing/playwright/scripts/sizing-matrix-aggregate.test.ts new file mode 100644 index 00000000000..83d9a7d4d7c --- /dev/null +++ b/packages/testing/playwright/scripts/sizing-matrix-aggregate.test.ts @@ -0,0 +1,129 @@ +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { extractInlineRunReports } from './sizing-matrix-aggregate'; +import { RunReportBuilder } from '../utils/benchmark/run-report'; + +function makeReport(spec: string) { + return new RunReportBuilder( + { spec, dimensions: { commitSha: 'abc123' } }, + { totalMs: 60_000, wallClockMs: 60_000 }, + { execPerSec: 10, p50Ms: 5, p99Ms: 20 }, + ).build(); +} + +/** Minimal Playwright JSON report wrapping run-report.json inline attachments. */ +function testResultsWithReports(specs: string[]) { + return { + config: {}, + errors: [], + stats: {}, + suites: [ + { + title: 'benchmark', + file: 'bench.spec.ts', + column: 0, + line: 0, + specs: specs.map((spec) => ({ + title: spec, + ok: true, + tags: [], + id: spec, + file: 'bench.spec.ts', + line: 0, + column: 0, + tests: [ + { + results: [ + { + attachments: [ + { + name: 'run-report.json', + contentType: 'application/json', + path: '', + body: Buffer.from(JSON.stringify(makeReport(spec))).toString('base64'), + }, + { name: 'trace', contentType: 'application/zip', path: 'trace.zip' }, + ], + }, + ], + }, + ], + })), + }, + ], + }; +} + +describe('extractInlineRunReports', () => { + const dirs: string[] = []; + + afterEach(() => { + // vitest cleans the OS tmpdir; nothing persistent to remove between tests. + dirs.length = 0; + }); + + function setupLane(name: string, specs: string[]): string { + const root = mkdtempSync(join(tmpdir(), 'sizing-extract-')); + dirs.push(root); + const laneDir = join(root, name); + mkdirSync(laneDir, { recursive: true }); + writeFileSync( + join(laneDir, 'test-results.json'), + JSON.stringify(testResultsWithReports(specs)), + ); + return root; + } + + it('decodes inline run-report.json attachments into loose files', () => { + const root = setupLane('benchmark-webhook-shard-1', [ + 'webhook/webhook-single-instance.spec.ts', + ]); + + const extracted = extractInlineRunReports(root); + + expect(extracted).toBe(1); + const laneDir = join(root, 'benchmark-webhook-shard-1'); + const looseFiles = readdirSync(laneDir).filter((f) => f.includes('run-report')); + expect(looseFiles).toHaveLength(1); + + const written = JSON.parse(readFileSync(join(laneDir, looseFiles[0]), 'utf8')); + expect(written.schemaVersion).toBe(1); + expect(written.scenario.spec).toBe('webhook/webhook-single-instance.spec.ts'); + }); + + it('extracts one loose file per spec in a multi-spec shard', () => { + const root = setupLane('benchmark-kafka-shard-1', [ + 'kafka/single-instance-ceiling.spec.ts', + 'kafka/burst-drain-capacity.spec.ts', + ]); + + const extracted = extractInlineRunReports(root); + + expect(extracted).toBe(2); + const laneDir = join(root, 'benchmark-kafka-shard-1'); + const looseFiles = readdirSync(laneDir).filter((f) => f.includes('run-report')); + expect(looseFiles).toHaveLength(2); + }); + + it('returns 0 when no test-results.json contains run-report attachments', () => { + const root = mkdtempSync(join(tmpdir(), 'sizing-extract-')); + dirs.push(root); + writeFileSync( + join(root, 'test-results.json'), + JSON.stringify({ suites: [{ specs: [{ tests: [{ results: [{ attachments: [] }] }] }] }] }), + ); + + expect(extractInlineRunReports(root)).toBe(0); + }); + + it('tolerates malformed test-results.json without throwing', () => { + const root = mkdtempSync(join(tmpdir(), 'sizing-extract-')); + dirs.push(root); + writeFileSync(join(root, 'test-results.json'), '{ not valid json'); + + expect(extractInlineRunReports(root)).toBe(0); + }); +}); diff --git a/packages/testing/playwright/scripts/sizing-matrix-aggregate.ts b/packages/testing/playwright/scripts/sizing-matrix-aggregate.ts index 596037ae298..ab6330c994d 100644 --- a/packages/testing/playwright/scripts/sizing-matrix-aggregate.ts +++ b/packages/testing/playwright/scripts/sizing-matrix-aggregate.ts @@ -6,18 +6,26 @@ * * CURRENTS_API_KEY=… ... --currents-run * + * In `--input` mode the report can be either a loose `run-report.json` file or + * an inline base64 attachment inside a Playwright `test-results.json` (how the + * benchmark lanes emit it). Inline attachments are decoded to loose files first + * so the filesystem scan picks them up — no Currents/secret coupling needed. + * * Hardware defaults to the Blacksmith CI runner (8 vCPU / 16 GB). Override * via `--hardware-runner/--hardware-vcpu/--hardware-ram-gb` or * `SIZING_MATRIX_RUNNER/VCPU/RAM_GB` env when running off-CI, or the matrix * will mis-attribute the source. */ +import type { JSONReport, JSONReportSuite } from '@playwright/test/reporter'; import { readdirSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { basename, dirname, join, relative, resolve } from 'node:path'; const CURRENTS_API = 'https://api.currents.dev/v1'; +const RUN_REPORT_ATTACHMENT = 'run-report.json'; + // Repo root anchored to this file's location, so version/sha auto-detection // works regardless of the cwd the script is invoked from. const REPO_ROOT = resolve(__dirname, '../../../..'); @@ -217,7 +225,7 @@ function sanitiseFilename(spec: string): string { return spec.replace(/[^a-z0-9._-]+/gi, '-').slice(0, 120); } -function findRunReports(root: string): string[] { +function findFiles(root: string, match: (path: string) => boolean): string[] { const found: string[] = []; const stack = [root]; while (stack.length) { @@ -225,7 +233,7 @@ function findRunReports(root: string): string[] { if (!current) continue; const stat = statSync(current); if (stat.isFile()) { - if (current.endsWith('.json') && current.includes('run-report')) found.push(current); + if (match(current)) found.push(current); continue; } if (!stat.isDirectory()) continue; @@ -236,6 +244,60 @@ function findRunReports(root: string): string[] { return found.sort(); } +function findRunReports(root: string): string[] { + return findFiles(root, (p) => p.endsWith('.json') && p.includes('run-report')); +} + +/** Decodes every `run-report.json` attachment in a Playwright JSON report. */ +function collectRunReportBodies(suites: JSONReportSuite[]): string[] { + const bodies: string[] = []; + const walk = (suite: JSONReportSuite): void => { + for (const spec of suite.specs ?? []) { + for (const test of spec.tests ?? []) { + for (const result of test.results ?? []) { + for (const attachment of result.attachments ?? []) { + if (attachment.name === RUN_REPORT_ATTACHMENT && attachment.body) { + bodies.push(Buffer.from(attachment.body, 'base64').toString('utf8')); + } + } + } + } + } + for (const child of suite.suites ?? []) walk(child); + }; + for (const suite of suites) walk(suite); + return bodies; +} + +/** + * Each benchmark lane uploads its Playwright `test-results.json`, which carries + * the run-report as an inline base64 attachment (from + * `testInfo.attach('run-report.json', { body })`) rather than as a loose file. + * The filesystem scan in `findRunReports` only sees loose files, so decode each + * inline attachment to a loose `run-report.*.json` next to its source report + * first. Returns the number of reports written. Idempotent across re-runs. + */ +export function extractInlineRunReports(root: string): number { + let extracted = 0; + for (const file of findFiles(root, (p) => basename(p) === 'test-results.json')) { + let report: JSONReport; + try { + report = JSON.parse(readFileSync(file, 'utf8')) as JSONReport; + } catch (error) { + console.warn(`[sizing-matrix] Failed to parse ${file}: ${(error as Error).message}`); + continue; + } + const bodies = collectRunReportBodies(report.suites ?? []); + const laneDir = dirname(file); + const label = sanitiseFilename(relative(root, laneDir) || 'lane'); + bodies.forEach((body, index) => { + writeFileSync(join(laneDir, `${label}.run-report.${index}.json`), body); + extracted++; + }); + } + return extracted; +} + function loadReport(path: string): { path: string; report: RunReport } | undefined { try { const report = JSON.parse(readFileSync(path, 'utf8')) as RunReport; @@ -262,6 +324,12 @@ async function main(): Promise { reportPaths = fetched.reportPaths; if (fetched.commitSha) commitSha = fetched.commitSha; } else if (args.input) { + const extracted = extractInlineRunReports(args.input); + if (extracted > 0) { + console.log( + `[sizing-matrix] Extracted ${extracted} inline run-report.json attachment(s) from test-results.json`, + ); + } reportPaths = findRunReports(args.input); } else { throw new Error('Either --input or --currents-run is required'); @@ -316,7 +384,9 @@ async function main(): Promise { } } -main().catch((error) => { - console.error(`[sizing-matrix] ${(error as Error).message}`); - process.exit(1); -}); +if (require.main === module) { + main().catch((error) => { + console.error(`[sizing-matrix] ${(error as Error).message}`); + process.exit(1); + }); +}