#!/usr/bin/env node /** * Coverage gap analysis across frontend and nodes. * * Queries Codecov flag data to surface files with the most uncovered lines — * the highest ROI targets for new tests. * * Usage: * node scripts/coverage-analysis.mjs [options] * * --domain=frontend|nodes|all Which domain to analyse (default: all) * --top= Files per list (default: 20) * --min-lines= Minimum file size to include (default: 50) * --gap-threshold= Max combined coverage % to be a gap (default: 60) * --json Structured JSON — AI-ready, includes type + recommendation fields * --md GitHub-Flavored Markdown — for $GITHUB_STEP_SUMMARY * --out-json= Write JSON to a file alongside --md (avoids double API call in CI) * --deep= Per-line unit vs E2E breakdown for a single file * * Environment: * CODECOV_API_TOKEN Required in CI. Rate limits are strict without it. * * JSON schema (for AI agents): * Top-level: { generatedAt, config, results[] } * Each result: { domain, totalGapFiles, gaps[], hotPath[] } * Each gap entry: * domain — "frontend" | "nodes" * file — full repo-relative path (use with Read tool) * type — composable | store | api | utility | component | node | other * recommendation — "unit-test" | "e2e-test" | "nock-test" * lines — total tracked lines * unitPct — unit test coverage % * e2ePct — E2E coverage % (0 for nodes domain — no E2E flag exists) * combinedPct — optimistic union coverage % (assumes zero overlap; actual may be lower) * uncovered — uncovered line count (primary sort key — highest = most impactful) * codecovUrl — direct link to per-line file report on Codecov * * Note on combinedPct: computed as min(unit_hits + e2e_hits, total_lines) / total_lines. * This assumes the two layers cover non-overlapping lines, so it is an upper bound. * Real combined coverage may be lower if both layers cover the same lines. */ const BASE_URL = 'https://codecov.io/api/v2/github/n8n-io/repos/n8n'; const CODECOV_FILE_BASE = 'https://app.codecov.io/github/n8n-io/n8n/blob/master'; const DOMAINS = { frontend: { label: 'Frontend (editor-ui)', unitFlag: 'frontend', e2eFlag: 'frontend-e2e' }, nodes: { label: 'Nodes (nodes-base)', unitFlag: 'nodes-unit', e2eFlag: null }, }; // ── CLI ──────────────────────────────────────────────────────────────────────── const args = Object.fromEntries( process.argv .slice(2) .filter((a) => a.startsWith('--')) .map((a) => { const [k, v] = a.slice(2).split('='); return [k, v ?? true]; }), ); const DOMAIN = args.domain ?? 'all'; const TOP = parseInt(args.top ?? '20', 10); const MIN_LINES = parseInt(args['min-lines'] ?? '50', 10); const GAP_THRESHOLD = parseFloat(args['gap-threshold'] ?? '60'); const OUTPUT_JSON = args.json === true; const OUTPUT_MD = args.md === true; const OUT_JSON_FILE = args['out-json']; const DEEP_FILE = args.deep; const activeDomains = DOMAIN === 'all' ? Object.entries(DOMAINS) : [[DOMAIN, DOMAINS[DOMAIN]]]; if (DOMAIN !== 'all' && !DOMAINS[DOMAIN]) { console.error(`Unknown domain "${DOMAIN}". Use: frontend, nodes, or all.`); process.exit(1); } if (DEEP_FILE && (DEEP_FILE.startsWith('http') || DEEP_FILE.includes('..'))) { console.error( '--deep must be a relative repository file path (e.g. packages/editor-ui/src/App.vue)', ); process.exit(1); } const TOKEN = process.env.CODECOV_API_TOKEN; const headers = TOKEN ? { Authorization: `Bearer ${TOKEN}` } : {}; // ── File type inference ──────────────────────────────────────────────────────── /** * Infers the file type and appropriate test recommendation from the file path. * Used by AI agents to prioritise the easiest/most impactful gaps to address. * * Order matters — more specific patterns first. */ export function inferType(filePath) { const name = filePath.split('/').pop() ?? ''; const lpath = filePath.toLowerCase(); // Node files (.node.ts) — test with nock HTTP mocking if (/\.node\.(ts|js)$/.test(name)) { return { type: 'node', recommendation: 'nock-test' }; } // Composables — must be in /composables/ dir AND match use* prefix to avoid // false positives like UserManager.ts, useRootStore (store), etc. if (lpath.includes('/composables/') && /^use[A-Z]/.test(name)) { return { type: 'composable', recommendation: 'unit-test' }; } // Stores if (/\.store\.(ts|js)$/.test(name) || lpath.includes('/stores/')) { return { type: 'store', recommendation: 'unit-test' }; } // API modules if (/\.api\.(ts|js)$/.test(name)) { return { type: 'api', recommendation: 'unit-test' }; } // Utilities/helpers if (/\.(utils|helpers|util|helper)\.(ts|js)$/.test(name)) { return { type: 'utility', recommendation: 'unit-test' }; } // Vue components — prefer E2E for UI behaviour, unit for pure logic if (/\.vue$/.test(name)) { return { type: 'component', recommendation: 'e2e-test' }; } return { type: 'other', recommendation: 'unit-test' }; } // ── Fetch ────────────────────────────────────────────────────────────────────── /** Fetch with exponential backoff retry — handles 429 rate limits and transient 5xx. */ async function fetchJson(url, retries = 3) { for (let attempt = 0; attempt <= retries; attempt++) { const res = await fetch(url, { headers }); if (res.ok) return res.json(); // Retry on rate limit or transient server error if ((res.status === 429 || res.status >= 500) && attempt < retries) { const delay = Math.pow(2, attempt) * 1000 + Math.random() * 500; process.stderr.write(` [${res.status}] retrying in ${Math.round(delay / 1000)}s...\n`); await new Promise((r) => setTimeout(r, delay)); continue; } const body = await res.text(); throw new Error(`HTTP ${res.status} ${url}\n${body.slice(0, 200)}`); } } /** * Returns Map for a given Codecov flag. * Warns if the flag returns no files — likely a propagation delay after upload. */ async function fetchFlagMap(flag) { const map = new Map(); let url = `${BASE_URL}/report/?flag=${flag}&page_size=100`; let page = 1; process.stderr.write(` ${flag}:`); while (url) { process.stderr.write(` p${page}`); const data = await fetchJson(url); for (const f of data.files ?? []) { const t = f.totals ?? {}; const lines = Number(t.lines) || 0; const hits = Number(t.hits) || 0; if (lines === 0) continue; // skip files with no tracked lines map.set(f.name, { lines, hits, pct: (hits / lines) * 100 }); } url = data.next ?? null; page++; } process.stderr.write(` → ${map.size} files\n`); if (map.size === 0) { process.stderr.write( ` WARNING: flag "${flag}" returned 0 files. Codecov may still be indexing the upload.\n` + ` Results for this flag will be empty. Re-run in a few minutes if this looks wrong.\n`, ); } return map; } async function fetchLineData(filePath, flag) { const data = await fetchJson( `${BASE_URL}/file_report/${encodeURIComponent(filePath)}?flag=${flag}`, ); return data.line_coverage ?? []; } // ── Analysis ─────────────────────────────────────────────────────────────────── export function findGaps(unitMap, e2eMap, domainKey) { const all = new Set([...unitMap.keys(), ...(e2eMap?.keys() ?? [])]); const gaps = []; for (const name of all) { const unit = unitMap.get(name) ?? { lines: 0, hits: 0, pct: 0 }; const e2e = e2eMap?.get(name) ?? { lines: 0, hits: 0, pct: 0 }; const lines = Math.max(unit.lines, e2e.lines); // Both zero means no line data at all — skip entirely if (lines === 0 || lines < MIN_LINES) continue; const combinedHits = Math.min(unit.hits + e2e.hits, lines); const combinedPct = (combinedHits / lines) * 100; const uncovered = lines - combinedHits; if (combinedPct >= GAP_THRESHOLD) continue; const { type, recommendation } = inferType(name); gaps.push({ domain: domainKey, file: name, type, recommendation, lines, unitPct: unit.pct, e2ePct: e2e.pct, combinedPct, uncovered, codecovUrl: `${CODECOV_FILE_BASE}/${name}`, }); } const sorted = gaps.sort((a, b) => b.uncovered - a.uncovered); return { gaps: sorted.slice(0, TOP), totalGapFiles: sorted.length }; } export function findHotPath(unitMap, e2eMap) { if (!e2eMap) return []; const all = new Set([...unitMap.keys(), ...e2eMap.keys()]); const result = []; for (const name of all) { const unit = unitMap.get(name) ?? { lines: 0, pct: 0 }; const e2e = e2eMap.get(name) ?? { lines: 0, pct: 0 }; const lines = Math.max(unit.lines, e2e.lines); if (lines === 0 || lines < MIN_LINES) continue; if (e2e.pct >= 80 && unit.pct < 15) { result.push({ file: name, lines, unitPct: unit.pct, e2ePct: e2e.pct }); } } return result.sort((a, b) => b.e2ePct - a.e2ePct).slice(0, 10); } // ── Terminal output ──────────────────────────────────────────────────────────── const W = 112; function bar(pct, width = 14) { const filled = Math.round((pct / 100) * width); return '[' + '█'.repeat(filled) + '░'.repeat(width - filled) + ']'; } function trunc(str, max) { return str.length > max ? '…' + str.slice(-(max - 1)) : str; } function printGapTable(label, gaps, hasE2E, totalGapFiles) { console.log(`\n${'═'.repeat(W)}`); console.log( ` GAPS — ${label} (${totalGapFiles} files below ${GAP_THRESHOLD}% — top ${gaps.length} by uncovered lines)`, ); console.log('═'.repeat(W)); const e2eHdr = hasE2E ? `${'E2E%'.padStart(7)} ` : ''; console.log( ` ${'#'.padEnd(4)} ${'File'.padEnd(48)} ${'Type'.padEnd(12)} ${'Lines'.padStart(6)} ${'Unit%'.padStart(7)} ${e2eHdr}${'Combined%'.padStart(11)} ${'Uncovered'.padStart(10)} Bar`, ); console.log('─'.repeat(W)); for (const [i, f] of gaps.entries()) { const e2eCol = hasE2E ? `${f.e2ePct.toFixed(1).padStart(6)}% ` : ''; console.log( ` ${String(i + 1).padEnd(4)} ${trunc(f.file, 48).padEnd(48)} ${f.type.padEnd(12)} ${String(f.lines).padStart(6)} ` + `${f.unitPct.toFixed(1).padStart(6)}% ${e2eCol}${f.combinedPct.toFixed(1).padStart(10)}% ${String(f.uncovered).padStart(10)} ${bar(f.combinedPct)}`, ); } if (!gaps.length) console.log(' (none)'); } function printHotPathTable(hotPath) { if (!hotPath.length) return; console.log(`\n${'─'.repeat(W)}`); console.log( ` E2E HOT-PATH (E2E ≥ 80%, Unit < 15%) — covered by navigation, not deliberate unit tests`, ); console.log('─'.repeat(W)); for (const [i, f] of hotPath.entries()) { console.log( ` ${String(i + 1).padEnd(3)} ${trunc(f.file, 75).padEnd(75)} unit=${f.unitPct.toFixed(1).padStart(5)}% e2e=${f.e2ePct.toFixed(1).padStart(5)}%`, ); } } // ── Markdown output (GitHub Job Summary) ────────────────────────────────────── export function mdEscape(str) { // Escape characters that break GFM table cells or link syntax return str.replace(/\|/g, '\\|').replace(/"/g, '"'); } function renderMarkdown(results, date) { const lines = []; lines.push('## Coverage Gap Report'); lines.push( `_${date} · gap threshold <${GAP_THRESHOLD}% · min ${MIN_LINES} lines · [Full report on Codecov](https://app.codecov.io/github/n8n-io/n8n)_`, ); lines.push(''); lines.push('> Files ranked by uncovered lines — highest ROI targets for new tests.'); lines.push(''); for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) { lines.push(`### ${domain.label}`); lines.push( `_${totalGapFiles} files below ${GAP_THRESHOLD}% combined coverage — top ${gaps.length} by uncovered lines_`, ); lines.push(''); const e2eHdr = hasE2E ? ' E2E% |' : ''; const e2eSep = hasE2E ? ' ----: |' : ''; lines.push(`| # | File | Type | Lines | Unit% |${e2eHdr} Combined% | Uncovered |`); lines.push(`| --: | ---- | ---- | ----: | ----: |${e2eSep} --------: | --------: |`); for (const [i, f] of gaps.entries()) { const fileName = mdEscape(f.file.split('/').pop()); const dir = mdEscape(f.file.split('/').slice(0, -1).join('/')); const fileCell = `[\`${fileName}\`](${f.codecovUrl} "${dir}")`; const e2eCol = hasE2E ? ` ${f.e2ePct.toFixed(1)}% |` : ''; lines.push( `| ${i + 1} | ${fileCell} | \`${f.type}\` | ${f.lines} | ${f.unitPct.toFixed(1)}% |${e2eCol} **${f.combinedPct.toFixed(1)}%** | **${f.uncovered}** |`, ); } if (hotPath.length) { lines.push(''); lines.push('
'); lines.push( `E2E hot-path — ${hotPath.length} files covered by test navigation (not deliberate unit tests)`, ); lines.push(''); lines.push('| File | Unit% | E2E% |'); lines.push('| ---- | ----: | ---: |'); for (const f of hotPath) { lines.push( `| \`${mdEscape(f.file.split('/').pop())}\` | ${f.unitPct.toFixed(1)}% | ${f.e2ePct.toFixed(1)}% |`, ); } lines.push('
'); } lines.push(''); } return lines.join('\n'); } // ── Deep dive ────────────────────────────────────────────────────────────────── async function deepDive(filePath) { const domain = DOMAINS.frontend; process.stderr.write('Fetching line data...\n'); const [unitLines, e2eLines] = await Promise.all([ fetchLineData(filePath, domain.unitFlag), domain.e2eFlag ? fetchLineData(filePath, domain.e2eFlag) : Promise.resolve([]), ]); const unitMap = new Map(unitLines.map(([ln, h]) => [ln, h])); const e2eMap = new Map(e2eLines.map(([ln, h]) => [ln, h])); const all = new Set([...unitMap.keys(), ...e2eMap.keys()]); let both = 0, unitOnly = 0, e2eOnly = 0, neither = 0; const uncoveredLineNums = []; for (const ln of all) { const u = (unitMap.get(ln) ?? 0) > 0; const e = (e2eMap.get(ln) ?? 0) > 0; if (u && e) both++; else if (u) unitOnly++; else if (e) e2eOnly++; else { neither++; uncoveredLineNums.push(ln); } } console.log(`\nPer-line breakdown: ${filePath}\n`); console.log(`Lines tracked : ${all.size}`); console.log(`Unit only : ${unitOnly}`); console.log(`E2E only : ${e2eOnly}`); console.log(`Both : ${both}`); console.log(`Neither : ${neither} ← true gaps`); if (uncoveredLineNums.length) { console.log(`\nUncovered lines: ${uncoveredLineNums.sort((a, b) => a - b).join(', ')}`); } } // ── Main ─────────────────────────────────────────────────────────────────────── async function main() { if (DEEP_FILE) { await deepDive(DEEP_FILE); return; } const results = []; for (const [domainKey, domain] of activeDomains) { process.stderr.write(`\nFetching ${domain.label}...\n`); const [unitMap, e2eMap] = await Promise.all([ fetchFlagMap(domain.unitFlag), domain.e2eFlag ? fetchFlagMap(domain.e2eFlag) : Promise.resolve(null), ]); const { gaps, totalGapFiles } = findGaps(unitMap, e2eMap, domainKey); const hotPath = domain.e2eFlag ? findHotPath(unitMap, e2eMap) : []; results.push({ domainKey, domain, gaps, hotPath, hasE2E: !!domain.e2eFlag, totalGapFiles }); } const date = new Date().toISOString().split('T')[0]; const jsonPayload = { generatedAt: date, config: { gapThreshold: GAP_THRESHOLD, minLines: MIN_LINES, top: TOP }, results: results.map(({ domainKey, gaps, hotPath, totalGapFiles }) => ({ domain: domainKey, totalGapFiles, gaps, hotPath, })), }; // Write JSON to file alongside any other output mode (avoids a second API call in CI) if (OUT_JSON_FILE) { const { writeFileSync } = await import('node:fs'); writeFileSync(OUT_JSON_FILE, JSON.stringify(jsonPayload, null, 2)); process.stderr.write(` JSON written to ${OUT_JSON_FILE}\n`); } if (OUTPUT_JSON) { console.log(JSON.stringify(jsonPayload, null, 2)); return; } if (OUTPUT_MD) { console.log(renderMarkdown(results, date)); return; } console.log( `\nCoverage Gap Analysis | ${date} | gap <${GAP_THRESHOLD}% | min ${MIN_LINES} lines`, ); for (const { domain, gaps, hotPath, hasE2E, totalGapFiles } of results) { printGapTable(domain.label, gaps, hasE2E, totalGapFiles); printHotPathTable(hotPath); } console.log(`\n${'─'.repeat(W)}`); console.log('Tip: --deep= per-line unit vs E2E breakdown'); console.log(' --json AI-ready structured output'); console.log(' --domain=nodes nodes-only gaps\n'); } // Allow pure-function imports in tests without triggering network calls if (process.argv[1] === new URL(import.meta.url).pathname) { main().catch((err) => { console.error('Error:', err.message); process.exit(1); }); }