import { spawn } from 'child_process'; import { randomUUID } from 'node:crypto'; import * as os from 'os'; import type { StackConfig } from './services/types'; export interface StackTelemetryRecord { /** Unique identity for one stack startup attempt. */ attemptId: string; /** Correlation context shared by CI and Playwright evidence. */ correlation: { profile: string | null; shard: string | null; worker: string | null; retry: number | null; restartReason: string | null; }; /** ISO timestamp when stack creation started */ timestamp: string; /** Git context from environment */ git: { sha: string; branch: string; pr?: number; }; /** CI context from GitHub Actions environment */ ci: { runId?: string; job?: string; workflow?: string; attempt?: number; }; /** Runner info - provider detected from env vars, specs from runtime */ runner: { provider: 'github' | 'blacksmith' | 'local'; cpuCores: number; memoryGb: number; }; /** Stack configuration */ stack: { type: 'single' | 'queue' | 'multi-main'; mains: number; workers: number; postgres: boolean; services: string[]; }; /** Timing metrics in milliseconds */ timing: { total: number; network: number; n8nStartup: number; services: Record; }; /** Stage evidence for this startup attempt, including failed stages. */ stages: StartupStageRecord[]; /** Container counts */ containers: { total: number; services: number; n8n: number; }; /** Outcome */ success: boolean; failurePhase?: string; errorMessage?: string; } export interface StartupStageRecord { name: string; source: 'local' | 'hosted' | 'unknown'; startedAt: string; elapsedMs: number; outcome: 'success' | 'failure' | 'cancelled'; errorMessage?: string; } function getGitContext(): StackTelemetryRecord['git'] { const ref = process.env.GITHUB_REF ?? ''; const prMatch = ref.match(/refs\/pull\/(\d+)/); return { sha: process.env.GITHUB_SHA?.slice(0, 8) ?? 'local', branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? 'local', pr: prMatch ? parseInt(prMatch[1], 10) : undefined, }; } function getCIContext(): StackTelemetryRecord['ci'] { return { runId: process.env.GITHUB_RUN_ID, job: process.env.GITHUB_JOB, workflow: process.env.GITHUB_WORKFLOW, attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : undefined, }; } function getRunnerProvider(): StackTelemetryRecord['runner']['provider'] { // Not in CI = local development if (!process.env.CI) { return 'local'; } // GitHub-hosted runners explicitly identify themselves if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') { return 'github'; } // Everything else in CI is Blacksmith return 'blacksmith'; } function getRunnerInfo(): StackTelemetryRecord['runner'] { const cpus = os.cpus(); return { provider: getRunnerProvider(), cpuCores: cpus.length, memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10, }; } function inferStackType(config: StackConfig): 'single' | 'queue' | 'multi-main' { if ((config.mains ?? 1) > 1) return 'multi-main'; if ((config.workers ?? 0) > 0) return 'queue'; return 'single'; } function inferPostgres(config: StackConfig): boolean { const isQueueMode = (config.mains ?? 1) > 1 || (config.workers ?? 0) > 0; const hasKeycloak = config.services?.includes('keycloak') ?? false; return (config.postgres ?? false) || isQueueMode || hasKeycloak; } function safeValue(value: string | undefined): string | null { if (!value || !/^[a-z0-9_.:-]{1,64}$/i.test(value)) return null; return value; } function sanitizeError(message: string): string { return message .replace(/([?&](?:token|key|secret|password|authorization)=)[^&\s]+/gi, '$1[REDACTED]') .replace(/(Bearer\s+|Basic\s+)[^\s]+/gi, '$1[REDACTED]') .replace(/((?:token|key|secret|password|authorization)\s*[:=]\s*)[^\s,;]+/gi, '$1[REDACTED]') .replace(/\bAKIA[0-9A-Z]{16}\b/g, '[REDACTED]') .replace(/\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g, '[REDACTED]') .replace(/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g, '[REDACTED]') .replace(/\bsk-[A-Za-z0-9_-]{20,}\b/g, '[REDACTED]') .replace(/(https?:\/\/)([^\s/@]+):([^\s/@]+)@/gi, '$1[REDACTED]@'); } function getErrorMessage(error: unknown): string | undefined { if (!error) return undefined; if (error instanceof Error) return error.message; if (typeof error === 'string') return error; try { return JSON.stringify(error); } catch { return undefined; } } function getCorrelationContext(config: StackConfig): StackTelemetryRecord['correlation'] { const retry = Number(process.env.GITHUB_RUN_ATTEMPT ?? ''); const restartReason = process.env.N8N_TEST_RESTART_REASON; const knownRestartReasons = new Set([ 'initial-start', 'retry', 'profile-transition', 'intentional-isolation', 'unknown', ]); const resolvedRestartReason = restartReason ? knownRestartReasons.has(restartReason) ? restartReason : 'unknown' : null; return { profile: resolveProfile(config), shard: safeValue(process.env.TEST_SHARD ?? process.env.CI_NODE_INDEX), worker: safeValue(process.env.TEST_WORKER_INDEX ?? process.env.PLAYWRIGHT_WORKER_INDEX), retry: Number.isFinite(retry) && retry > 0 ? retry : null, restartReason: resolvedRestartReason, }; } function resolveProfile(config: StackConfig): string { const configured = process.env.N8N_TEST_PROFILE ?? process.env.TEST_PROFILE; if (configured && /^[a-z0-9_.:-]{1,64}$/i.test(configured)) return configured; if ((config.mains ?? 1) > 1) return 'multi-main'; if ((config.workers ?? 0) > 0 || (config.webhooks ?? 0) > 0) return 'queue'; if (config.postgres || config.services?.includes('keycloak')) return 'postgres'; return 'sqlite'; } export class TelemetryRecorder { private readonly id = randomUUID(); private startTimestamp = Date.now(); private startPerf = performance.now(); private networkTime = 0; private n8nStartupTime = 0; private serviceTimings: Record = {}; private serviceCount = 0; private n8nCount = 0; private stages: StartupStageRecord[] = []; private activeStage: { name: string; source: StartupStageRecord['source']; started: number; startedAt: string; } | null = null; private failurePhase: string | undefined; constructor(private config: StackConfig) {} get attemptId(): string { return this.id; } get failurePhaseValue(): string | undefined { return this.failurePhase; } recordNetwork(durationMs: number): void { this.networkTime = durationMs; } recordService(name: string, durationMs: number): void { this.serviceTimings[name] = durationMs; this.serviceCount++; } recordN8nStartup(durationMs: number, count: number): void { this.n8nStartupTime = durationMs; this.n8nCount = count; } startStage(name: string, source: StartupStageRecord['source'] = 'local'): void { this.activeStage = { name, source, started: performance.now(), startedAt: new Date().toISOString(), }; } finishStage(outcome: StartupStageRecord['outcome'] = 'success', error?: unknown): void { if (!this.activeStage) return; const { name, source, started, startedAt } = this.activeStage; const message = getErrorMessage(error); const actualOutcome = outcome === 'failure' && error instanceof Error && error.name === 'AbortError' ? 'cancelled' : outcome; this.stages.push({ name, source, startedAt, elapsedMs: Math.max(0, Math.round(performance.now() - started)), outcome: actualOutcome, ...(message ? { errorMessage: sanitizeError(message) } : {}), }); if (actualOutcome !== 'success') this.failurePhase = name; this.activeStage = null; } setFailurePhase(phase: string): void { this.failurePhase ??= phase; } private buildRecord(success: boolean, errorMessage?: string): StackTelemetryRecord { return { attemptId: this.id, correlation: getCorrelationContext(this.config), timestamp: new Date(this.startTimestamp).toISOString(), git: getGitContext(), ci: getCIContext(), runner: getRunnerInfo(), stack: { type: inferStackType(this.config), mains: this.config.mains ?? 1, workers: this.config.workers ?? 0, postgres: inferPostgres(this.config), services: [...(this.config.services ?? [])], }, timing: { total: Math.round(performance.now() - this.startPerf), network: this.networkTime, n8nStartup: this.n8nStartupTime, services: { ...this.serviceTimings }, }, stages: [...this.stages], containers: { total: this.serviceCount + this.n8nCount, services: this.serviceCount, n8n: this.n8nCount, }, success, ...(this.failurePhase ? { failurePhase: this.failurePhase } : {}), ...(errorMessage ? { errorMessage: sanitizeError(errorMessage) } : {}), }; } /** * Flush telemetry - outputs based on environment configuration * * Output modes: * - QA_METRICS_WEBHOOK_URL: Send via detached process (non-blocking, survives parent exit) * - CONTAINER_TELEMETRY_VERBOSE=1: Full breakdown + elapsed logs (via utils.ts) * - Default: One-liner summary only */ flush(success: boolean, errorMessage?: string): void { const record = this.buildRecord(success, errorMessage); const isVerbose = process.env.CONTAINER_TELEMETRY_VERBOSE === '1'; const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL; if (webhookUrl) { this.sendToWebhook(record, webhookUrl); } if (isVerbose) { console.log(JSON.stringify(record, null, 2)); } else { this.printSummaryLine(record); } } private printSummaryLine(record: StackTelemetryRecord): void { const time = this.formatMs(record.timing.total); const containers = record.containers.total; const services = record.stack.services.length; if (record.success) { const serviceInfo = services > 0 ? `, ${services} services` : ''; console.log(`\x1b[32m✓\x1b[0m Stack ready (${time}, ${containers} containers${serviceInfo})`); } else { console.log(`\x1b[31m✗\x1b[0m Stack failed: ${record.errorMessage} (${time})`); } } private buildUnifiedPayload(record: StackTelemetryRecord): object { const runId = record.ci.runId ?? null; const repo = process.env.GITHUB_REPOSITORY ?? null; const metrics: Array<{ metric_name: string; value: number; unit: string; dimensions: Record; }> = [ ...record.stages.map((stage) => ({ metric_name: 'stack-startup-stage', value: stage.elapsedMs, unit: 'ms', dimensions: { attempt_id: record.attemptId, stage: stage.name, source: stage.source, outcome: stage.outcome, }, })), { metric_name: 'stack-startup-total', value: record.timing.total, unit: 'ms', dimensions: { attempt_id: record.attemptId, stack_type: record.stack.type, mains: record.stack.mains, workers: record.stack.workers, postgres: record.stack.postgres ? 'true' : 'false', success: record.success ? 'true' : 'false', }, }, { metric_name: 'stack-startup-network', value: record.timing.network, unit: 'ms', dimensions: { stack_type: record.stack.type }, }, { metric_name: 'stack-startup-n8n', value: record.timing.n8nStartup, unit: 'ms', dimensions: { stack_type: record.stack.type }, }, ...Object.entries(record.timing.services).map(([service, duration]) => ({ metric_name: 'stack-startup-service', value: duration, unit: 'ms', dimensions: { service, stack_type: record.stack.type }, })), { metric_name: 'container-count', value: record.containers.total, unit: 'count', dimensions: { stack_type: record.stack.type }, }, ]; return { timestamp: record.timestamp, benchmark_name: 'container-telemetry', git: { sha: record.git.sha, branch: record.git.branch, pr: record.git.pr ?? null, }, ci: { runId, runUrl: runId && repo ? `https://github.com/${repo}/actions/runs/${runId}` : null, job: record.ci.job ?? null, workflow: record.ci.workflow ?? null, attempt: record.ci.attempt ?? null, }, correlation: record.correlation, success: record.success, error_message: record.errorMessage ?? null, attempt_id: record.attemptId, stages: record.stages, failure_phase: record.failurePhase ?? null, runner: record.runner, metrics, }; } /** * Send telemetry via a detached child process. * The process runs independently and survives parent exit, ensuring delivery * even when the main process throws/exits immediately after flush(). */ private sendToWebhook(record: StackTelemetryRecord, webhookUrl: string): void { const webhookUser = process.env.QA_METRICS_WEBHOOK_USER; const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD; if (!webhookUser || !webhookPassword) { console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping telemetry.'); return; } const payload = JSON.stringify(this.buildUnifiedPayload(record)); const authHeader = `Basic ${Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64')}`; // node -e