diff --git a/.github/CI-TELEMETRY.md b/.github/CI-TELEMETRY.md index 60f97aad4ee..6b0fac46221 100644 --- a/.github/CI-TELEMETRY.md +++ b/.github/CI-TELEMETRY.md @@ -86,6 +86,18 @@ return 'blacksmith'; | Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total | | Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration | | Container stack | `packages/testing/containers/telemetry.ts` | E2E stack startup times per service | +| Merge queue | `.github/scripts/send-merge-queue-stats.mjs` | Enqueue and dequeue counts, dequeue outcome, queue duration | + +### Merge Queue Metrics + +`merge-queue-event` counts each enqueue and dequeue event. Dequeue events have +an `outcome` dimension with the value `merged`, `manual`, or `rejected`. They +also have the GitHub `reason` dimension. Use these dimensions to calculate the +rejection rate and to group rejections by cause. + +`merge-queue-duration` records the seconds between queue entry and queue exit. +It has the same dequeue dimensions. Use this metric to monitor latency by +outcome and to alert on percentile changes. ## Secrets diff --git a/.github/scripts/send-merge-queue-stats.mjs b/.github/scripts/send-merge-queue-stats.mjs new file mode 100644 index 00000000000..5d6a122dfee --- /dev/null +++ b/.github/scripts/send-merge-queue-stats.mjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node +/** + * Sends merge queue outcomes and durations to the unified QA metrics webhook. + * + * GitHub records queue entry and removal times in the pull request timeline. + * The dequeue webhook does not include the entry time, so this script reads the + * timeline and pairs the current removal with its preceding queue entry. + */ + +import { readFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +import { metric, sendMetrics } from './send-metrics.mjs'; + +const SUCCESS_REASONS = new Set(['MERGE', 'ALREADY_MERGED']); + +export function classifyDequeue(reason) { + if (SUCCESS_REASONS.has(reason)) return 'merged'; + if (reason === 'MANUAL') return 'manual'; + return 'rejected'; +} + +export function findQueueDuration(timeline, targetTime) { + const queueEvents = timeline + .filter(({ event }) => event === 'added_to_merge_queue' || event === 'removed_from_merge_queue') + .map((event) => ({ ...event, timestamp: Date.parse(event.created_at) })) + .filter(({ timestamp }) => Number.isFinite(timestamp)) + .sort((a, b) => a.timestamp - b.timestamp); + + const removals = queueEvents.filter(({ event }) => event === 'removed_from_merge_queue'); + if (removals.length === 0) return null; + + const targetTimestamp = Date.parse(targetTime); + const removal = Number.isFinite(targetTimestamp) + ? removals.reduce((nearest, candidate) => + Math.abs(candidate.timestamp - targetTimestamp) < + Math.abs(nearest.timestamp - targetTimestamp) + ? candidate + : nearest, + ) + : removals.at(-1); + + const entry = queueEvents + .filter( + ({ event, timestamp }) => event === 'added_to_merge_queue' && timestamp <= removal.timestamp, + ) + .at(-1); + + if (!entry) return null; + return (removal.timestamp - entry.timestamp) / 1000; +} + +function nextPage(linkHeader) { + if (!linkHeader) return null; + const next = linkHeader + .split(',') + .map((part) => part.trim().match(/^<([^>]+)>; rel="([^"]+)"$/)) + .find((match) => match?.[2] === 'next'); + return next?.[1] ?? null; +} + +export async function fetchTimeline(repository, prNumber, token, fetchImpl = fetch) { + let url = `https://api.github.com/repos/${repository}/issues/${prNumber}/timeline?per_page=100`; + const timeline = []; + + while (url) { + const response = await fetchImpl(url, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + }); + if (!response.ok) + throw new Error(`GitHub timeline request failed with status ${response.status}`); + + timeline.push(...(await response.json())); + url = nextPage(response.headers.get('link')); + } + + return timeline; +} + +export async function buildMetrics(event, token, fetchImpl = fetch) { + const action = event.action; + if (action === 'enqueued') { + return [metric('merge-queue-event', 1, 'count', { action })]; + } + + const reason = event.reason ?? 'UNKNOWN'; + const outcome = classifyDequeue(reason); + const dimensions = { action: 'dequeued', outcome, reason }; + const metrics = [metric('merge-queue-event', 1, 'count', dimensions)]; + + try { + const timeline = await fetchTimeline( + event.repository.full_name, + event.pull_request.number, + token, + fetchImpl, + ); + const duration = findQueueDuration(timeline, event.pull_request.updated_at); + if (duration !== null) metrics.push(metric('merge-queue-duration', duration, 's', dimensions)); + else console.warn('[metrics] Could not find a matching merge queue entry in the PR timeline.'); + } catch (error) { + console.warn(`[metrics] Could not calculate merge queue duration: ${error.message}`); + } + + return metrics; +} + +async function main() { + const event = JSON.parse(await readFile(process.env.GITHUB_EVENT_PATH, 'utf8')); + const metrics = await buildMetrics(event, process.env.GITHUB_TOKEN); + await sendMetrics(metrics, 'merge-queue'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => console.warn(`[metrics] send failed: ${error.message}`)); +} diff --git a/.github/scripts/send-merge-queue-stats.test.mjs b/.github/scripts/send-merge-queue-stats.test.mjs new file mode 100644 index 00000000000..9cfec79b0bd --- /dev/null +++ b/.github/scripts/send-merge-queue-stats.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { buildMetrics, classifyDequeue, findQueueDuration } from './send-merge-queue-stats.mjs'; + +test('classifies dequeue reasons', () => { + assert.equal(classifyDequeue('MERGE'), 'merged'); + assert.equal(classifyDequeue('ALREADY_MERGED'), 'merged'); + assert.equal(classifyDequeue('MANUAL'), 'manual'); + assert.equal(classifyDequeue('CI_FAILURE'), 'rejected'); +}); + +test('finds the queue duration for the removal nearest to the webhook time', () => { + const timeline = [ + { event: 'added_to_merge_queue', created_at: '2026-08-13T11:58:15Z' }, + { event: 'removed_from_merge_queue', created_at: '2026-08-13T12:08:17Z' }, + { event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' }, + { event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' }, + ]; + + assert.equal(findQueueDuration(timeline, '2026-08-13T12:08:18Z'), 602); + assert.equal(findQueueDuration(timeline, '2026-08-13T12:22:42Z'), 163); +}); + +test('builds an enqueue count without requesting the timeline', async () => { + const fetchImpl = () => assert.fail('fetch must not run for enqueue events'); + const metrics = await buildMetrics({ action: 'enqueued' }, 'token', fetchImpl); + + assert.deepEqual(metrics, [ + { + metric_name: 'merge-queue-event', + value: 1, + unit: 'count', + dimensions: { action: 'enqueued' }, + }, + ]); +}); + +test('builds dequeue count and duration metrics', async () => { + const fetchImpl = async () => ({ + ok: true, + json: async () => [ + { event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' }, + { event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' }, + ], + headers: new Headers(), + }); + const event = { + action: 'dequeued', + reason: 'CI_FAILURE', + repository: { full_name: 'n8n-io/n8n' }, + pull_request: { number: 123, updated_at: '2026-08-13T12:22:43Z' }, + }; + + const metrics = await buildMetrics(event, 'token', fetchImpl); + + assert.deepEqual(metrics, [ + { + metric_name: 'merge-queue-event', + value: 1, + unit: 'count', + dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' }, + }, + { + metric_name: 'merge-queue-duration', + value: 163, + unit: 's', + dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' }, + }, + ]); +}); diff --git a/.github/workflows/util-notify-merge-queue-removal.yml b/.github/workflows/util-notify-merge-queue-removal.yml index 798c93f50f1..998e47b6cb0 100644 --- a/.github/workflows/util-notify-merge-queue-removal.yml +++ b/.github/workflows/util-notify-merge-queue-removal.yml @@ -1,17 +1,37 @@ -name: 'Util: Notify Merge Queue Removal' +name: 'Util: Merge Queue Status' on: pull_request: - types: [dequeued] - -permissions: - pull-requests: write + types: [enqueued, dequeued] jobs: + observe: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + # Read trusted scripts because this job has access to repository secrets. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: | + .github/scripts/send-merge-queue-stats.mjs + .github/scripts/send-metrics.mjs + - name: Report merge queue metrics + run: node .github/scripts/send-merge-queue-stats.mjs + env: + GITHUB_TOKEN: ${{ github.token }} + QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }} + QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }} + QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }} + notify: runs-on: ubuntu-latest + permissions: + pull-requests: write # MERGE/ALREADY_MERGED are successful exits, MANUAL is the author's own doing - if: ${{ !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }} + if: ${{ github.event.action == 'dequeued' && !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }} steps: # A failing PR can dequeue others behind it, so the reason may not be this PR's fault - run: |