ci: Add merge queue observability

This commit is contained in:
Charlie Kolb 2026-10-01 11:14:03 +02:00
parent d51c9a92ed
commit c2923db766
No known key found for this signature in database
4 changed files with 229 additions and 6 deletions

View File

@ -86,6 +86,18 @@ return 'blacksmith';
| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total |
| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration |
| Container stack | `packages/testing/containers/telemetry.ts` | E2E stack startup times per service |
| Merge queue | `.github/scripts/send-merge-queue-stats.mjs` | Enqueue and dequeue counts, dequeue outcome, queue duration |
### Merge Queue Metrics
`merge-queue-event` counts each enqueue and dequeue event. Dequeue events have
an `outcome` dimension with the value `merged`, `manual`, or `rejected`. They
also have the GitHub `reason` dimension. Use these dimensions to calculate the
rejection rate and to group rejections by cause.
`merge-queue-duration` records the seconds between queue entry and queue exit.
It has the same dequeue dimensions. Use this metric to monitor latency by
outcome and to alert on percentile changes.
## Secrets

View File

@ -0,0 +1,120 @@
#!/usr/bin/env node
/**
* Sends merge queue outcomes and durations to the unified QA metrics webhook.
*
* GitHub records queue entry and removal times in the pull request timeline.
* The dequeue webhook does not include the entry time, so this script reads the
* timeline and pairs the current removal with its preceding queue entry.
*/
import { readFile } from 'node:fs/promises';
import { pathToFileURL } from 'node:url';
import { metric, sendMetrics } from './send-metrics.mjs';
const SUCCESS_REASONS = new Set(['MERGE', 'ALREADY_MERGED']);
export function classifyDequeue(reason) {
if (SUCCESS_REASONS.has(reason)) return 'merged';
if (reason === 'MANUAL') return 'manual';
return 'rejected';
}
export function findQueueDuration(timeline, targetTime) {
const queueEvents = timeline
.filter(({ event }) => event === 'added_to_merge_queue' || event === 'removed_from_merge_queue')
.map((event) => ({ ...event, timestamp: Date.parse(event.created_at) }))
.filter(({ timestamp }) => Number.isFinite(timestamp))
.sort((a, b) => a.timestamp - b.timestamp);
const removals = queueEvents.filter(({ event }) => event === 'removed_from_merge_queue');
if (removals.length === 0) return null;
const targetTimestamp = Date.parse(targetTime);
const removal = Number.isFinite(targetTimestamp)
? removals.reduce((nearest, candidate) =>
Math.abs(candidate.timestamp - targetTimestamp) <
Math.abs(nearest.timestamp - targetTimestamp)
? candidate
: nearest,
)
: removals.at(-1);
const entry = queueEvents
.filter(
({ event, timestamp }) => event === 'added_to_merge_queue' && timestamp <= removal.timestamp,
)
.at(-1);
if (!entry) return null;
return (removal.timestamp - entry.timestamp) / 1000;
}
function nextPage(linkHeader) {
if (!linkHeader) return null;
const next = linkHeader
.split(',')
.map((part) => part.trim().match(/^<([^>]+)>; rel="([^"]+)"$/))
.find((match) => match?.[2] === 'next');
return next?.[1] ?? null;
}
export async function fetchTimeline(repository, prNumber, token, fetchImpl = fetch) {
let url = `https://api.github.com/repos/${repository}/issues/${prNumber}/timeline?per_page=100`;
const timeline = [];
while (url) {
const response = await fetchImpl(url, {
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${token}`,
'X-GitHub-Api-Version': '2022-11-28',
},
});
if (!response.ok)
throw new Error(`GitHub timeline request failed with status ${response.status}`);
timeline.push(...(await response.json()));
url = nextPage(response.headers.get('link'));
}
return timeline;
}
export async function buildMetrics(event, token, fetchImpl = fetch) {
const action = event.action;
if (action === 'enqueued') {
return [metric('merge-queue-event', 1, 'count', { action })];
}
const reason = event.reason ?? 'UNKNOWN';
const outcome = classifyDequeue(reason);
const dimensions = { action: 'dequeued', outcome, reason };
const metrics = [metric('merge-queue-event', 1, 'count', dimensions)];
try {
const timeline = await fetchTimeline(
event.repository.full_name,
event.pull_request.number,
token,
fetchImpl,
);
const duration = findQueueDuration(timeline, event.pull_request.updated_at);
if (duration !== null) metrics.push(metric('merge-queue-duration', duration, 's', dimensions));
else console.warn('[metrics] Could not find a matching merge queue entry in the PR timeline.');
} catch (error) {
console.warn(`[metrics] Could not calculate merge queue duration: ${error.message}`);
}
return metrics;
}
async function main() {
const event = JSON.parse(await readFile(process.env.GITHUB_EVENT_PATH, 'utf8'));
const metrics = await buildMetrics(event, process.env.GITHUB_TOKEN);
await sendMetrics(metrics, 'merge-queue');
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => console.warn(`[metrics] send failed: ${error.message}`));
}

View File

@ -0,0 +1,71 @@
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { buildMetrics, classifyDequeue, findQueueDuration } from './send-merge-queue-stats.mjs';
test('classifies dequeue reasons', () => {
assert.equal(classifyDequeue('MERGE'), 'merged');
assert.equal(classifyDequeue('ALREADY_MERGED'), 'merged');
assert.equal(classifyDequeue('MANUAL'), 'manual');
assert.equal(classifyDequeue('CI_FAILURE'), 'rejected');
});
test('finds the queue duration for the removal nearest to the webhook time', () => {
const timeline = [
{ event: 'added_to_merge_queue', created_at: '2026-08-13T11:58:15Z' },
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:08:17Z' },
{ event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' },
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' },
];
assert.equal(findQueueDuration(timeline, '2026-08-13T12:08:18Z'), 602);
assert.equal(findQueueDuration(timeline, '2026-08-13T12:22:42Z'), 163);
});
test('builds an enqueue count without requesting the timeline', async () => {
const fetchImpl = () => assert.fail('fetch must not run for enqueue events');
const metrics = await buildMetrics({ action: 'enqueued' }, 'token', fetchImpl);
assert.deepEqual(metrics, [
{
metric_name: 'merge-queue-event',
value: 1,
unit: 'count',
dimensions: { action: 'enqueued' },
},
]);
});
test('builds dequeue count and duration metrics', async () => {
const fetchImpl = async () => ({
ok: true,
json: async () => [
{ event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' },
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' },
],
headers: new Headers(),
});
const event = {
action: 'dequeued',
reason: 'CI_FAILURE',
repository: { full_name: 'n8n-io/n8n' },
pull_request: { number: 123, updated_at: '2026-08-13T12:22:43Z' },
};
const metrics = await buildMetrics(event, 'token', fetchImpl);
assert.deepEqual(metrics, [
{
metric_name: 'merge-queue-event',
value: 1,
unit: 'count',
dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' },
},
{
metric_name: 'merge-queue-duration',
value: 163,
unit: 's',
dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' },
},
]);
});

View File

@ -1,17 +1,37 @@
name: 'Util: Notify Merge Queue Removal'
name: 'Util: Merge Queue Status'
on:
pull_request:
types: [dequeued]
permissions:
pull-requests: write
types: [enqueued, dequeued]
jobs:
observe:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
steps:
# Read trusted scripts because this job has access to repository secrets.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
sparse-checkout: |
.github/scripts/send-merge-queue-stats.mjs
.github/scripts/send-metrics.mjs
- name: Report merge queue metrics
run: node .github/scripts/send-merge-queue-stats.mjs
env:
GITHUB_TOKEN: ${{ github.token }}
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
notify:
runs-on: ubuntu-latest
permissions:
pull-requests: write
# MERGE/ALREADY_MERGED are successful exits, MANUAL is the author's own doing
if: ${{ !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }}
if: ${{ github.event.action == 'dequeued' && !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }}
steps:
# A failing PR can dequeue others behind it, so the reason may not be this PR's fault
- run: |