mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
ci: Add merge queue observability
This commit is contained in:
parent
d51c9a92ed
commit
c2923db766
12
.github/CI-TELEMETRY.md
vendored
12
.github/CI-TELEMETRY.md
vendored
|
|
@ -86,6 +86,18 @@ return 'blacksmith';
|
|||
| Build stats | `.github/scripts/send-build-stats.mjs` | Per-package build duration, cache hit/miss, run total |
|
||||
| Docker stats | `.github/scripts/send-docker-stats.mjs` | Image size per platform, docker build duration |
|
||||
| Container stack | `packages/testing/containers/telemetry.ts` | E2E stack startup times per service |
|
||||
| Merge queue | `.github/scripts/send-merge-queue-stats.mjs` | Enqueue and dequeue counts, dequeue outcome, queue duration |
|
||||
|
||||
### Merge Queue Metrics
|
||||
|
||||
`merge-queue-event` counts each enqueue and dequeue event. Dequeue events have
|
||||
an `outcome` dimension with the value `merged`, `manual`, or `rejected`. They
|
||||
also have the GitHub `reason` dimension. Use these dimensions to calculate the
|
||||
rejection rate and to group rejections by cause.
|
||||
|
||||
`merge-queue-duration` records the seconds between queue entry and queue exit.
|
||||
It has the same dequeue dimensions. Use this metric to monitor latency by
|
||||
outcome and to alert on percentile changes.
|
||||
|
||||
## Secrets
|
||||
|
||||
|
|
|
|||
120
.github/scripts/send-merge-queue-stats.mjs
vendored
Normal file
120
.github/scripts/send-merge-queue-stats.mjs
vendored
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Sends merge queue outcomes and durations to the unified QA metrics webhook.
|
||||
*
|
||||
* GitHub records queue entry and removal times in the pull request timeline.
|
||||
* The dequeue webhook does not include the entry time, so this script reads the
|
||||
* timeline and pairs the current removal with its preceding queue entry.
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
import { metric, sendMetrics } from './send-metrics.mjs';
|
||||
|
||||
const SUCCESS_REASONS = new Set(['MERGE', 'ALREADY_MERGED']);
|
||||
|
||||
export function classifyDequeue(reason) {
|
||||
if (SUCCESS_REASONS.has(reason)) return 'merged';
|
||||
if (reason === 'MANUAL') return 'manual';
|
||||
return 'rejected';
|
||||
}
|
||||
|
||||
export function findQueueDuration(timeline, targetTime) {
|
||||
const queueEvents = timeline
|
||||
.filter(({ event }) => event === 'added_to_merge_queue' || event === 'removed_from_merge_queue')
|
||||
.map((event) => ({ ...event, timestamp: Date.parse(event.created_at) }))
|
||||
.filter(({ timestamp }) => Number.isFinite(timestamp))
|
||||
.sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
const removals = queueEvents.filter(({ event }) => event === 'removed_from_merge_queue');
|
||||
if (removals.length === 0) return null;
|
||||
|
||||
const targetTimestamp = Date.parse(targetTime);
|
||||
const removal = Number.isFinite(targetTimestamp)
|
||||
? removals.reduce((nearest, candidate) =>
|
||||
Math.abs(candidate.timestamp - targetTimestamp) <
|
||||
Math.abs(nearest.timestamp - targetTimestamp)
|
||||
? candidate
|
||||
: nearest,
|
||||
)
|
||||
: removals.at(-1);
|
||||
|
||||
const entry = queueEvents
|
||||
.filter(
|
||||
({ event, timestamp }) => event === 'added_to_merge_queue' && timestamp <= removal.timestamp,
|
||||
)
|
||||
.at(-1);
|
||||
|
||||
if (!entry) return null;
|
||||
return (removal.timestamp - entry.timestamp) / 1000;
|
||||
}
|
||||
|
||||
function nextPage(linkHeader) {
|
||||
if (!linkHeader) return null;
|
||||
const next = linkHeader
|
||||
.split(',')
|
||||
.map((part) => part.trim().match(/^<([^>]+)>; rel="([^"]+)"$/))
|
||||
.find((match) => match?.[2] === 'next');
|
||||
return next?.[1] ?? null;
|
||||
}
|
||||
|
||||
export async function fetchTimeline(repository, prNumber, token, fetchImpl = fetch) {
|
||||
let url = `https://api.github.com/repos/${repository}/issues/${prNumber}/timeline?per_page=100`;
|
||||
const timeline = [];
|
||||
|
||||
while (url) {
|
||||
const response = await fetchImpl(url, {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
});
|
||||
if (!response.ok)
|
||||
throw new Error(`GitHub timeline request failed with status ${response.status}`);
|
||||
|
||||
timeline.push(...(await response.json()));
|
||||
url = nextPage(response.headers.get('link'));
|
||||
}
|
||||
|
||||
return timeline;
|
||||
}
|
||||
|
||||
export async function buildMetrics(event, token, fetchImpl = fetch) {
|
||||
const action = event.action;
|
||||
if (action === 'enqueued') {
|
||||
return [metric('merge-queue-event', 1, 'count', { action })];
|
||||
}
|
||||
|
||||
const reason = event.reason ?? 'UNKNOWN';
|
||||
const outcome = classifyDequeue(reason);
|
||||
const dimensions = { action: 'dequeued', outcome, reason };
|
||||
const metrics = [metric('merge-queue-event', 1, 'count', dimensions)];
|
||||
|
||||
try {
|
||||
const timeline = await fetchTimeline(
|
||||
event.repository.full_name,
|
||||
event.pull_request.number,
|
||||
token,
|
||||
fetchImpl,
|
||||
);
|
||||
const duration = findQueueDuration(timeline, event.pull_request.updated_at);
|
||||
if (duration !== null) metrics.push(metric('merge-queue-duration', duration, 's', dimensions));
|
||||
else console.warn('[metrics] Could not find a matching merge queue entry in the PR timeline.');
|
||||
} catch (error) {
|
||||
console.warn(`[metrics] Could not calculate merge queue duration: ${error.message}`);
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const event = JSON.parse(await readFile(process.env.GITHUB_EVENT_PATH, 'utf8'));
|
||||
const metrics = await buildMetrics(event, process.env.GITHUB_TOKEN);
|
||||
await sendMetrics(metrics, 'merge-queue');
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((error) => console.warn(`[metrics] send failed: ${error.message}`));
|
||||
}
|
||||
71
.github/scripts/send-merge-queue-stats.test.mjs
vendored
Normal file
71
.github/scripts/send-merge-queue-stats.test.mjs
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
|
||||
import { buildMetrics, classifyDequeue, findQueueDuration } from './send-merge-queue-stats.mjs';
|
||||
|
||||
test('classifies dequeue reasons', () => {
|
||||
assert.equal(classifyDequeue('MERGE'), 'merged');
|
||||
assert.equal(classifyDequeue('ALREADY_MERGED'), 'merged');
|
||||
assert.equal(classifyDequeue('MANUAL'), 'manual');
|
||||
assert.equal(classifyDequeue('CI_FAILURE'), 'rejected');
|
||||
});
|
||||
|
||||
test('finds the queue duration for the removal nearest to the webhook time', () => {
|
||||
const timeline = [
|
||||
{ event: 'added_to_merge_queue', created_at: '2026-08-13T11:58:15Z' },
|
||||
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:08:17Z' },
|
||||
{ event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' },
|
||||
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' },
|
||||
];
|
||||
|
||||
assert.equal(findQueueDuration(timeline, '2026-08-13T12:08:18Z'), 602);
|
||||
assert.equal(findQueueDuration(timeline, '2026-08-13T12:22:42Z'), 163);
|
||||
});
|
||||
|
||||
test('builds an enqueue count without requesting the timeline', async () => {
|
||||
const fetchImpl = () => assert.fail('fetch must not run for enqueue events');
|
||||
const metrics = await buildMetrics({ action: 'enqueued' }, 'token', fetchImpl);
|
||||
|
||||
assert.deepEqual(metrics, [
|
||||
{
|
||||
metric_name: 'merge-queue-event',
|
||||
value: 1,
|
||||
unit: 'count',
|
||||
dimensions: { action: 'enqueued' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('builds dequeue count and duration metrics', async () => {
|
||||
const fetchImpl = async () => ({
|
||||
ok: true,
|
||||
json: async () => [
|
||||
{ event: 'added_to_merge_queue', created_at: '2026-08-13T12:20:00Z' },
|
||||
{ event: 'removed_from_merge_queue', created_at: '2026-08-13T12:22:43Z' },
|
||||
],
|
||||
headers: new Headers(),
|
||||
});
|
||||
const event = {
|
||||
action: 'dequeued',
|
||||
reason: 'CI_FAILURE',
|
||||
repository: { full_name: 'n8n-io/n8n' },
|
||||
pull_request: { number: 123, updated_at: '2026-08-13T12:22:43Z' },
|
||||
};
|
||||
|
||||
const metrics = await buildMetrics(event, 'token', fetchImpl);
|
||||
|
||||
assert.deepEqual(metrics, [
|
||||
{
|
||||
metric_name: 'merge-queue-event',
|
||||
value: 1,
|
||||
unit: 'count',
|
||||
dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' },
|
||||
},
|
||||
{
|
||||
metric_name: 'merge-queue-duration',
|
||||
value: 163,
|
||||
unit: 's',
|
||||
dimensions: { action: 'dequeued', outcome: 'rejected', reason: 'CI_FAILURE' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -1,17 +1,37 @@
|
|||
name: 'Util: Notify Merge Queue Removal'
|
||||
name: 'Util: Merge Queue Status'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [dequeued]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
types: [enqueued, dequeued]
|
||||
|
||||
jobs:
|
||||
observe:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
steps:
|
||||
# Read trusted scripts because this job has access to repository secrets.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
sparse-checkout: |
|
||||
.github/scripts/send-merge-queue-stats.mjs
|
||||
.github/scripts/send-metrics.mjs
|
||||
- name: Report merge queue metrics
|
||||
run: node .github/scripts/send-merge-queue-stats.mjs
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
QA_METRICS_WEBHOOK_URL: ${{ secrets.QA_METRICS_WEBHOOK_URL }}
|
||||
QA_METRICS_WEBHOOK_USER: ${{ secrets.QA_METRICS_WEBHOOK_USER }}
|
||||
QA_METRICS_WEBHOOK_PASSWORD: ${{ secrets.QA_METRICS_WEBHOOK_PASSWORD }}
|
||||
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
# MERGE/ALREADY_MERGED are successful exits, MANUAL is the author's own doing
|
||||
if: ${{ !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }}
|
||||
if: ${{ github.event.action == 'dequeued' && !contains(fromJSON('["MERGE", "ALREADY_MERGED", "MANUAL"]'), github.event.reason) }}
|
||||
steps:
|
||||
# A failing PR can dequeue others behind it, so the reason may not be this PR's fault
|
||||
- run: |
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user