diff --git a/.github/workflows/util-update-e2e-metrics.yml b/.github/workflows/util-update-e2e-metrics.yml new file mode 100644 index 00000000000..4ffdea34e46 --- /dev/null +++ b/.github/workflows/util-update-e2e-metrics.yml @@ -0,0 +1,132 @@ +name: 'Util: Update E2E Metrics' + +on: + schedule: + # Run every Monday at 01:00 UTC, after a full week of PR-CI data. + - cron: '0 1 * * 1' + workflow_dispatch: # Allow a manual run for testing. + +concurrency: + group: update-e2e-metrics + cancel-in-progress: true + +permissions: {} + +jobs: + update-metrics: + name: Refresh E2E duration metrics + # A dispatch from a feature branch must not create an update PR from that branch. + if: | + github.ref_name == github.event.repository.default_branch && + (github.event_name == 'workflow_dispatch' || + (github.event_name == 'schedule' && github.repository == 'n8n-io/n8n')) + runs-on: ${{ vars.RUNNER_PROVIDER == 'github' && 'ubuntu-latest' || 'blacksmith-2vcpu-ubuntu-2204' }} + permissions: + contents: read + outputs: + pull-request-number: ${{ steps.create-pr.outputs.pull-request-number }} + steps: + - name: Generate GitHub App Token + id: generate-token + uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1 + with: + app-id: ${{ secrets.N8N_ASSISTANT_APP_ID }} + private-key: ${{ secrets.N8N_ASSISTANT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ steps.generate-token.outputs.token }} + persist-credentials: false + + - name: Setup Node.js and dependencies + uses: ./.github/actions/setup-nodejs + with: + # Playwright test discovery reads source files and does not need build outputs. + build-command: '' + + - name: Preserve current metrics + run: cp .github/test-metrics/playwright.json '${{ runner.temp }}/playwright-metrics-before.json' + + - name: Fetch E2E metrics from Currents + env: + CURRENTS_API_KEY: ${{ secrets.CURRENTS_API_KEY }} + run: node packages/quality/testing/playwright/scripts/fetch-currents-metrics.mjs --project=nHHLA5 + + - name: Validate refreshed metrics + env: + BEFORE: ${{ runner.temp }}/playwright-metrics-before.json + AFTER: .github/test-metrics/playwright.json + run: | + node - <<'NODE' + const fs = require('node:fs'); + + const before = JSON.parse(fs.readFileSync(process.env.BEFORE, 'utf8')); + const after = JSON.parse(fs.readFileSync(process.env.AFTER, 'utf8')); + const beforeSpecs = Object.values(before.specs ?? {}); + const afterSpecs = Object.values(after.specs ?? {}); + const beforeMeasured = beforeSpecs.filter((spec) => spec.testCount > 0).length; + const afterMeasured = afterSpecs.filter((spec) => spec.testCount > 0).length; + const specFloor = Math.floor(beforeSpecs.length * 0.96); + const measuredFloor = Math.floor(beforeMeasured * 0.8); + + if (after.source !== 'currents' || after.projectId !== 'nHHLA5') { + throw new Error('Refreshed metrics have an unexpected source or project ID'); + } + if (afterSpecs.length < specFloor) { + throw new Error(`Refreshed metrics have ${afterSpecs.length} specs. Expected at least ${specFloor}.`); + } + if (afterMeasured < measuredFloor) { + throw new Error(`Only ${afterMeasured} specs have Currents data. Expected at least ${measuredFloor}.`); + } + + console.log(`Metrics are complete: ${afterSpecs.length} specs, ${afterMeasured} with Currents data.`); + NODE + + - name: Check for changes + id: check-changes + run: | + if git diff --quiet .github/test-metrics/playwright.json; then + echo 'E2E metrics are unchanged' + echo 'has_changes=false' >> "$GITHUB_OUTPUT" + else + echo 'E2E metrics have changed' + echo 'has_changes=true' >> "$GITHUB_OUTPUT" + fi + + - name: Create Pull Request + if: steps.check-changes.outputs.has_changes == 'true' + id: create-pr + uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 + with: + token: ${{ steps.generate-token.outputs.token }} + commit-message: 'chore: Update E2E duration metrics (no-changelog)' + labels: 'automation:scheduled-update' + title: 'chore: Update E2E duration metrics (no-changelog)' + body: | + This automated PR refreshes the E2E duration and flakiness metrics + from the last seven days of PR CI runs in Currents. + + The E2E orchestrator uses these metrics to balance specs across CI + shards. A weekly refresh reduces slow shard tails as test durations + change. + + _Generated by the weekly E2E metrics update workflow._ + branch: update-e2e-metrics + base: master + delete-branch: true + author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + + approve-and-automerge: + needs: [update-metrics] + if: needs.update-metrics.outputs.pull-request-number != '' + permissions: + contents: write + pull-requests: write + uses: ./.github/workflows/util-approve-and-set-automerge.yml # zizmor: ignore[secrets-inherit] + secrets: inherit + with: + pull-request-number: ${{ needs.update-metrics.outputs.pull-request-number }} diff --git a/packages/quality/testing/playwright/docs/ORCHESTRATION.md b/packages/quality/testing/playwright/docs/ORCHESTRATION.md index ed9caf2e181..ad958f4e495 100644 --- a/packages/quality/testing/playwright/docs/ORCHESTRATION.md +++ b/packages/quality/testing/playwright/docs/ORCHESTRATION.md @@ -152,10 +152,10 @@ CURRENTS_API_KEY= node packages/quality/testing/playwright/scripts/fetch-cu This fetches the last 7 days of test durations from Currents, aggregates by spec, and writes to `.github/test-metrics/playwright.json`. The PR-CI project is `nHHLA5` (n8n-ci); the legacy `LRxcNt` project still backs the nightly e2e workflows. -**When to refresh:** -- Weekly (recommended) -- After significant test changes -- When adding new specs (optional - they get 60s default) +The `Util: Update E2E Metrics` workflow refreshes this file every Monday and +opens an automated pull request when it changes. You can also run the workflow +manually after a significant test change. New specs use the 60-second default +until Currents has data for them. Stale metrics do not cause an obvious failure. The packer still reports balanced shards, because it balances against the durations in the file. The real spread is diff --git a/packages/quality/testing/playwright/scripts/distribute-tests.mjs b/packages/quality/testing/playwright/scripts/distribute-tests.mjs index caff2847775..7b1b867cd37 100644 --- a/packages/quality/testing/playwright/scripts/distribute-tests.mjs +++ b/packages/quality/testing/playwright/scripts/distribute-tests.mjs @@ -44,8 +44,36 @@ const SELECT_AFFECTED_E2E = path.resolve(__dirname, 'select-affected-e2e.mjs'); const DISTRIBUTION_REPORTER = path.resolve(__dirname, 'distribution-counter-reporter.ts'); const PLAYWRIGHT_CLI = createRequire(import.meta.url).resolve('@playwright/test/cli'); const PLAYWRIGHT_PREFIX = path.relative(REPO_ROOT, PLAYWRIGHT_DIR) + path.sep; +const METRICS_PATH = path.join(REPO_ROOT, '.github', 'test-metrics', 'playwright.json'); +const MAX_METRICS_AGE_MS = 7 * 24 * 60 * 60 * 1000; const CONTAINER_STARTUP_TIME = 22_500; // 22.5s average per fixture +function warnIfMetricsAreStale() { + try { + const metrics = JSON.parse(readFileSync(METRICS_PATH, 'utf8')); + const updatedAt = Date.parse(metrics.updatedAt); + if (!Number.isFinite(updatedAt)) throw new Error('updatedAt is missing or invalid'); + + const ageMs = Date.now() - updatedAt; + if (ageMs <= MAX_METRICS_AGE_MS) return; + + const ageDays = Math.floor(ageMs / (24 * 60 * 60 * 1000)); + const message = `E2E duration metrics are ${ageDays} days old. Shard estimates can be inaccurate.`; + console.error( + process.env.GITHUB_ACTIONS + ? `::warning file=.github/test-metrics/playwright.json::${message}` + : `⚠ ${message}`, + ); + } catch (error) { + const message = `Could not check E2E duration metrics freshness: ${String(error)}`; + console.error( + process.env.GITHUB_ACTIONS + ? `::warning file=.github/test-metrics/playwright.json::${message}` + : `⚠ ${message}`, + ); + } +} + // Specs excluded from orchestrated distribution. Under coverage instrumentation // (no container reuse + slower execution), high-flake specs balloon into // multi-minute retry tails that tip a single shard over its timeout. Quarantine @@ -290,6 +318,7 @@ if (impactMode && filesArg) { let groupsFile; if (orchestrateMode || !matrixMode) { + warnIfMetricsAreStale(); const generated = generateDistributionGroups(project, grepInvert); groupsFile = generated.groupsPath; const candidates = includeSpecsFile diff --git a/packages/quality/testing/playwright/scripts/fetch-currents-metrics.mjs b/packages/quality/testing/playwright/scripts/fetch-currents-metrics.mjs index 038a9c35813..3ecf4f06dae 100644 --- a/packages/quality/testing/playwright/scripts/fetch-currents-metrics.mjs +++ b/packages/quality/testing/playwright/scripts/fetch-currents-metrics.mjs @@ -77,6 +77,9 @@ function getPlaywrightSpecs() { console.log(`Found ${specs.size} specs in Playwright`); return specs; } catch (e) { + if (process.env.CI) { + throw new Error(`Could not get Playwright specs: ${String(e)}`); + } console.warn('Warning: Could not get Playwright specs, skipping validation'); return null; }