From f8794d2bbf0a0eca445a99ba7f4ae1ff1a3e3cc2 Mon Sep 17 00:00:00 2001
From: Chris Sherwood <chris@crosstalksolutions.com>
Date: Thu, 23 Jul 2026 14:50:23 -0700
Subject: [PATCH] fix(benchmark): warm the AI model before timed runs for
 reproducible scores

The AI benchmark evicted resident models (forcing the benchmark model cold)
and then went straight into the timed median-of-N loop with no warm-up. On a
cold box the model-load + GPU spin-up cost landed inside the timed runs
(observed: 173s TTFT / 5.83 tok/s vs a ~80 tok/s warm steady-state), and
consecutive runs weren't isolated (a prior run left the model warm). Because
the AI channel is uncapped and ~30% of the composite, the same machine could
post a ~2x-different NOMAD Score depending on warm/cold state (888 vs 1958
observed back-to-back).

Add one discarded warm-up inference after eviction and before the timed loop
so every timed run measures warm, steady-state throughput. Cold and warm
invocations now converge on the same score. Best-effort: a warm-up hiccup
never fails the run.

Closes #1139
---
 admin/app/services/benchmark_service.ts | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts
index 03d1f1b9..4eaa9b6f 100644
--- a/admin/app/services/benchmark_service.ts
+++ b/admin/app/services/benchmark_service.ts
@@ -883,10 +883,25 @@ export class BenchmarkService {
       // model the whole GPU. Best-effort: never fail the run on an eviction hiccup.
       await this._unloadResidentModels(ollamaAPIURL)
 
+      // Warm-up pass (result discarded). The eviction above forces the benchmark
+      // model cold, so without this the first *timed* inference would pay the full
+      // model-load + GPU spin-up cost. On a cold box that lands as a huge outlier
+      // (observed: 173s TTFT / 5.83 tok/s on a run whose warm steady-state was
+      // ~80 tok/s), and since the AI channel is uncapped and ~30% of the composite
+      // it swings the whole NOMAD Score ~2x between a cold first run and a warm
+      // re-run of the same machine. Measure warm, steady-state throughput instead:
+      // load the model and warm the context here, then time the median-of-N below.
+      // Best-effort — a warm-up hiccup must not fail the run (the timed loop will
+      // surface any real inference error). See issue #1139.
+      this._updateStatus('running_ai', 'Warming up AI model...')
+      await this._runSingleAIInference(ollamaAPIURL).catch(() => null)
+
       // Run inference AI_BENCHMARK_RUNS times and take the median run by tok/s (W7).
       // A single inference is noisy (scheduler, thermal, cache); the median damps
-      // outliers without a warmup-run bias. TTFT is reported from the same run that
-      // is the tok/s median, so the two figures always describe one real inference.
+      // outliers. The model is already warm (see the warm-up pass above), so every
+      // timed run measures steady-state throughput. TTFT is reported from the same
+      // run that is the tok/s median, so the two figures always describe one real
+      // inference.
       const runs: { tps: number; ttft: number }[] = []
       for (let i = 0; i < AI_BENCHMARK_RUNS; i++) {
         this._updateStatus('running_ai', `Running AI benchmark (${i + 1}/${AI_BENCHMARK_RUNS})...`)