project-nomad/admin/tests/unit/eval_retrieval_metrics.spec.ts

/**
 * Tests for the retrieval metrics.
 *
 * Expected values are hand-computed from the formulas in the module doc, not
 * captured from a run — a snapshot test of a wrong implementation just freezes
 * the wrong answer. nDCG in particular is easy to get subtly wrong (log base,
 * off-by-one in the rank, what the ideal ranking is normalized against), and a
 * silently wrong metric is worse than no metric.
 *
 *   npm run test:unit
 */
import * as assert from 'node:assert/strict'
import { test } from 'node:test'

import {
  aggregate,
  aggregateByTag,
  describeScores,
  hitRateAtK,
  meanOf,
  ndcgAtK,
  precisionAtK,
  recallAtK,
  reciprocalRank,
  scoreCase,
  toDocumentRanking,
  type RetrievalCase,
  type ScoredChunk,
} from '../../app/utils/eval/retrieval_metrics.js'

/** Build a ranked chunk list from doc ids, best first. */
const chunks = (...docIds: Array<string | null>): ScoredChunk[] =>
  docIds.map((docId, i) => ({ docId, score: 1 - i * 0.05, semanticScore: 0.9 - i * 0.05 }))

const close = (actual: number | null, expected: number, msg?: string) => {
  assert.ok(actual !== null, msg ?? 'expected a value, got null')
  assert.ok(Math.abs(actual - expected) < 1e-9, `${msg ?? ''} expected ${expected}, got ${actual}`)
}

// --- document ranking ---------------------------------------------------------

test('document ranking keeps each document at its best rank', () => {
  assert.deepEqual(toDocumentRanking(chunks('a', 'b', 'a', 'c', 'b')), ['a', 'b', 'c'])
})

test('document ranking drops chunks with no resolvable document', () => {
  assert.deepEqual(toDocumentRanking(chunks('a', null, 'b')), ['a', 'b'])
})

// --- recall -------------------------------------------------------------------

test('recall@k finds both documents of a multi-hop question', () => {
  close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 5), 1)
})

test('recall@k is partial when only one required document is in the window', () => {
  // 'b' sits at rank 3, outside k=2. One of two relevant docs found.
  close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 2), 0.5)
})

test('recall@k counts distinct documents, not chunks', () => {
  // Three chunks, all from 'a'. That is one document found, not three.
  close(recallAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 0.5)
})

test('recall is null for out-of-corpus cases rather than 0', () => {
  // Scoring these as 0 would drag the mean down for questions that are
  // supposed to retrieve nothing — punishing correct behaviour.
  assert.equal(recallAtK(chunks('x'), [], 5), null)
})

test('recall is 0 when nothing relevant was retrieved', () => {
  close(recallAtK(chunks('x', 'y'), ['a'], 5), 0)
})

// --- hit rate -----------------------------------------------------------------

test('hit rate is 1 when any relevant document appears', () => {
  assert.equal(hitRateAtK(chunks('x', 'a'), ['a', 'b'], 5), 1)
})

test('hit rate and recall diverge on multi-hop, which is the point of having both', () => {
  const retrieved = chunks('a', 'x', 'y')
  assert.equal(hitRateAtK(retrieved, ['a', 'b'], 5), 1)
  close(recallAtK(retrieved, ['a', 'b'], 5), 0.5)
})

test('hit rate respects the k window', () => {
  assert.equal(hitRateAtK(chunks('x', 'y', 'a'), ['a'], 2), 0)
})

// --- precision ----------------------------------------------------------------

test('precision@k is chunk level and deliberately not deduped', () => {
  // Two of four injected chunks are noise, regardless of how many documents
  // they came from — the small model pays for all four.
  close(precisionAtK(chunks('a', 'a', 'x', 'y'), ['a'], 4), 0.5)
})

test('precision divides by what was actually retrieved, not by k', () => {
  // Returning 2 good chunks should score 1.0, not 0.4 — declining to pad the
  // context with noise is correct behaviour and must not be penalised.
  close(precisionAtK(chunks('a', 'a'), ['a'], 5), 1)
})

test('precision is 0 when nothing was retrieved', () => {
  close(precisionAtK([], ['a'], 5), 0)
})

// --- reciprocal rank ----------------------------------------------------------

test('reciprocal rank is 1 when the first chunk is relevant', () => {
  close(reciprocalRank(chunks('a', 'x'), ['a']), 1)
})

test('reciprocal rank is 1/3 when the first relevant chunk is third', () => {
  close(reciprocalRank(chunks('x', 'y', 'a'), ['a']), 1 / 3)
})

test('reciprocal rank is 0 when nothing relevant was retrieved', () => {
  close(reciprocalRank(chunks('x', 'y'), ['a']), 0)
})

// --- nDCG ---------------------------------------------------------------------

test('nDCG is 1 when the single relevant document ranks first', () => {
  close(ndcgAtK(chunks('a', 'x', 'y'), ['a'], 5), 1)
})

test('nDCG at rank 2 equals 1/log2(3)', () => {
  // DCG  = 1/log2(2+1) = 1/1.58496 = 0.63093
  // IDCG = 1/log2(1+1) = 1
  close(ndcgAtK(chunks('x', 'a'), ['a'], 5), 1 / Math.log2(3))
})

test('nDCG normalizes against the known relevant count, not the retrieved set', () => {
  // One of three required documents, ranked first.
  //   DCG  = 1/log2(2)                          = 1
  //   IDCG = 1/log2(2) + 1/log2(3) + 1/log2(4)  = 1 + 0.63093 + 0.5 = 2.13093
  // Normalizing against the retrieved set instead would report a perfect 1.0
  // for a run that missed two thirds of the answer.
  const idcg = 1 + 1 / Math.log2(3) + 1 / Math.log2(4)
  close(ndcgAtK(chunks('a', 'x', 'y'), ['a', 'b', 'c'], 5), 1 / idcg)
})

test('nDCG punishes ordering even when recall is unchanged', () => {
  // This is the regression nDCG exists to catch: same documents retrieved,
  // pushed down the list, recall identical.
  const good = chunks('a', 'b', 'x', 'y')
  const bad = chunks('x', 'y', 'a', 'b')
  close(recallAtK(good, ['a', 'b'], 5), 1)
  close(recallAtK(bad, ['a', 'b'], 5), 1)
  const nGood = ndcgAtK(good, ['a', 'b'], 5)!
  const nBad = ndcgAtK(bad, ['a', 'b'], 5)!
  assert.equal(nGood, 1)
  assert.ok(nBad < nGood, `expected ${nBad} < ${nGood}`)
})

test('nDCG ideal ranking is capped at k', () => {
  // Three relevant docs but k=1: the best achievable is one hit at rank 1.
  close(ndcgAtK(chunks('a', 'b', 'c'), ['a', 'b', 'c'], 1), 1)
})

test('nDCG collapses duplicate chunks from the same document', () => {
  // Three chunks of 'a' must not be credited as three separate hits.
  const idcg = 1 + 1 / Math.log2(3)
  close(ndcgAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 1 / idcg)
})

test('nDCG is 0 when nothing relevant is retrieved', () => {
  close(ndcgAtK(chunks('x', 'y'), ['a'], 5), 0)
})

// --- score distribution ---------------------------------------------------------

test('describeScores reports values that actually occurred', () => {
  const d = describeScores([0.1, 0.2, 0.3, 0.4, 0.5])!
  assert.equal(d.count, 5)
  assert.equal(d.min, 0.1)
  assert.equal(d.max, 0.5)
  assert.equal(d.median, 0.3)
  close(d.mean, 0.3)
  // Nearest-rank, so every reported percentile is a real observation.
  assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p10))
  assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p90))
})

test('describeScores returns null for an empty sample', () => {
  assert.equal(describeScores([]), null)
})

test('meanOf ignores nulls and returns null when everything is null', () => {
  close(meanOf([1, null, 3]), 2)
  assert.equal(meanOf([null, null]), null)
})

// --- aggregation ----------------------------------------------------------------

const mkCase = (over: Partial<RetrievalCase>): RetrievalCase => ({
  id: 'c',
  tags: [],
  retrieved: chunks('a'),
  relevantDocIds: ['a'],
  expectRefusal: false,
  ...over,
})

test('aggregate separates answerable cases from refusal cases', () => {
  const cases = [
    mkCase({ id: 'q1', retrieved: chunks('a'), relevantDocIds: ['a'] }),
    mkCase({ id: 'q2', retrieved: [], relevantDocIds: [], expectRefusal: true }),
  ]
  const results = cases.map((c) => scoreCase(c))
  const agg = aggregate(cases, results)
  assert.equal(agg.cases, 2)
  assert.equal(agg.answerable, 1)
  // The refusal case contributes null to recall, so the mean is over q1 alone.
  close(agg.recall[5], 1)
})

test('empty rate on answerable questions surfaces an over-tight threshold', () => {
  const cases = [
    mkCase({ id: 'q1', retrieved: [] }),
    mkCase({ id: 'q2', retrieved: chunks('a') }),
  ]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
  close(agg.emptyRateOnAnswerable, 0.5)
})

test('non-empty rate on refusal questions surfaces an over-loose threshold', () => {
  const cases = [
    mkCase({ id: 'r1', retrieved: chunks('x'), relevantDocIds: [], expectRefusal: true }),
    mkCase({ id: 'r2', retrieved: [], relevantDocIds: [], expectRefusal: true }),
  ]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
  close(agg.nonEmptyRateOnRefusal, 0.5)
})

test('score distributions split relevant from irrelevant chunks', () => {
  const cases = [mkCase({ retrieved: chunks('a', 'x', 'y'), relevantDocIds: ['a'] })]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
  assert.equal(agg.relevantScores!.count, 1)
  assert.equal(agg.irrelevantScores!.count, 2)
  // The relevant chunk ranked first, so it should score above the noise.
  assert.ok(agg.relevantScores!.median > agg.irrelevantScores!.median)
})

test('refusal cases contribute only to the irrelevant score population', () => {
  const cases = [mkCase({ retrieved: chunks('x', 'y'), relevantDocIds: [], expectRefusal: true })]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
  assert.equal(agg.relevantScores, null)
  assert.equal(agg.irrelevantScores!.count, 2)
})

test('per-tag aggregation slices the same cases without recomputing them wrong', () => {
  const cases = [
    mkCase({ id: 'q1', tags: ['single-hop'], retrieved: chunks('a'), relevantDocIds: ['a'] }),
    mkCase({ id: 'q2', tags: ['multi-hop'], retrieved: chunks('a'), relevantDocIds: ['a', 'b'] }),
  ]
  const byTag = aggregateByTag(cases, cases.map((c) => scoreCase(c)))
  close(byTag['single-hop'].recall[5], 1)
  close(byTag['multi-hop'].recall[5], 0.5)
  assert.equal(byTag['single-hop'].cases, 1)
})

test('aggregate reports the semantic and final axes separately', () => {
  // A chunk carries two scores, and the two cutoffs filter different ones:
  // Qdrant's score_threshold sees the semantic score, RAG_MIN_FINAL_SCORE sees
  // the post-rerank score. Collapsing them would make one cutoff calibrate
  // against the other's percentiles, off by the reranker's boost factor.
  const cases: RetrievalCase[] = [
    {
      id: 'c1',
      tags: [],
      relevantDocIds: ['doc-a'],
      expectRefusal: false,
      retrieved: [
        { docId: 'doc-a', score: 0.9, semanticScore: 0.7 },
        { docId: 'doc-b', score: 0.5, semanticScore: 0.4 },
      ],
    },
  ]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))

  assert.equal(agg.relevantScores!.median, 0.7)
  assert.equal(agg.irrelevantScores!.median, 0.4)
  assert.equal(agg.relevantFinalScores!.median, 0.9)
  assert.equal(agg.irrelevantFinalScores!.median, 0.5)
})

test('aggregate falls back to the final score when no semantic score was recorded', () => {
  // The ablation stages record only one number per row, so the semantic axis
  // has to degrade to it rather than reporting an empty distribution.
  const cases: RetrievalCase[] = [
    {
      id: 'c1',
      tags: [],
      relevantDocIds: ['doc-a'],
      expectRefusal: false,
      retrieved: [{ docId: 'doc-a', score: 0.8 }],
    },
  ]
  const agg = aggregate(cases, cases.map((c) => scoreCase(c)))

  assert.equal(agg.relevantScores!.median, 0.8)
  assert.equal(agg.relevantFinalScores!.median, 0.8)
})