project-nomad/admin/tests/unit/context_window.spec.ts

import { test } from 'node:test'
import assert from 'node:assert/strict'
import {
  MIN_CONTEXT,
  UNKNOWN_BACKEND_CONTEXT,
  computeKvBytesPerToken,
  estimateKvBytesPerToken,
  parseParameterBillions,
  parseUserContextCap,
  readContextLength,
  readModelfileNumCtx,
  resolveContextWindow,
  snapToLadder,
} from '../../app/utils/context_window.js'

const GB = 2 ** 30
const KIB = 1024

/** Real /api/show model_info for llama3:8b, trimmed to the keys we read. */
const LLAMA3_8B_INFO = {
  'general.architecture': 'llama',
  'llama.attention.head_count': 32,
  'llama.attention.head_count_kv': 8,
  'llama.block_count': 32,
  'llama.context_length': 8192,
  'llama.embedding_length': 4096,
}

test('reads context length via the architecture prefix', () => {
  assert.equal(readContextLength(LLAMA3_8B_INFO), 8192)
})

test('falls back to any *.context_length when architecture is missing', () => {
  assert.equal(readContextLength({ 'qwen3.context_length': 32768 }), 32768)
})

test('readContextLength copes with a Map, and with nothing', () => {
  assert.equal(readContextLength(new Map(Object.entries(LLAMA3_8B_INFO))), 8192)
  assert.equal(readContextLength(undefined), undefined)
  assert.equal(readContextLength({}), undefined)
  assert.equal(readContextLength({ 'llama.block_count': 32 }), undefined)
})

test('reads num_ctx out of the modelfile parameters block', () => {
  assert.equal(readModelfileNumCtx('num_keep 24\nnum_ctx                        16384\nstop "<|eot_id|>"'), 16384)
  assert.equal(readModelfileNumCtx('num_keep 24\nstop "<|eot_id|>"'), undefined)
  assert.equal(readModelfileNumCtx(undefined), undefined)
})

test('computes exact KV bytes per token from GGUF metadata', () => {
  // 2 (K+V) * 32 layers * 8 kv-heads * 128 head_dim * 2 bytes = 128 KiB/token,
  // which puts an 8k window at exactly 1 GiB.
  assert.equal(computeKvBytesPerToken(LLAMA3_8B_INFO), 128 * KIB)
  assert.equal(computeKvBytesPerToken(LLAMA3_8B_INFO)! * 8192, 1 * GB)
})

test('uses grouped-query KV head count, not attention head count', () => {
  // head_count 32 vs head_count_kv 8: using the wrong one overstates cost 4x and
  // would hand every GQA model a quarter of the window it can afford.
  const noGqa = { ...LLAMA3_8B_INFO, 'llama.attention.head_count_kv': 32 }
  assert.equal(computeKvBytesPerToken(noGqa), 4 * computeKvBytesPerToken(LLAMA3_8B_INFO)!)
})

test('KV computation returns undefined on incomplete metadata', () => {
  assert.equal(computeKvBytesPerToken({ 'general.architecture': 'llama' }), undefined)
  assert.equal(computeKvBytesPerToken({}), undefined)
  assert.equal(computeKvBytesPerToken(undefined), undefined)
})

test('KV estimate falls back by parameter size, monotonically', () => {
  assert.ok(estimateKvBytesPerToken(1) < estimateKvBytesPerToken(8))
  assert.ok(estimateKvBytesPerToken(8) < estimateKvBytesPerToken(70))
  // Unknown size is treated as mid-range rather than free.
  assert.equal(estimateKvBytesPerToken(undefined), estimateKvBytesPerToken(8))
})

test('parses parameter size from details, falling back to the tag name', () => {
  assert.equal(parseParameterBillions('8.0B'), 8)
  assert.equal(parseParameterBillions('1.5B'), 1.5)
  assert.equal(parseParameterBillions(undefined, 'qwen2.5:0.5b'), 0.5)
  assert.equal(parseParameterBillions(undefined, 'llama3.2:3b'), 3)
  // The case the old name-regex got wrong: no size in the name at all. Better to
  // say "unknown" than to silently assume 8B.
  assert.equal(parseParameterBillions(undefined, 'phi3'), undefined)
})

test('snaps down to a ladder rung', () => {
  assert.equal(snapToLadder(9000), 8192)
  assert.equal(snapToLadder(8192), 8192)
  assert.equal(snapToLadder(100), MIN_CONTEXT)
})

test('the model trained context is a hard ceiling', () => {
  const d = resolveContextWindow({
    modelMaxCtx: 8192,
    kvBytesPerToken: 128 * KIB,
    availableBytes: 80 * GB,
  })
  assert.equal(d.contextWindow, 8192)
  assert.equal(d.limitedBy, 'model')
})

test('never exceeds trained context even to reach the floor', () => {
  // tinyllama trains to 2048. Requesting 4096 pushes RoPE past anything the
  // weights saw; a smaller-than-floor window is the correct answer here.
  const d = resolveContextWindow({
    modelMaxCtx: 2048,
    kvBytesPerToken: 22 * KIB,
    availableBytes: 8 * GB,
  })
  assert.equal(d.contextWindow, 2048)
  assert.equal(d.limitedBy, 'model')
})

test('memory limits the window when the model would allow more', () => {
  // gemma-class KV cost against a small card.
  const d = resolveContextWindow({
    modelMaxCtx: 262144,
    kvBytesPerToken: 1536 * KIB,
    availableBytes: 24 * GB,
    modelBytes: 7.6 * GB,
  })
  assert.ok(d.contextWindow < 262144)
  assert.ok(d.contextWindow >= MIN_CONTEXT)
})

test('a user cap only ever lowers the result', () => {
  const capped = resolveContextWindow({
    modelMaxCtx: 131072,
    kvBytesPerToken: 112 * KIB,
    availableBytes: 24 * GB,
    userCap: 8192,
  })
  assert.equal(capped.contextWindow, 8192)
  assert.equal(capped.limitedBy, 'user')

  // A cap above what the model or memory allows must not raise anything.
  const notRaised = resolveContextWindow({
    modelMaxCtx: 8192,
    kvBytesPerToken: 128 * KIB,
    availableBytes: 24 * GB,
    userCap: 131072,
  })
  assert.equal(notRaised.contextWindow, 8192)
})

test('weights are subtracted from the memory pool', () => {
  const inputs = { modelMaxCtx: 131072, kvBytesPerToken: 128 * KIB, availableBytes: 8 * GB }
  const withoutWeights = resolveContextWindow(inputs)
  const withWeights = resolveContextWindow({ ...inputs, modelBytes: 5 * GB })
  assert.ok(withWeights.contextWindow < withoutWeights.contextWindow)
})

test('falls back to a conservative default when nothing is known', () => {
  const d = resolveContextWindow({ kvBytesPerToken: 128 * KIB })
  assert.equal(d.contextWindow, UNKNOWN_BACKEND_CONTEXT)
  assert.equal(d.limitedBy, 'default')
})

test('the resolver is deterministic — the same inputs give the same window', () => {
  // Stability is the whole point: a window that varies between requests makes
  // Ollama unload and reload the model, stalling the turn and dropping the KV cache.
  const inputs = {
    modelMaxCtx: 131072,
    kvBytesPerToken: 128 * KIB,
    availableBytes: 8 * GB,
    modelBytes: 4.7 * GB,
  }
  const first = resolveContextWindow(inputs)
  for (let i = 0; i < 5; i++) {
    assert.deepEqual(resolveContextWindow(inputs), first)
  }
})

test('parses the user context cap setting', () => {
  assert.equal(parseUserContextCap('16384'), 16384)
  assert.equal(parseUserContextCap('auto'), undefined)
  assert.equal(parseUserContextCap(null), undefined)
  assert.equal(parseUserContextCap(''), undefined)
  assert.equal(parseUserContextCap('nonsense'), undefined)
  // Below the floor is meaningless; treat it as unset rather than crippling chat.
  assert.equal(parseUserContextCap('512'), undefined)
})