From 9a95f7b2b87ecf57b53c1c9fb7c1311edbce23d7 Mon Sep 17 00:00:00 2001
From: Jake Turner <52841588+jakeaturner@users.noreply.github.com>
Date: Mon, 17 Aug 2026 22:10:10 -0700
Subject: [PATCH] feat(AI): budget the prompt and actually set the context
window (#1253)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Chat was run through Ollama's OpenAI-compatible /v1 endpoint, which has no
context-size field, so the num_ctx we sent was silently discarded and the
entire NUM_CTX_LADDER was dead code. Every conversation ran at the backend
default and was truncated from the middle (where history and
retrieved context actually sit). Measured with identical ~7k-token prompts:
2,060 tokens processed against 7,049 on the native endpoint, and with the
needed fact in the discarded middle the old path hallucinated a plausible
wrong number.
Transport: add a native /api/chat path using the already-installed ollama
package, sending a real options object (num_ctx, num_predict, temperature,
seed) plus keep_alive. The OpenAI path is still used for LM Studio and llama.cpp,
minus the ineffective num_ctx and with include_usage so both paths return
prompt token counts. /api/show is now read for trained context, parameter
size, quantization and any modelfile num_ctx. The stream normalizer
moves to app/utils/think_stream.ts so both transports share it; it gains a
flush() that fixes a dangling partial tag being dropped at end of stream.
Token accounting: replace chars/N with a structural segmenter scaled by a
per-model factor learned by EWMA from the prompt_eval_count every response
already returned and threw away. End-to-end estimator error is 4.6% against
18-20% for a fixed divisor — qwen2.5 converges to k=1.26 where llama3 sits
at 1.00, so no single global constant serves both.
Context window: resolved once per model from trained context, the exact KV
cost derived from GGUF metadata (head_count_kv, not head_count — GQA makes
them differ 4x), and available VRAM, then memoized. Memoization is
load-bearing: asking for a different num_ctx makes Ollama unload and reload
the model, stalling the turn and discarding the KV cache. The floor never
overrides the model's trained ceiling — tinyllama trains to 2048, and a
blind 4096 floor pushes RoPE past anything the weights saw.
Prompt budget: planPrompt allocates explicitly instead of delegating
overflow to silent backend truncation — response reserve, then fixed blocks
(system and current query, never dropped), then RAG chunks whole and
best-first, then history in whole user+assistant turns, evicted newest-first
in blocks with an elision marker. Block eviction keeps the prefix stable for
several turns rather than shifting on every one.
KV-cache ordering: the per-turn retrieved block moves behind history to the
tail, so the stable prefix stays byte-identical turn to turn. At turn 14 of
a long conversation, prefill is 231ms at the tail against 539ms at the front
(~1ms/turn growth against ~25ms). The win is conditional — with a large
retrieved block and short history the two measure the same — so both
orderings ship behind RAG_PLACEMENT. The per-turn query rewrite also moves
to the tasks model; on a single-slot Ollama it was evicting the chat model's
cached prefix on every single turn.
Also: OLLAMA_CONTEXT_LENGTH and OLLAMA_KV_CACHE_TYPE=q8_0 on the container
(q8_0 is treated as headroom, never as a budgeting assumption, since
unsupported architectures fall back to f16 silently), a Context Window
setting in Settings > Models, and budget fields on PipelineTrace.
Eval: golden suites (--suite) so the new long_context goldens cannot move
the aggregates the committed baselines were recorded against, plus budget
and estimator-error metrics through the report path. Retrieval is unchanged
and eval:compare reports no metric outside the tolerance band; the
long_context suite scores correctness 1.000 with historyElided 0.500 (the
two eviction cases, by design). Unit tests 337 -> 386 passing, same 7
pre-existing failures.
Ingest-side CHAR_TO_TOKEN_RATIO is deliberately untouched: changing it
re-chunks the corpus and invalidates retrieval baselines, which belongs in
its own measured change.
---
admin/app/controllers/ollama_controller.ts | 59 ++-
admin/app/controllers/settings_controller.ts | 20 +-
admin/app/services/chat_service.ts | 40 +-
admin/app/services/context_window_service.ts | 160 ++++++
admin/app/services/docker_service.ts | 12 +
admin/app/services/eval_corpus_service.ts | 38 +-
admin/app/services/eval_generation_service.ts | 82 +++-
admin/app/services/eval_report_service.ts | 4 +
admin/app/services/ollama_service.ts | 455 +++++++++++++-----
admin/app/services/rag_pipeline_service.ts | 148 ++++--
.../app/services/token_calibration_service.ts | 123 +++++
admin/app/utils/context_budget.ts | 292 +++++++++++
admin/app/utils/context_window.ts | 282 +++++++++++
admin/app/utils/eval/report.ts | 10 +
admin/app/utils/rag_prompt.ts | 90 +---
admin/app/utils/tasks_model.ts | 64 +++
admin/app/utils/think_stream.ts | 105 ++++
admin/app/utils/token_estimate.ts | 191 ++++++++
admin/app/validators/settings.ts | 25 +
admin/commands/eval/corpus.ts | 6 +-
admin/commands/eval/generation.ts | 32 +-
admin/commands/eval/retrieval.ts | 9 +-
admin/constants/kv_store.ts | 2 +
admin/constants/ollama.ts | 71 +++
admin/inertia/pages/settings/models.tsx | 42 +-
admin/tests/eval/README.md | 66 ++-
admin/tests/eval/goldens/long_context.jsonl | 24 +
admin/tests/unit/context_budget.spec.ts | 249 ++++++++++
admin/tests/unit/context_window.spec.ts | 183 +++++++
admin/tests/unit/rag_pipeline_prompt.spec.ts | 98 +---
admin/tests/unit/rag_retrieval_toggle.spec.ts | 24 +-
admin/tests/unit/think_stream.spec.ts | 97 ++++
admin/tests/unit/token_estimate.spec.ts | 140 ++++++
admin/types/kv_store.ts | 13 +
admin/types/rag.ts | 14 +
35 files changed, 2886 insertions(+), 384 deletions(-)
create mode 100644 admin/app/services/context_window_service.ts
create mode 100644 admin/app/services/token_calibration_service.ts
create mode 100644 admin/app/utils/context_budget.ts
create mode 100644 admin/app/utils/context_window.ts
create mode 100644 admin/app/utils/tasks_model.ts
create mode 100644 admin/app/utils/think_stream.ts
create mode 100644 admin/app/utils/token_estimate.ts
create mode 100644 admin/tests/eval/goldens/long_context.jsonl
create mode 100644 admin/tests/unit/context_budget.spec.ts
create mode 100644 admin/tests/unit/context_window.spec.ts
create mode 100644 admin/tests/unit/think_stream.spec.ts
create mode 100644 admin/tests/unit/token_estimate.spec.ts
diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts
index 51019492..0930cb56 100644
--- a/admin/app/controllers/ollama_controller.ts
+++ b/admin/app/controllers/ollama_controller.ts
@@ -1,6 +1,7 @@
import { ChatService } from '#services/chat_service'
import { DockerService } from '#services/docker_service'
-import { OllamaService } from '#services/ollama_service'
+import { OllamaService, type NomadChatUsage } from '#services/ollama_service'
+import { TokenCalibrationService } from '#services/token_calibration_service'
import { RagPipelineService } from '#services/rag_pipeline_service'
import { RagService } from '#services/rag_service'
import Service from '#models/service'
@@ -11,6 +12,8 @@ import { assertNotCloudMetadataUrl } from '#validators/common'
import { inject } from '@adonisjs/core'
import type { HttpContext } from '@adonisjs/core/http'
import { SERVICE_NAMES } from '../../constants/service_names.js'
+import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js'
+import type { PipelineTrace } from '../../types/rag.js'
import logger from '@adonisjs/core/services/logger'
@inject()
@@ -20,7 +23,8 @@ export default class OllamaController {
private dockerService: DockerService,
private ollamaService: OllamaService,
private ragPipelineService: RagPipelineService,
- private ragService: RagService
+ private ragService: RagService,
+ private tokenCalibration: TokenCalibrationService
) { }
async availableModels({ request }: HttpContext) {
@@ -74,6 +78,11 @@ export default class OllamaController {
})
reqData.messages = trace.messages
const numCtx = trace.numCtx
+ const numPredict = trace.numPredict
+ // Keeping the model resident is what makes the KV cache worth building:
+ // Ollama's default evicts after 5 minutes, which is well inside the time a
+ // user spends reading an answer and typing the next question.
+ const keepAlive = (await KVStore.getValue('ai.keepAlive')) || DEFAULT_KEEP_ALIVE
// Check if the model supports "thinking" capability for enhanced response generation.
// Thinking is only enabled when the model supports it AND the user wants it: the explicit
@@ -114,6 +123,8 @@ export default class OllamaController {
think,
thinkingCapable: thinkingCapability,
numCtx,
+ numPredict,
+ keepAlive,
signal: abortController.signal,
})
let fullContent = ''
@@ -122,6 +133,9 @@ export default class OllamaController {
if (chunk.message?.content) {
fullContent += chunk.message.content
}
+ if (chunk.usage) {
+ this._recordUsage(reqData.model, trace, chunk.usage)
+ }
response.response.write(`data: ${JSON.stringify(chunk)}\n\n`)
}
} catch (err) {
@@ -147,7 +161,17 @@ export default class OllamaController {
}
// Non-streaming (legacy) path
- const result = await this.ollamaService.chat({ ...ollamaRequest, think, thinkingCapable: thinkingCapability, numCtx })
+ const result = await this.ollamaService.chat({
+ ...ollamaRequest,
+ think,
+ thinkingCapable: thinkingCapability,
+ numCtx,
+ numPredict,
+ keepAlive,
+ })
+ if (result?.usage) {
+ this._recordUsage(reqData.model, trace, result.usage)
+ }
if (sessionId && result?.message?.content) {
await this.chatService.addMessage(sessionId, 'assistant', result.message.content)
@@ -170,6 +194,35 @@ export default class OllamaController {
}
}
+ /**
+ * Close the loop on token accounting.
+ *
+ * The backend just told us exactly how many tokens the prompt really was.
+ * Comparing that against what we estimated is what lets the estimator correct
+ * itself per model — free ground truth that used to be discarded. Also logs
+ * the prefill rate, which is the practical prefix-cache-hit signal: a reused
+ * KV prefix means many prompt tokens for very little prefill time.
+ *
+ * Best-effort throughout; a calibration failure must never affect the reply.
+ */
+ private _recordUsage(model: string, trace: PipelineTrace, usage: NomadChatUsage): void {
+ if (trace.uncalibratedPromptTokens && usage.promptTokens) {
+ this.tokenCalibration
+ .record(model, trace.uncalibratedPromptTokens, usage.promptTokens)
+ .catch((err) => {
+ logger.debug(`[OllamaController] Token calibration failed: ${err?.message ?? err}`)
+ })
+
+ if (usage.promptEvalMs !== undefined && usage.promptEvalMs > 0) {
+ const tokensPerMs = usage.promptTokens / usage.promptEvalMs
+ logger.debug(
+ `[OllamaController] Prefill: ${usage.promptTokens} tokens in ${usage.promptEvalMs.toFixed(0)}ms ` +
+ `(${tokensPerMs.toFixed(1)} tok/ms; a high rate means the KV prefix was reused)`
+ )
+ }
+ }
+ }
+
async remoteStatus() {
const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
if (!remoteUrl) {
diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts
index 5b1b39fc..f3e19a17 100644
--- a/admin/app/controllers/settings_controller.ts
+++ b/admin/app/controllers/settings_controller.ts
@@ -1,5 +1,6 @@
import KVStore from '#models/kv_store'
import { BenchmarkService } from '#services/benchmark_service'
+import { ContextWindowService } from '#services/context_window_service'
import { MapService } from '#services/map_service'
import { OllamaService } from '#services/ollama_service'
import { SystemService } from '#services/system_service'
@@ -14,7 +15,8 @@ export default class SettingsController {
private systemService: SystemService,
private mapService: MapService,
private benchmarkService: BenchmarkService,
- private ollamaService: OllamaService
+ private ollamaService: OllamaService,
+ private contextWindowService: ContextWindowService
) {}
async system({ inertia }: HttpContext) {
@@ -73,6 +75,20 @@ export default class SettingsController {
const autoThinking = await KVStore.getValue('ai.autoThinking')
const tasksModel = await KVStore.getValue('ai.tasksModel')
const ragEnabled = await KVStore.getValue('rag.enabled')
+ const contextWindow = await KVStore.getValue('ai.contextWindow')
+ // Resolved window per installed model, so the setting shows what "Auto"
+ // actually produced rather than leaving the user to guess. Best-effort:
+ // a model whose metadata can't be read simply doesn't get a badge.
+ const resolvedContextWindows: Record = {}
+ await Promise.all(
+ (installedModels || []).map(async (model) => {
+ try {
+ resolvedContextWindows[model.name] = await this.contextWindowService.windowFor(model.name)
+ } catch {
+ /* leave unset */
+ }
+ })
+ )
return inertia.render('settings/models', {
models: {
availableModels: availableModels?.models || [],
@@ -85,7 +101,9 @@ export default class SettingsController {
autoThinking: autoThinking ?? false,
tasksModel: tasksModel ?? '',
ragEnabled: ragEnabled ?? true,
+ contextWindow: contextWindow ?? 'auto',
},
+ resolvedContextWindows,
},
})
}
diff --git a/admin/app/services/chat_service.ts b/admin/app/services/chat_service.ts
index 7ac41eaf..2be952d1 100644
--- a/admin/app/services/chat_service.ts
+++ b/admin/app/services/chat_service.ts
@@ -6,7 +6,8 @@ import { DateTime } from 'luxon'
import { inject } from '@adonisjs/core'
import { OllamaService } from './ollama_service.js'
import { SYSTEM_PROMPTS } from '../../constants/ollama.js'
-import { pickTasksModel, toTitleCase } from '../utils/misc.js'
+import { toTitleCase } from '../utils/misc.js'
+import { resolveTasksModel } from '../utils/tasks_model.js'
@inject()
export class ChatService {
@@ -26,42 +27,7 @@ export class ChatService {
fallback: string | null,
installed?: { name: string }[]
): Promise {
- let configured: string | null = null
- try {
- configured = await KVStore.getValue('ai.tasksModel')
- } catch (error) {
- logger.error(
- `[ChatService] Failed to read ai.tasksModel: ${error instanceof Error ? error.message : error}`
- )
- return fallback
- }
- if (!configured?.trim()) {
- return fallback
- }
-
- let models = installed
- if (!models) {
- try {
- models = await this.ollamaService.getModels()
- } catch (error) {
- logger.error(
- `[ChatService] Failed to list models while resolving the tasks model: ${error instanceof Error ? error.message : error}`
- )
- return fallback
- }
- }
-
- const { model, staleConfigured } = pickTasksModel(
- configured,
- (models ?? []).map((m) => m.name),
- fallback
- )
- if (staleConfigured) {
- logger.warn(
- `[ChatService] Configured tasks model "${staleConfigured}" is not installed; falling back to "${fallback ?? 'none'}"`
- )
- }
- return model
+ return resolveTasksModel(this.ollamaService, fallback, installed, '[ChatService]')
}
async getAllSessions() {
diff --git a/admin/app/services/context_window_service.ts b/admin/app/services/context_window_service.ts
new file mode 100644
index 00000000..0d02dbd1
--- /dev/null
+++ b/admin/app/services/context_window_service.ts
@@ -0,0 +1,160 @@
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import os from 'node:os'
+import KVStore from '#models/kv_store'
+import { OllamaService } from '#services/ollama_service'
+import { SystemService } from '#services/system_service'
+import {
+ UNKNOWN_BACKEND_CONTEXT,
+ computeKvBytesPerToken,
+ estimateKvBytesPerToken,
+ parseParameterBillions,
+ parseUserContextCap,
+ resolveContextWindow,
+ type ContextWindowDecision,
+} from '../utils/context_window.js'
+
+/**
+ * Decides how large a context window each chat model gets, and then holds that
+ * answer still.
+ *
+ * The "holds it still" part is the whole point. Ollama unloads and reloads a
+ * model whenever a request asks for a different `num_ctx` than the loaded
+ * instance — so a window that grew with the conversation would stall a turn and
+ * throw away the KV cache exactly when the conversation got interesting. The
+ * decision is therefore made once per model and memoized for the process
+ * lifetime; it is a property of (model, hardware), not of the current prompt.
+ *
+ * Inputs, in decreasing order of authority:
+ * - the user's cap (`ai.contextWindow`), which can only lower the result
+ * - the model's trained context length, from /api/show `model_info`
+ * - what the hardware can actually afford, from the exact per-token KV cost in
+ * the GGUF metadata against detected VRAM (or a share of system RAM)
+ */
+@inject()
+export class ContextWindowService {
+ private decisions: Map> = new Map()
+ private availableBytes: Promise | null = null
+
+ constructor(
+ private ollamaService: OllamaService,
+ private systemService: SystemService
+ ) {}
+
+ /**
+ * The context window to request for `model`. Memoized — repeat calls within a
+ * process return the identical number, which is what keeps the model loaded.
+ */
+ async windowFor(model: string): Promise {
+ return (await this.decisionFor(model)).contextWindow
+ }
+
+ async decisionFor(model: string): Promise {
+ const cached = this.decisions.get(model)
+ if (cached) return cached
+
+ const pending = this._resolve(model).catch((error) => {
+ logger.warn(
+ `[ContextWindow] Falling back to ${UNKNOWN_BACKEND_CONTEXT} for ${model}: ${error instanceof Error ? error.message : error}`
+ )
+ return { contextWindow: UNKNOWN_BACKEND_CONTEXT, limitedBy: 'default' as const }
+ })
+ this.decisions.set(model, pending)
+ return pending
+ }
+
+ private async _resolve(model: string): Promise {
+ const [info, userCapRaw] = await Promise.all([
+ this.ollamaService.getModelInfo(model),
+ KVStore.getValue('ai.contextWindow'),
+ ])
+ const userCap = parseUserContextCap(userCapRaw)
+
+ // No /api/show means a non-Ollama backend, which sets its context at server
+ // start and ignores anything we send. Budget against a conservative value so
+ // the prompt planner still bounds itself rather than trusting the backend to
+ // truncate gracefully — it won't; it will truncate silently.
+ if (!info.contextLength && !info.rawModelInfo) {
+ const decision: ContextWindowDecision = {
+ contextWindow: userCap ?? UNKNOWN_BACKEND_CONTEXT,
+ limitedBy: userCap ? 'user' : 'default',
+ }
+ logger.info(
+ `[ContextWindow] ${model}: no model metadata available, budgeting at ${decision.contextWindow}`
+ )
+ return decision
+ }
+
+ const kvBytesPerToken =
+ computeKvBytesPerToken(info.rawModelInfo) ??
+ estimateKvBytesPerToken(parseParameterBillions(info.parameterSize, model))
+
+ const decision = resolveContextWindow({
+ modelMaxCtx: info.contextLength,
+ modelfileNumCtx: info.modelfileNumCtx,
+ kvBytesPerToken,
+ availableBytes: await this._availableBytes(),
+ modelBytes: await this._modelBytes(model),
+ userCap,
+ })
+
+ logger.info(
+ `[ContextWindow] ${model}: ${decision.contextWindow} tokens (limited by ${decision.limitedBy}; ` +
+ `${(kvBytesPerToken / 1024).toFixed(0)} KiB/token KV` +
+ `${decision.affordableTokens !== undefined ? `, ~${decision.affordableTokens} affordable` : ''})`
+ )
+ return decision
+ }
+
+ /**
+ * Memory available for inference, in bytes.
+ *
+ * Prefers the VRAM figure Ollama itself reports at startup — it is the number
+ * that actually governs whether a window fits, and it already accounts for
+ * whichever device Ollama chose. Falls back to a conservative share of system
+ * RAM for CPU-only hosts, which is the low end of the hardware NOMAD targets
+ * and precisely where getting this wrong hurts most.
+ */
+ private async _availableBytes(): Promise {
+ if (!this.availableBytes) {
+ this.availableBytes = (async () => {
+ try {
+ const compute = await this.systemService.getOllamaInferenceComputeFromLogs()
+ if (compute && compute.vramMiB > 0) {
+ return compute.vramMiB * 1024 * 1024
+ }
+ } catch (error) {
+ logger.debug(
+ `[ContextWindow] GPU probe failed, assuming CPU inference: ${error instanceof Error ? error.message : error}`
+ )
+ }
+ // CPU inference shares host RAM with MySQL, Redis, Qdrant and every other
+ // container on the box, so claim only a modest slice of it.
+ return os.totalmem() * CPU_RAM_SHARE
+ })()
+ }
+ return this.availableBytes
+ }
+
+ private async _modelBytes(model: string): Promise {
+ try {
+ const models = await this.ollamaService.getModels(true)
+ return models.find((m) => m.name === model)?.size || undefined
+ } catch {
+ return undefined
+ }
+ }
+
+ /** Drop memoized decisions — call when the backend or the user's cap changes. */
+ invalidate(): void {
+ this.decisions.clear()
+ this.availableBytes = null
+ }
+}
+
+/**
+ * Share of system RAM assumed available for CPU inference. The rest of the NOMAD
+ * stack (MySQL, Redis, Qdrant, Kiwix, the admin app itself) lives in the same
+ * pool, and a machine that starts swapping is worse than one with a small window.
+ */
+const CPU_RAM_SHARE = 0.5
diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts
index bba85327..8849b872 100644
--- a/admin/app/services/docker_service.ts
+++ b/admin/app/services/docker_service.ts
@@ -26,6 +26,7 @@ import { randomBytes } from 'node:crypto'
import KVStore from '#models/kv_store'
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
+import { DEFAULT_OLLAMA_CONTEXT_LENGTH } from '../../constants/ollama.js'
@inject()
export class DockerService {
@@ -810,7 +811,18 @@ export class DockerService {
const flashAttentionEnabled = await KVStore.getValue('ai.ollamaFlashAttention')
if (flashAttentionEnabled !== false) {
ollamaEnv.push('OLLAMA_FLASH_ATTENTION=1')
+ // KV cache quantization requires flash attention. q8_0 roughly halves the
+ // memory a context window costs for a perplexity delta in the noise, which is
+ // what makes an 8-16k window affordable on modest hardware. Ollama silently
+ // falls back to f16 on architectures that don't support it, so this is treated
+ // as headroom — the context-window resolver still budgets against f16.
+ ollamaEnv.push('OLLAMA_KV_CACHE_TYPE=q8_0')
}
+ // Floor for any request that doesn't carry its own num_ctx (the OpenAI-compatible
+ // path can't set one at all). Ollama's own default is 4096 on machines under 24GB
+ // VRAM, and anything past the window is silently truncated with no error — so a
+ // conversation that outgrows it just quietly loses its early turns.
+ ollamaEnv.push(`OLLAMA_CONTEXT_LENGTH=${DEFAULT_OLLAMA_CONTEXT_LENGTH}`)
if (amdGpuConfigured) {
// gfx-aware HSA override — only set for cards that actually need it. See
// _resolveAmdHsaOverride() for the resolution order and gfx → version mapping.
diff --git a/admin/app/services/eval_corpus_service.ts b/admin/app/services/eval_corpus_service.ts
index 9776f864..cf2b5c5d 100644
--- a/admin/app/services/eval_corpus_service.ts
+++ b/admin/app/services/eval_corpus_service.ts
@@ -17,6 +17,13 @@ import { EVAL_CORPUS_DIR } from '../utils/eval/corpus_source.js'
/** Where the golden question set lives, relative to the app root. */
export const EVAL_GOLDENS_DIR = 'tests/eval/goldens'
+/**
+ * The suite that runs when none is named — i.e. `core.jsonl`. Every other
+ * `.jsonl` in the goldens directory is opt-in via `--suite`, so adding one
+ * cannot move the numbers an existing baseline was recorded against.
+ */
+export const DEFAULT_GOLDEN_SUITE = 'core'
+
// EVAL_CORPUS_DIR and docIdFromSource live in app/utils/eval/corpus_source.ts:
// they are pure path logic, and keeping them out of this service is what lets
// the leak guard be unit-tested without booting AdonisJS.
@@ -72,11 +79,34 @@ export class EvalCorpusService {
)
}
- /** Load and validate every golden file, cross-checked against the corpus. */
- async loadGoldens(): Promise {
+ /**
+ * Load and validate golden files, cross-checked against the corpus.
+ *
+ * Each `.jsonl` in the goldens directory is a *suite*, named after the file.
+ * Only `core` loads by default. That matters for comparability: aggregate
+ * metrics are means over whatever ran, so silently adding goldens to the
+ * default set would move recall@k and correctness for reasons unrelated to any
+ * change under test, and quietly invalidate every committed baseline. Opting
+ * in keeps a suite's numbers separate until someone asks for them.
+ */
+ async loadGoldens(suites: string[] = [DEFAULT_GOLDEN_SUITE]): Promise {
const dir = resolve(join(process.cwd(), EVAL_GOLDENS_DIR))
- const files = (await readdir(dir)).filter((f) => f.endsWith('.jsonl')).sort()
- if (files.length === 0) throw new Error(`No golden files found in ${dir}`)
+ const available = (await readdir(dir)).filter((f) => f.endsWith('.jsonl')).sort()
+ if (available.length === 0) throw new Error(`No golden files found in ${dir}`)
+
+ // '*' loads every suite. Used by `eval:corpus`, which validates the fixtures
+ // rather than measuring anything — a malformed golden should fail validation
+ // whether or not its suite is part of the default run.
+ const wanted = new Set(suites)
+ const files = wanted.has('*')
+ ? available
+ : available.filter((f) => wanted.has(basename(f, '.jsonl')))
+ if (files.length === 0) {
+ throw new Error(
+ `No golden suites matched [${suites.join(', ')}]. Available: ` +
+ available.map((f) => basename(f, '.jsonl')).join(', ')
+ )
+ }
const goldens: Golden[] = []
const seen = new Set()
diff --git a/admin/app/services/eval_generation_service.ts b/admin/app/services/eval_generation_service.ts
index 2aba609e..560657a3 100644
--- a/admin/app/services/eval_generation_service.ts
+++ b/admin/app/services/eval_generation_service.ts
@@ -1,5 +1,5 @@
import { EvalCorpusService } from '#services/eval_corpus_service'
-import { OllamaService } from '#services/ollama_service'
+import { OllamaService, type NomadChatUsage } from '#services/ollama_service'
import { RagPipelineService } from '#services/rag_pipeline_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
@@ -75,6 +75,27 @@ export type GenerationCaseResult = {
groundedness: NumericSummary | null
retrievedDocIds: string[]
injectedChunks: number
+ /**
+ * What the budget planner did with the window on this case.
+ *
+ * Without this a low score is ambiguous: it could mean the model answered
+ * badly, or it could mean the model never saw the material because the
+ * relevant turns were evicted. These fields tell the two apart.
+ */
+ budget?: {
+ contextWindow: number
+ estimatedPromptTokens: number
+ promptBudget: number
+ turnsDropped: number
+ chunksDropped: number
+ historyElided: boolean
+ queryTruncated: boolean
+ }
+ /**
+ * Estimated vs. the backend's real prompt token count. The estimator is the
+ * foundation every budget decision rests on, so its error is worth gating on.
+ */
+ promptTokenError?: number
/** Non-null only when something went wrong talking to the model. */
error?: string
}
@@ -91,6 +112,13 @@ export type GenerationAggregate = {
markdownRate: number | null
groundedness: NumericSummary | null
meanAnswerLength: number | null
+ /** Share of cases where history had to be trimmed to fit the window. */
+ historyElidedRate: number | null
+ /** Share of cases where a retrieved chunk didn't fit the context budget. */
+ chunksDroppedRate: number | null
+ /** Mean absolute error of the token estimator against real prompt counts. */
+ promptTokenError: NumericSummary | null
+ meanPromptTokens: number | null
}
export type GenerationRunResult = {
@@ -228,6 +256,8 @@ export class EvalGenerationService {
const scores: GenerationScores[] = []
let retrievedDocIds: string[] = []
let injectedChunks = 0
+ let budget: GenerationCaseResult['budget']
+ let promptTokenError: number | undefined
let error: string | undefined
for (let attempt = 0; attempt < repeats; attempt++) {
@@ -236,10 +266,31 @@ export class EvalGenerationService {
retrievedDocIds = uniqueDocIds(trace.retrieved)
injectedChunks = trace.injected.length
const context = trace.injected.map((c) => c.text).join('\n\n')
+ if (trace.budget) {
+ budget = {
+ contextWindow: trace.budget.contextWindow,
+ estimatedPromptTokens: trace.budget.estimatedPromptTokens,
+ promptBudget: trace.budget.promptBudget,
+ turnsDropped: trace.budget.turnsDropped,
+ chunksDropped: trace.budget.chunksDropped,
+ historyElided: trace.budget.historyElided,
+ queryTruncated: trace.budget.queryTruncated,
+ }
+ }
- const answer = isMock
- ? mockAnswer(context)
- : await this.generate(options.model, trace.messages, trace.numCtx)
+ const generated = isMock
+ ? { answer: mockAnswer(context), usage: undefined }
+ : await this.generate(options.model, trace.messages, trace.numCtx, trace.numPredict)
+ const answer = generated.answer
+
+ // The backend just reported the prompt's real token count. Comparing it
+ // to the estimate is the only way to know whether the budget above was
+ // built on a number worth trusting.
+ if (generated.usage?.promptTokens && trace.uncalibratedPromptTokens) {
+ promptTokenError =
+ Math.abs(generated.usage.promptTokens - trace.uncalibratedPromptTokens) /
+ generated.usage.promptTokens
+ }
answers.push(answer)
scores.push(
@@ -269,19 +320,27 @@ export class EvalGenerationService {
groundedness: summarizeNumeric(scores.map((s) => s.numericGroundedness)),
retrievedDocIds,
injectedChunks,
+ budget,
+ promptTokenError,
error,
}
}
- private async generate(model: string, messages: OllamaChatMessage[], numCtx?: number): Promise {
+ private async generate(
+ model: string,
+ messages: OllamaChatMessage[],
+ numCtx?: number,
+ numPredict?: number
+ ): Promise<{ answer: string; usage?: NomadChatUsage }> {
const response = await this.ollamaService.chat({
model,
messages,
numCtx,
+ numPredict,
temperature: EVAL_TEMPERATURE,
seed: EVAL_SEED,
})
- return response.message.content.trim()
+ return { answer: response.message.content.trim(), usage: response.usage }
}
}
@@ -333,6 +392,17 @@ export function aggregateGeneration(cases: GenerationCaseResult[]): GenerationAg
markdownRate: rate(allScores.map((s) => s.markdownFormatted)),
groundedness: summarizeNumeric(allScores.map((s) => s.numericGroundedness)),
meanAnswerLength: allScores.length === 0 ? null : mean(allScores.map((s) => s.length)),
+ historyElidedRate: rate(cases.filter((c) => c.budget).map((c) => c.budget!.historyElided)),
+ chunksDroppedRate: rate(cases.filter((c) => c.budget).map((c) => c.budget!.chunksDropped > 0)),
+ promptTokenError: summarizeNumeric(
+ cases.map((c) => c.promptTokenError).filter((e): e is number => e !== undefined)
+ ),
+ meanPromptTokens: (() => {
+ const values = cases
+ .map((c) => c.budget?.estimatedPromptTokens)
+ .filter((v): v is number => v !== undefined)
+ return values.length === 0 ? null : mean(values)
+ })(),
}
}
diff --git a/admin/app/services/eval_report_service.ts b/admin/app/services/eval_report_service.ts
index 337a9610..ceec0645 100644
--- a/admin/app/services/eval_report_service.ts
+++ b/admin/app/services/eval_report_service.ts
@@ -70,6 +70,10 @@ export class EvalReportService {
thinkTagLeakRate: agg.thinkTagLeakRate,
markdownRate: agg.markdownRate,
groundedness: agg.groundedness?.mean ?? null,
+ historyElidedRate: agg.historyElidedRate,
+ chunksDroppedRate: agg.chunksDroppedRate,
+ promptTokenError: agg.promptTokenError?.mean ?? null,
+ meanPromptTokens: agg.meanPromptTokens,
})
return {
meta,
diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts
index 27841a35..b7859ad3 100644
--- a/admin/app/services/ollama_service.ts
+++ b/admin/app/services/ollama_service.ts
@@ -2,6 +2,9 @@ import { inject } from '@adonisjs/core'
import OpenAI from 'openai'
import type { ChatCompletionChunk, ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
import type { Stream } from 'openai/streaming.js'
+import { Ollama } from 'ollama'
+import { ThinkTagSplitter } from '../utils/think_stream.js'
+import { readContextLength, readModelfileNumCtx } from '../utils/context_window.js'
import { NomadOllamaModel } from '../../types/ollama.js'
import { EMBEDDING_MODEL_NAME, FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js'
import fs from 'node:fs/promises'
@@ -21,6 +24,34 @@ const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models'
const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000 // 24 hours
+/** Ollama reports durations in nanoseconds. */
+function nsToMs(ns: number | undefined): number | undefined {
+ return typeof ns === 'number' ? ns / 1e6 : undefined
+}
+
+/**
+ * What `/api/show` tells us about a model. All fields are optional because
+ * non-Ollama backends don't expose the endpoint at all, and because Ollama's
+ * `model_info` keys are architecture-prefixed and not guaranteed present.
+ */
+export type NomadModelInfo = {
+ hasThinking: boolean
+ /** Trained context length, e.g. `llama.context_length`. The hard ceiling for num_ctx. */
+ contextLength?: number
+ /** e.g. "8.0B" — the honest source for model size, vs. guessing from the tag name. */
+ parameterSize?: string
+ quantizationLevel?: string
+ /** num_ctx baked into the modelfile, if the author set one. */
+ modelfileNumCtx?: number
+ /**
+ * The raw `model_info` blob. Carries the GGUF architecture keys
+ * (block_count, attention.head_count_kv, embedding_length) that make an exact
+ * KV-cache cost calculation possible. Kept whole rather than picked apart here
+ * because the key names are architecture-prefixed and vary by family.
+ */
+ rawModelInfo?: Record
+}
+
export type NomadInstalledModel = {
name: string
size: number
@@ -28,15 +59,35 @@ export type NomadInstalledModel = {
details?: Record
}
+/**
+ * Token accounting reported back by the backend.
+ *
+ * `promptTokens` is the ground truth the token estimator calibrates against —
+ * it is the one number in the system that is not a guess. `promptEvalMs` is the
+ * prefill wall time; the ratio of the two is the prefix-cache-hit proxy (a
+ * reused KV prefix means many prompt tokens for almost no prefill time).
+ *
+ * Only the native transport reports prefill timing; the OpenAI-compatible shim
+ * gives token counts but no durations, so `promptEvalMs` is undefined there.
+ */
+export type NomadChatUsage = {
+ promptTokens?: number
+ completionTokens?: number
+ promptEvalMs?: number
+}
+
export type NomadChatResponse = {
message: { content: string; thinking?: string }
done: boolean
model: string
+ usage?: NomadChatUsage
}
export type NomadChatStreamChunk = {
message: { content: string; thinking?: string }
done: boolean
+ // Present only on the final chunk of a stream.
+ usage?: NomadChatUsage
}
type ChatInput = {
@@ -47,7 +98,14 @@ type ChatInput = {
// disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing).
thinkingCapable?: boolean
stream?: boolean
+ // Context window for this request. Only reaches the model on the native transport —
+ // Ollama's OpenAI-compatible endpoint has no context-size field and silently drops it.
numCtx?: number
+ // Cap on generated tokens, so a long answer can't run past the end of the window.
+ numPredict?: number
+ // How long to keep the model (and its KV cache) resident after this request.
+ // Unset inherits Ollama's 5-minute default.
+ keepAlive?: string | number
// Sampling controls. Left unset for normal chat, which inherits the backend's
// defaults — the historical behaviour. The eval harness sets temperature 0 and
// a fixed seed so repeated runs are as close to comparable as llama.cpp allows
@@ -62,13 +120,16 @@ type ChatInput = {
@inject()
export class OllamaService {
private openai: OpenAI | null = null
+ private ollama: Ollama | null = null
private baseUrl: string | null = null
private initPromise: Promise | null = null
private isOllamaNative: boolean | null = null
+ private nativeProbe: Promise | null = null
private activeDownloads: Map> = new Map()
- // Memoized `thinking` capability per model name (see checkModelHasThinking). Only successful
- // /api/show lookups are cached; transient failures are left uncached so they can be retried.
- private thinkingCapabilityCache: Map = new Map()
+ // Memoized /api/show result per model name (see getModelInfo). A model's capabilities and
+ // trained context length don't change at runtime. Only successful lookups are cached;
+ // transient failures are left uncached so they can be retried.
+ private modelInfoCache: Map = new Map()
constructor() {}
@@ -93,11 +154,51 @@ export class OllamaService {
apiKey: 'nomad', // Required by SDK; not validated by Ollama/LM Studio/llama.cpp
baseURL: `${this.baseUrl}/v1`,
})
+ // Native client for `/api/chat`. Only used when the backend is really Ollama
+ // (see _isNativeBackend) — it is the only transport that can set the context window.
+ this.ollama = new Ollama({ host: this.baseUrl })
})()
}
return this.initPromise
}
+ /**
+ * Whether the configured backend is Ollama itself rather than some other
+ * OpenAI-compatible server (LM Studio, llama.cpp, vLLM, ...).
+ *
+ * This matters for one reason above all: `num_ctx` cannot be set over the
+ * OpenAI-compatible endpoint. Ollama's compatibility layer has no
+ * context-size field, so a request that needs a specific window has to go
+ * through native `/api/chat`. Everything else degrades gracefully; the
+ * context window does not.
+ *
+ * `getModels` also sets `isOllamaNative` as a side effect, so a probe is only
+ * fired when nothing has established it yet. The probe result is memoized for
+ * the process lifetime — the backend URL cannot change without a restart.
+ */
+ private async _isNativeBackend(): Promise {
+ if (this.isOllamaNative !== null) return this.isOllamaNative
+ if (!this.nativeProbe) {
+ this.nativeProbe = (async () => {
+ try {
+ const response = await axios.get(`${this.baseUrl}/api/tags`, { timeout: 5000 })
+ // LM Studio answers 200 on unknown paths with an incompatible body — validate the shape.
+ this.isOllamaNative = Array.isArray(response.data?.models)
+ } catch {
+ this.isOllamaNative = false
+ }
+ if (!this.isOllamaNative) {
+ logger.info(
+ '[OllamaService] Backend is not Ollama-native; num_ctx cannot be set per request. ' +
+ 'Context budgeting will assume the backend default.'
+ )
+ }
+ return this.isOllamaNative
+ })()
+ }
+ return this.nativeProbe
+ }
+
private async _ensureDependencies() {
if (!this.openai) {
await this._initialize()
@@ -330,16 +431,39 @@ export class OllamaService {
}
}
- public async chat(chatRequest: ChatInput): Promise {
- await this._ensureDependencies()
- if (!this.openai) {
- throw new Error('AI client is not initialized.')
- }
+ /**
+ * Build the `options` bag for the native `/api/chat` endpoint.
+ *
+ * Note `num_ctx` must be *stable per model* across a session. Ollama unloads
+ * and reloads a model whenever a request asks for a different context size
+ * than the loaded instance, which stalls the turn and throws away the KV
+ * cache. Callers get their value from ContextWindowResolver, which memoizes
+ * per model precisely so this stays constant.
+ */
+ private _nativeOptions(chatRequest: ChatInput): Record {
+ const options: Record = {}
+ if (chatRequest.numCtx) options.num_ctx = chatRequest.numCtx
+ if (chatRequest.numPredict) options.num_predict = chatRequest.numPredict
+ if (chatRequest.temperature !== undefined) options.temperature = chatRequest.temperature
+ if (chatRequest.seed !== undefined) options.seed = chatRequest.seed
+ return options
+ }
+ /**
+ * Params for the OpenAI-compatible endpoint.
+ *
+ * Deliberately does NOT send `num_ctx`: Ollama's compatibility layer has no
+ * context-size field and silently discards it (the PR to add one, ollama#6137,
+ * was closed unmerged). It used to be sent here as a top-level field, which
+ * meant the entire context-window ladder was a no-op and every conversation
+ * silently ran at the backend default. `max_tokens` is the one generation
+ * bound this endpoint does honour.
+ */
+ private _compatParams(chatRequest: ChatInput, stream: boolean): any {
const params: any = {
model: chatRequest.model,
messages: chatRequest.messages as ChatCompletionMessageParam[],
- stream: false,
+ stream,
}
if (chatRequest.think) {
params.think = chatRequest.think
@@ -353,8 +477,8 @@ export class OllamaService {
} else if (chatRequest.thinkingCapable && chatRequest.think === false) {
params.reasoning_effort = 'none'
}
- if (chatRequest.numCtx) {
- params.num_ctx = chatRequest.numCtx
+ if (chatRequest.numPredict) {
+ params.max_tokens = chatRequest.numPredict
}
if (chatRequest.temperature !== undefined) {
params.temperature = chatRequest.temperature
@@ -362,7 +486,57 @@ export class OllamaService {
if (chatRequest.seed !== undefined) {
params.seed = chatRequest.seed
}
+ if (stream) {
+ // Without this the streaming path reports no token usage at all, and the
+ // estimator has nothing to calibrate against on non-native backends.
+ params.stream_options = { include_usage: true }
+ }
+ return params
+ }
+ public async chat(chatRequest: ChatInput): Promise {
+ await this._ensureDependencies()
+ if (await this._isNativeBackend()) {
+ return this._chatNative(chatRequest)
+ }
+ return this._chatCompat(chatRequest)
+ }
+
+ private async _chatNative(chatRequest: ChatInput): Promise {
+ if (!this.ollama) {
+ throw new Error('AI client is not initialized.')
+ }
+
+ const response = await this.ollama.chat({
+ model: chatRequest.model,
+ messages: chatRequest.messages,
+ stream: false,
+ ...(chatRequest.think ? { think: chatRequest.think } : {}),
+ ...(chatRequest.keepAlive !== undefined ? { keep_alive: chatRequest.keepAlive } : {}),
+ options: this._nativeOptions(chatRequest),
+ })
+
+ return {
+ message: {
+ content: response.message?.content ?? '',
+ thinking: response.message?.thinking ?? undefined,
+ },
+ done: true,
+ model: response.model,
+ usage: {
+ promptTokens: response.prompt_eval_count,
+ completionTokens: response.eval_count,
+ promptEvalMs: nsToMs(response.prompt_eval_duration),
+ },
+ }
+ }
+
+ private async _chatCompat(chatRequest: ChatInput): Promise {
+ if (!this.openai) {
+ throw new Error('AI client is not initialized.')
+ }
+
+ const params = this._compatParams(chatRequest, false)
const response = await this.openai.chat.completions.create(params, { signal: chatRequest.signal })
const choice = response.choices[0]
@@ -375,121 +549,158 @@ export class OllamaService {
},
done: true,
model: response.model,
+ usage: {
+ promptTokens: response.usage?.prompt_tokens,
+ completionTokens: response.usage?.completion_tokens,
+ },
}
}
public async chatStream(chatRequest: ChatInput): Promise> {
await this._ensureDependencies()
- if (!this.openai) {
+ if (await this._isNativeBackend()) {
+ return this._chatStreamNative(chatRequest)
+ }
+ return this._chatStreamCompat(chatRequest)
+ }
+
+ private async _chatStreamNative(
+ chatRequest: ChatInput
+ ): Promise> {
+ if (!this.ollama) {
throw new Error('AI client is not initialized.')
}
- const params: any = {
+ const iterator = await this.ollama.chat({
model: chatRequest.model,
- messages: chatRequest.messages as ChatCompletionMessageParam[],
+ messages: chatRequest.messages,
stream: true,
- }
- if (chatRequest.think) {
- params.think = chatRequest.think
- }
- // The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever.
- // Only touch it for thinking-capable models so non-Ollama backends never get an unexpected
- // param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to
- // suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op).
- if (chatRequest.think === 'medium') {
- params.reasoning_effort = 'medium'
- } else if (chatRequest.thinkingCapable && chatRequest.think === false) {
- params.reasoning_effort = 'none'
- }
- if (chatRequest.numCtx) {
- params.num_ctx = chatRequest.numCtx
- }
- if (chatRequest.temperature !== undefined) {
- params.temperature = chatRequest.temperature
- }
- if (chatRequest.seed !== undefined) {
- params.seed = chatRequest.seed
- }
+ ...(chatRequest.think ? { think: chatRequest.think } : {}),
+ ...(chatRequest.keepAlive !== undefined ? { keep_alive: chatRequest.keepAlive } : {}),
+ options: this._nativeOptions(chatRequest),
+ })
- const stream = (await this.openai.chat.completions.create(params, {
- signal: chatRequest.signal,
- })) as unknown as Stream
-
- // Returns how many trailing chars of `text` could be the start of `tag`
- function partialTagSuffix(tag: string, text: string): number {
- for (let len = Math.min(tag.length - 1, text.length); len >= 1; len--) {
- if (text.endsWith(tag.slice(0, len))) return len
- }
- return 0
+ // The native client aborts through the iterator rather than a fetch signal, so bridge
+ // the caller's AbortSignal onto it. Without this a disconnected client would leave the
+ // generation decoding server-side and block Ollama's single parallel slot (#1065).
+ const onAbort = () => iterator.abort()
+ if (chatRequest.signal) {
+ if (chatRequest.signal.aborted) iterator.abort()
+ else chatRequest.signal.addEventListener('abort', onAbort, { once: true })
}
async function* normalize(): AsyncGenerator {
- // Stateful parser for ... tags that may be split across chunks.
- // Ollama provides thinking natively via delta.thinking; OpenAI-compatible backends
- // (LM Studio, llama.cpp, etc.) embed them inline in delta.content.
- let tagBuffer = ''
- let inThink = false
-
- for await (const chunk of stream) {
- const delta = chunk.choices[0]?.delta
- // /v1 emits thinking as `reasoning`; native Ollama uses `thinking`. Read both (#1065).
- const nativeThinking: string = (delta as any)?.thinking ?? (delta as any)?.reasoning ?? ''
- const rawContent: string = delta?.content ?? ''
-
- // Parse tags out of the content stream
- tagBuffer += rawContent
- let parsedContent = ''
- let parsedThinking = ''
-
- while (tagBuffer.length > 0) {
- if (inThink) {
- const closeIdx = tagBuffer.indexOf('')
- if (closeIdx !== -1) {
- parsedThinking += tagBuffer.slice(0, closeIdx)
- tagBuffer = tagBuffer.slice(closeIdx + 8)
- inThink = false
- } else {
- const hold = partialTagSuffix('', tagBuffer)
- parsedThinking += tagBuffer.slice(0, tagBuffer.length - hold)
- tagBuffer = tagBuffer.slice(tagBuffer.length - hold)
- break
- }
- } else {
- const openIdx = tagBuffer.indexOf('')
- if (openIdx !== -1) {
- parsedContent += tagBuffer.slice(0, openIdx)
- tagBuffer = tagBuffer.slice(openIdx + 7)
- inThink = true
- } else {
- const hold = partialTagSuffix('', tagBuffer)
- parsedContent += tagBuffer.slice(0, tagBuffer.length - hold)
- tagBuffer = tagBuffer.slice(tagBuffer.length - hold)
- break
- }
+ // Ollama reports reasoning on its own field, so no tag parsing is needed here —
+ // but a model can still emit literal tags inside content, and the splitter
+ // is a no-op on text that has none.
+ const splitter = new ThinkTagSplitter()
+ try {
+ for await (const chunk of iterator) {
+ const split = splitter.push(chunk.message?.content ?? '')
+ const nativeThinking = chunk.message?.thinking ?? ''
+ yield {
+ message: {
+ content: split.content,
+ thinking: nativeThinking + split.thinking,
+ },
+ done: chunk.done === true,
+ ...(chunk.done
+ ? {
+ usage: {
+ promptTokens: chunk.prompt_eval_count,
+ completionTokens: chunk.eval_count,
+ promptEvalMs: nsToMs(chunk.prompt_eval_duration),
+ },
+ }
+ : {}),
}
}
-
- yield {
- message: {
- content: parsedContent,
- thinking: nativeThinking + parsedThinking,
- },
- done: chunk.choices[0]?.finish_reason !== null && chunk.choices[0]?.finish_reason !== undefined,
+ const tail = splitter.flush()
+ if (tail.content || tail.thinking) {
+ yield { message: { content: tail.content, thinking: tail.thinking }, done: true }
}
+ } finally {
+ chatRequest.signal?.removeEventListener('abort', onAbort)
}
}
return normalize()
}
- public async checkModelHasThinking(modelName: string): Promise {
- await this._ensureDependencies()
- if (!this.baseUrl) return false
+ private async _chatStreamCompat(
+ chatRequest: ChatInput
+ ): Promise> {
+ if (!this.openai) {
+ throw new Error('AI client is not initialized.')
+ }
- // A model's capabilities don't change at runtime, so memoize the /api/show result. Without
- // this, loading the chat picker fires one /api/show per installed model and every chat send
- // fires another — this collapses those to a single call per model per process.
- const cached = this.thinkingCapabilityCache.get(modelName)
+ const params = this._compatParams(chatRequest, true)
+ const stream = (await this.openai.chat.completions.create(params, {
+ signal: chatRequest.signal,
+ })) as unknown as Stream
+
+ async function* normalize(): AsyncGenerator {
+ // Stateful parser for ... tags that may be split across chunks.
+ // Ollama provides thinking natively via delta.thinking; OpenAI-compatible backends
+ // (LM Studio, llama.cpp, etc.) embed them inline in delta.content.
+ const splitter = new ThinkTagSplitter()
+
+ for await (const chunk of stream) {
+ // With stream_options.include_usage the final chunk carries usage and no choices.
+ if (chunk.usage && (chunk.choices?.length ?? 0) === 0) {
+ yield {
+ message: { content: '', thinking: '' },
+ done: true,
+ usage: {
+ promptTokens: chunk.usage.prompt_tokens,
+ completionTokens: chunk.usage.completion_tokens,
+ },
+ }
+ continue
+ }
+
+ const delta = chunk.choices[0]?.delta
+ // /v1 emits thinking as `reasoning`; native Ollama uses `thinking`. Read both (#1065).
+ const nativeThinking: string = (delta as any)?.thinking ?? (delta as any)?.reasoning ?? ''
+ const split = splitter.push(delta?.content ?? '')
+
+ yield {
+ message: {
+ content: split.content,
+ thinking: nativeThinking + split.thinking,
+ },
+ done: chunk.choices[0]?.finish_reason !== null && chunk.choices[0]?.finish_reason !== undefined,
+ }
+ }
+
+ // A partial tag held at end of stream was never a tag; emitting it prevents a
+ // silently truncated answer.
+ const tail = splitter.flush()
+ if (tail.content || tail.thinking) {
+ yield { message: { content: tail.content, thinking: tail.thinking }, done: true }
+ }
+ }
+
+ return normalize()
+ }
+
+ /**
+ * Everything `/api/show` knows about a model, memoized per model name.
+ *
+ * A model's capabilities and trained context length don't change at runtime, so
+ * this collapses to a single call per model per process. Without memoization,
+ * loading the chat picker fires one /api/show per installed model and every chat
+ * send fires another.
+ *
+ * Returns a zero-information record (not a throw) when the backend has no
+ * /api/show — every caller has to cope with a non-Ollama backend anyway, and a
+ * missing context length is a legitimate answer meaning "assume the default".
+ */
+ public async getModelInfo(modelName: string): Promise {
+ await this._ensureDependencies()
+ if (!this.baseUrl) return { hasThinking: false }
+
+ const cached = this.modelInfoCache.get(modelName)
if (cached !== undefined) return cached
try {
@@ -498,16 +709,28 @@ export class OllamaService {
{ model: modelName },
{ timeout: 5000 }
)
- const hasThinking =
- Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking')
- this.thinkingCapabilityCache.set(modelName, hasThinking)
- return hasThinking
+ const data = response.data ?? {}
+ const info: NomadModelInfo = {
+ hasThinking: Array.isArray(data.capabilities) && data.capabilities.includes('thinking'),
+ contextLength: readContextLength(data.model_info),
+ parameterSize: data.details?.parameter_size,
+ quantizationLevel: data.details?.quantization_level,
+ modelfileNumCtx: readModelfileNumCtx(data.parameters),
+ rawModelInfo: data.model_info,
+ }
+ this.modelInfoCache.set(modelName, info)
+ return info
} catch {
- // Non-Ollama backends don't expose /api/show — assume no thinking support
- return false
+ // Non-Ollama backends don't expose /api/show. Left uncached so a transient
+ // failure can be retried rather than poisoning the process.
+ return { hasThinking: false }
}
}
+ public async checkModelHasThinking(modelName: string): Promise {
+ return (await this.getModelInfo(modelName)).hasThinking
+ }
+
public async deleteModel(modelName: string): Promise<{ success: boolean; message: string }> {
await this._ensureDependencies()
if (!this.baseUrl) {
@@ -738,8 +961,20 @@ export class OllamaService {
return []
}
+ // The tasks model is exempt for the same reason the embedding model is: it is
+ // infrastructure, not "a chat model the user picked". It runs the per-turn
+ // query rewrite, so evicting it here would mean a cold load on the very next
+ // message — and the whole point of routing the rewrite off the chat model was
+ // to stop the two from fighting over Ollama's single slot.
+ let tasksModel: string | null = null
+ try {
+ tasksModel = (await KVStore.getValue('ai.tasksModel'))?.trim() || null
+ } catch {
+ // Setting unreadable; fall through and treat it as unset.
+ }
+
const toUnload = loadedModels.filter(
- (name) => name !== EMBEDDING_MODEL_NAME && name !== targetModel
+ (name) => name !== EMBEDDING_MODEL_NAME && name !== targetModel && name !== tasksModel
)
await Promise.all(
@@ -780,12 +1015,14 @@ export class OllamaService {
throw new Error('Not an Ollama-compatible /api/tags response')
}
this.isOllamaNative = true
+ this.nativeProbe = Promise.resolve(true)
const models: NomadInstalledModel[] = response.data.models
if (includeEmbeddings) return models
return models.filter((m) => !m.name.includes('embed'))
} catch {
// Fall back to the OpenAI-compatible /v1/models endpoint (LM Studio, llama.cpp, etc.)
this.isOllamaNative = false
+ this.nativeProbe = Promise.resolve(false)
logger.info('[OllamaService] /api/tags unavailable, falling back to /v1/models')
try {
const modelList = await this.openai!.models.list()
diff --git a/admin/app/services/rag_pipeline_service.ts b/admin/app/services/rag_pipeline_service.ts
index 228288e7..46ab52c6 100644
--- a/admin/app/services/rag_pipeline_service.ts
+++ b/admin/app/services/rag_pipeline_service.ts
@@ -1,22 +1,24 @@
+import { ContextWindowService } from '#services/context_window_service'
import { NomadMdService } from '#services/nomad_md_service'
import { OllamaService } from '#services/ollama_service'
import { RagService } from '#services/rag_service'
+import { TokenCalibrationService } from '#services/token_calibration_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import {
RAG_CONTEXT_LIMITS,
RAG_DEFAULT_SCORE_THRESHOLD,
RAG_DEFAULT_TOP_K,
+ RAG_PLACEMENT,
+ QUERY_REWRITE_MAX_TOKENS,
SYSTEM_PROMPTS,
} from '../../constants/ollama.js'
import type { OllamaChatMessage } from '../../types/ollama.js'
import type { PipelineOptions, PipelineTrace, RetrievedChunk } from '../../types/rag.js'
-import {
- buildContextBlock,
- deriveNumCtx,
- getContextLimitsForModel,
- trimToContextBudget,
-} from '../utils/rag_prompt.js'
+import { planPrompt } from '../utils/context_budget.js'
+import { estimateMessagesTokens } from '../utils/token_estimate.js'
+import { resolveTasksModel } from '../utils/tasks_model.js'
+import { buildContextBlock, getContextLimitsForModel } from '../utils/rag_prompt.js'
/**
* Everything that happens between "a user sent a message" and "a payload goes
@@ -37,7 +39,9 @@ export class RagPipelineService {
constructor(
private ollamaService: OllamaService,
private ragService: RagService,
- private nomadMdService: NomadMdService
+ private nomadMdService: NomadMdService,
+ private contextWindowService: ContextWindowService,
+ private tokenCalibration: TokenCalibrationService
) {}
/**
@@ -51,13 +55,18 @@ export class RagPipelineService {
model: string,
opts: PipelineOptions = {}
): Promise {
- const working: OllamaChatMessage[] = [...messages]
+ // Split the caller's conversation into the pieces the budget planner needs.
+ // Caller-supplied system messages are treated as stable prefix material and
+ // stay in front of everything, as before.
+ const callerSystem = messages.filter((msg) => msg.role === 'system')
+ const conversation = messages.filter((msg) => msg.role !== 'system')
+
+ const systemBlocks: OllamaChatMessage[] = [...callerSystem]
// Default formatting prompt, only when the caller supplied no system message.
- const hasSystemMessage = working.some((msg) => msg.role === 'system')
- if (!hasSystemMessage) {
+ if (callerSystem.length === 0) {
logger.debug('[RagPipeline] Injecting system prompt')
- working.unshift({ role: 'system', content: SYSTEM_PROMPTS.default })
+ systemBlocks.push({ role: 'system', content: SYSTEM_PROMPTS.default })
}
// The user-managed NOMAD.md goes in front of the formatting prompt so the
@@ -67,10 +76,23 @@ export class RagPipelineService {
const nomadPrompt = await this.nomadMdService.getSystemPrompt()
if (nomadPrompt) {
logger.debug('[RagPipeline] Injecting NOMAD.md system prompt')
- working.unshift({ role: 'system', content: nomadPrompt })
+ systemBlocks.unshift({ role: 'system', content: nomadPrompt })
}
}
+ // The current question is the last user message; everything before it is
+ // history. A conversation with no user message at all still has to produce a
+ // valid payload, so fall back to an empty question.
+ const lastUserIndex = conversation.map((m) => m.role).lastIndexOf('user')
+ const query: OllamaChatMessage =
+ lastUserIndex >= 0 ? conversation[lastUserIndex] : { role: 'user', content: '' }
+ const history = lastUserIndex >= 0 ? conversation.slice(0, lastUserIndex) : conversation
+
+ // Retrieval reads the conversation as the user wrote it — system prompts are
+ // deliberately excluded. They used to be inside the rewrite window, where
+ // they were transcribed as "Assistant" turns and polluted the rewritten query.
+ const working: OllamaChatMessage[] = [...systemBlocks, ...conversation]
+
const trace: PipelineTrace = {
rewrittenQuery: null,
didRewrite: false,
@@ -78,6 +100,7 @@ export class RagPipelineService {
injected: [],
messages: working,
numCtx: undefined,
+ numPredict: undefined,
contextLimits: { maxResults: RAG_DEFAULT_TOP_K, maxTokens: 0 },
timings: { rewriteMs: 0, retrievalMs: 0 },
}
@@ -96,15 +119,19 @@ export class RagPipelineService {
logger.debug('[RagPipeline] Retrieval disabled by setting, skipping')
} else {
const rewriteStart = Date.now()
- const { query, didRewrite } = await this.resolveRetrievalQuery(working, model, opts)
+ const { query: retrievalQuery, didRewrite } = await this.resolveRetrievalQuery(
+ conversation,
+ model,
+ opts
+ )
trace.timings.rewriteMs = Date.now() - rewriteStart
- trace.rewrittenQuery = query
+ trace.rewrittenQuery = retrievalQuery
trace.didRewrite = didRewrite
- if (query) {
+ if (retrievalQuery) {
const retrievalStart = Date.now()
relevantDocs = await this.ragService.searchSimilarDocuments(
- query,
+ retrievalQuery,
opts.topK ?? RAG_DEFAULT_TOP_K,
opts.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD,
opts.collection
@@ -112,36 +139,52 @@ export class RagPipelineService {
trace.timings.retrievalMs = Date.now() - retrievalStart
trace.retrieved = relevantDocs
logger.debug(
- `[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${query}"`
+ `[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${retrievalQuery}"`
)
}
}
- // --- Context trimming + injection -------------------------------------
- if (relevantDocs.length > 0) {
- const limits = getContextLimitsForModel(model, RAG_CONTEXT_LIMITS)
- trace.contextLimits = limits
- const trimmedDocs = trimToContextBudget(relevantDocs, limits)
- trace.injected = trimmedDocs
+ // --- Budgeted assembly -------------------------------------------------
+ //
+ // Everything above decided *what* is available; this decides what fits. The
+ // model-size tier still caps how many chunks a small model is asked to read
+ // (a 1B model does not get better answers from five chunks), and the budget
+ // planner then enforces the hard token limit on top of that.
+ const modelInfo = await this.ollamaService.getModelInfo(model).catch(() => undefined)
+ const limits = getContextLimitsForModel(model, RAG_CONTEXT_LIMITS, modelInfo?.parameterSize)
+ trace.contextLimits = limits
+ const candidateDocs = relevantDocs.slice(0, limits.maxResults)
- logger.debug(
- `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${model}, maxResults: ${limits.maxResults}, maxTokens: ${limits.maxTokens || 'unlimited'})`
+ const contextWindow = await this.contextWindowService.windowFor(model)
+ const ratio = await this.tokenCalibration.ratioFor(model)
+
+ const plan = planPrompt({
+ systemBlocks,
+ history,
+ query,
+ ragChunks: candidateDocs,
+ renderRagBlock: (chunks) =>
+ SYSTEM_PROMPTS.rag_context(buildContextBlock(chunks as RetrievedChunk[])),
+ contextWindow,
+ ratio,
+ ragPlacement: opts.ragPlacement ?? RAG_PLACEMENT,
+ })
+
+ trace.messages = plan.messages as OllamaChatMessage[]
+ trace.injected = candidateDocs.slice(0, plan.trace.chunksKept)
+ trace.budget = plan.trace
+ trace.numCtx = contextWindow
+ trace.numPredict = plan.numPredict
+ // Recorded uncalibrated so TokenCalibrationService can compare like with like;
+ // applying the ratio here and again there would make the EWMA chase itself.
+ trace.uncalibratedPromptTokens = estimateMessagesTokens(plan.messages, 1)
+
+ if (plan.trace.turnsDropped > 0 || plan.trace.chunksDropped > 0 || plan.trace.queryTruncated) {
+ logger.info(
+ `[RagPipeline] Budget for ${model}: ${plan.trace.estimatedPromptTokens}/${plan.trace.promptBudget} tokens ` +
+ `(window ${contextWindow}); dropped ${plan.trace.turnsDropped} turn(s), ` +
+ `${plan.trace.chunksDropped} chunk(s)${plan.trace.queryTruncated ? ', truncated the question' : ''}`
)
-
- const systemMessage: OllamaChatMessage = {
- role: 'system',
- content: SYSTEM_PROMPTS.rag_context(buildContextBlock(trimmedDocs)),
- }
-
- // After any existing system messages, before the first non-system message.
- const firstNonSystemIndex = working.findIndex((msg) => msg.role !== 'system')
- const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex
- working.splice(insertIndex, 0, systemMessage)
- }
-
- trace.numCtx = deriveNumCtx(working)
- if (trace.numCtx) {
- logger.debug(`[RagPipeline] Large system prompt, requesting num_ctx: ${trace.numCtx}`)
}
return trace
@@ -173,10 +216,11 @@ export class RagPipelineService {
// Last 6 messages ≈ 3 turns.
//
- // PRESERVED QUIRK: this slice is taken *after* system messages have been
- // unshifted, so on short conversations the system prompts land inside the
- // window and get labelled "Assistant" in the transcript below. Faithful
- // to the original; a candidate fix once the harness can measure it.
+ // The caller now passes the conversation *without* system messages, so the
+ // window can no longer scoop up system prompts and transcribe them as
+ // "Assistant" turns — which is what the previous slice did on short
+ // conversations, feeding the rewriter the formatting rules as if the model
+ // had said them.
const recentMessages = messages.slice(-6)
// Skip rewriting on the very first turn — with only one user message there
@@ -202,8 +246,17 @@ export class RagPipelineService {
})
.join('\n')
+ // Route to the tasks model when one is configured. This is the ancillary
+ // call that actually costs something: it runs every turn, before the
+ // answer, and it sends a prompt that shares no prefix with the chat. On a
+ // single-slot Ollama (OLLAMA_NUM_PARALLEL=1, the default) running it on the
+ // chat model evicts that model's cached prefix, so the conversation is
+ // re-prefilled from scratch on every turn no matter how stable we keep the
+ // ordering. A small dedicated model keeps the chat model's cache intact.
+ const rewriteModel = (await resolveTasksModel(this.ollamaService, model, undefined, '[RAG]')) ?? model
+
const response = await this.ollamaService.chat({
- model,
+ model: rewriteModel,
messages: [
{ role: 'system', content: SYSTEM_PROMPTS.query_rewrite },
{
@@ -211,6 +264,11 @@ export class RagPipelineService {
content: `Conversation:\n${conversationContext}\n\nRewritten Query:`,
},
],
+ // A rewrite is a short, mechanical transformation. Cap it so a chatty
+ // small model can't spend a thousand tokens restating the question, and
+ // pin the sampler so the same conversation rewrites the same way.
+ numPredict: QUERY_REWRITE_MAX_TOKENS,
+ temperature: 0,
})
const rewrittenQuery = response.message.content.trim()
diff --git a/admin/app/services/token_calibration_service.ts b/admin/app/services/token_calibration_service.ts
new file mode 100644
index 00000000..7bccbe6b
--- /dev/null
+++ b/admin/app/services/token_calibration_service.ts
@@ -0,0 +1,123 @@
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import KVStore from '#models/kv_store'
+import { clampRatio, updateEwma } from '../utils/token_estimate.js'
+
+/**
+ * Learns, per model, how far the token estimator is off — and corrects it.
+ *
+ * The estimator in `token_estimate.ts` cannot tokenize exactly (no vocabulary,
+ * no tokenize endpoint, offline appliance). But every chat response already
+ * carries the exact answer: Ollama reports `prompt_eval_count`, the true token
+ * count of the prompt we just sent. That is free ground truth on every single
+ * turn, and before this it was thrown away.
+ *
+ * So: record what we estimated, compare it to what the backend actually counted,
+ * and fold `actual / estimated` into a per-model EWMA. The budget planner
+ * multiplies by that factor from then on. Measured convergence is a couple of
+ * turns, and it matters — Qwen tokenizes ~26% finer than Llama on the same text,
+ * which no single global constant can absorb.
+ *
+ * Persisted as one JSON blob (`ai.tokenRatios`) rather than a key per model,
+ * because KVStoreKey is a closed union. The data is pure cache: deleting the row
+ * costs nothing but a short re-learn.
+ */
+@inject()
+export class TokenCalibrationService {
+ /** Model name -> learned correction factor. Authoritative once loaded. */
+ private ratios: Map = new Map()
+ private loaded: Promise | null = null
+ /** Last persisted value per model, so we only write when the ratio really moved. */
+ private persisted: Map = new Map()
+
+ /**
+ * A ratio has to move by more than this before it earns a database write.
+ * Once converged the factor barely twitches, and a row update per chat turn
+ * would be pure noise on a machine that may be running from an SD card.
+ */
+ private static PERSIST_THRESHOLD = 0.02
+
+ private async _load(): Promise {
+ if (!this.loaded) {
+ this.loaded = (async () => {
+ try {
+ const raw = await KVStore.getValue('ai.tokenRatios')
+ if (!raw) return
+ const parsed = JSON.parse(raw)
+ if (parsed && typeof parsed === 'object') {
+ for (const [model, value] of Object.entries(parsed)) {
+ if (typeof value === 'number' && Number.isFinite(value)) {
+ const clamped = clampRatio(value)
+ this.ratios.set(model, clamped)
+ this.persisted.set(model, clamped)
+ }
+ }
+ }
+ } catch (error) {
+ // Corrupt or hand-edited JSON. This is a cache; start over rather than
+ // failing a chat request over it.
+ logger.warn(
+ `[TokenCalibration] Could not read stored ratios, starting fresh: ${error instanceof Error ? error.message : error}`
+ )
+ }
+ })()
+ }
+ return this.loaded
+ }
+
+ /**
+ * The correction factor to apply when estimating for `model`.
+ * Returns 1 for a model we've never seen — an uncalibrated estimate, which is
+ * still materially better than a fixed chars-per-token divisor.
+ */
+ async ratioFor(model: string): Promise {
+ await this._load()
+ return this.ratios.get(model) ?? 1
+ }
+
+ /**
+ * Record one observation.
+ *
+ * `estimatedTokens` must be the *uncalibrated* estimate for exactly the
+ * messages that were sent — otherwise the factor gets applied twice and the
+ * EWMA chases its own tail.
+ */
+ async record(model: string, estimatedTokens: number, actualTokens: number | undefined): Promise {
+ if (!actualTokens || !estimatedTokens || estimatedTokens <= 0 || actualTokens <= 0) return
+ // Tiny prompts are dominated by chat-template overhead and teach us little
+ // about how this model tokenizes real content.
+ if (actualTokens < 32) return
+
+ await this._load()
+
+ const observed = clampRatio(actualTokens / estimatedTokens)
+ const previous = this.ratios.get(model) ?? null
+ const next = clampRatio(updateEwma(previous, observed))
+ this.ratios.set(model, next)
+
+ logger.debug(
+ `[TokenCalibration] ${model}: estimated ${estimatedTokens}, actual ${actualTokens} ` +
+ `(observed ${observed.toFixed(3)}, ratio ${next.toFixed(3)})`
+ )
+
+ const lastWritten = this.persisted.get(model)
+ if (lastWritten === undefined || Math.abs(next - lastWritten) > TokenCalibrationService.PERSIST_THRESHOLD) {
+ await this._persist(model, next)
+ }
+ }
+
+ private async _persist(model: string, ratio: number): Promise {
+ try {
+ const blob: Record = {}
+ for (const [name, value] of this.ratios) blob[name] = Number(value.toFixed(4))
+ await KVStore.setValue('ai.tokenRatios', JSON.stringify(blob))
+ this.persisted.set(model, ratio)
+ } catch (error) {
+ // Calibration is an optimization. Losing a write means a slightly worse
+ // estimate next boot, never a failed chat.
+ logger.warn(
+ `[TokenCalibration] Failed to persist ratios: ${error instanceof Error ? error.message : error}`
+ )
+ }
+ }
+}
diff --git a/admin/app/utils/context_budget.ts b/admin/app/utils/context_budget.ts
new file mode 100644
index 00000000..1df2023e
--- /dev/null
+++ b/admin/app/utils/context_budget.ts
@@ -0,0 +1,292 @@
+/**
+ * Decides what actually goes into the prompt, and what gets left out.
+ *
+ * Before this existed, nothing trimmed anything: the browser posted the whole
+ * session, the pipeline copied it verbatim, and overflow was left to the
+ * backend. That is the worst possible arrangement, because backend truncation is
+ * *silent* and *positional* — llama.cpp drops from the middle of the window, so
+ * what gets discarded is conversation history and retrieved knowledge-base
+ * context, while the model gives no indication anything is missing. Measured on
+ * a live Ollama, a 7,000-token prompt came back having processed 2,060 tokens,
+ * and the model confabulated an answer from the fragment that survived.
+ *
+ * So the rule here is: never hand the backend more than it can hold. Decide
+ * explicitly what to drop, drop it in a way the model can reason about, and
+ * record what happened so the eval harness can see it.
+ *
+ * Allocation order, most protected first:
+ * 1. Response reserve — tokens held back for the answer itself.
+ * 2. Fixed blocks — system prompts and the current question. Never dropped.
+ * 3. Retrieved context — whole chunks, best-first, until its share is spent.
+ * 4. History — whole turns, newest-first, into whatever remains.
+ *
+ * Pure and dependency-free so it runs under bare `node --test`, the same
+ * convention as `rag_prompt.ts`.
+ */
+
+import { estimateMessagesTokens, estimateTokens } from './token_estimate.js'
+
+export type BudgetRole = 'system' | 'user' | 'assistant'
+export type BudgetMessage = { role: BudgetRole; content: string }
+
+/** A retrieved chunk, already ranked — index 0 is the best match. */
+export type BudgetChunk = { text: string; metadata?: Record }
+
+/**
+ * Where the per-turn retrieved-context block goes.
+ *
+ * `tail` places it immediately before the current question, leaving
+ * [system][history] as a byte-identical prefix across turns so the backend can
+ * reuse its KV cache. `front` is the historical behaviour: a system message
+ * ahead of all history, which changes content every turn and therefore
+ * invalidates the cached prefix for the entire conversation behind it.
+ *
+ * The common advice to "inject RAG into the system prompt" assumes the context
+ * is fixed for the session (an attached document), where a fixed position does
+ * preserve the prefix. When context is re-retrieved every turn, the opposite is
+ * true. Both are kept so `eval:generation` can measure the difference rather
+ * than the choice resting on argument alone.
+ */
+export type RagPlacement = 'tail' | 'front'
+
+export type BudgetInputs = {
+ /** Stable system blocks (NOMAD.md, formatting rules), in final order. */
+ systemBlocks: BudgetMessage[]
+ /** Prior conversation, oldest first, excluding the current question. */
+ history: BudgetMessage[]
+ /** The current user message. Never dropped. */
+ query: BudgetMessage
+ /** Retrieved chunks, best first. May be empty. */
+ ragChunks: BudgetChunk[]
+ /** Renders surviving chunks into the final context message. */
+ renderRagBlock: (chunks: BudgetChunk[]) => string
+ /** The resolved context window, in tokens. */
+ contextWindow: number
+ /** Learned per-model token-estimator correction. */
+ ratio?: number
+ ragPlacement?: RagPlacement
+ /** Overrides for the response reserve and RAG share; mostly for tests. */
+ responseReserve?: number
+ ragShare?: number
+}
+
+export type BudgetTrace = {
+ contextWindow: number
+ responseReserve: number
+ promptBudget: number
+ estimatedPromptTokens: number
+ systemTokens: number
+ queryTokens: number
+ ragTokens: number
+ historyTokens: number
+ chunksKept: number
+ chunksDropped: number
+ turnsKept: number
+ turnsDropped: number
+ historyElided: boolean
+ queryTruncated: boolean
+ ragPlacement: RagPlacement
+}
+
+export type BudgetResult = {
+ messages: BudgetMessage[]
+ trace: BudgetTrace
+ /** What to send as num_predict, so generation can't run past the window. */
+ numPredict: number
+}
+
+/** Ceiling on tokens reserved for the answer. */
+export const MAX_RESPONSE_RESERVE = 1024
+/** Share of the window reserved for the answer when that is smaller. */
+export const RESPONSE_RESERVE_FRACTION = 0.25
+/** Share of the remaining budget the retrieved context may claim. */
+export const DEFAULT_RAG_SHARE = 0.35
+
+/**
+ * Marker inserted where turns were dropped.
+ *
+ * Worth the handful of tokens: without it the model sees a conversation that
+ * appears to begin mid-thought and has no way to tell that it is missing
+ * context, which is exactly when small models start confidently inventing what
+ * "we discussed earlier".
+ */
+export const ELISION_MARKER =
+ '[Earlier messages in this conversation have been omitted to fit the context window.]'
+
+/**
+ * Turns are dropped in blocks rather than one at a time.
+ *
+ * Dropping a single turn per request would shift the prompt prefix on *every*
+ * subsequent turn, invalidating the backend's KV cache each time — trading a
+ * small saving in tokens for a full re-prefill. Dropping several at once means
+ * the prefix then stays stable for several turns.
+ */
+export const HISTORY_EVICTION_BLOCK = 4
+
+/**
+ * Group a flat message list into whole turns.
+ *
+ * A turn is a user message plus everything that answers it. Truncating on a
+ * message boundary instead can leave the transcript starting with an assistant
+ * reply to a question the model can no longer see — which reads, to the model,
+ * as though it said something unprompted.
+ */
+export function groupIntoTurns(history: BudgetMessage[]): BudgetMessage[][] {
+ const turns: BudgetMessage[][] = []
+ for (const message of history) {
+ if (message.role === 'user' || turns.length === 0) {
+ turns.push([message])
+ } else {
+ turns[turns.length - 1].push(message)
+ }
+ }
+ return turns
+}
+
+/**
+ * Plan the prompt.
+ *
+ * Never returns a message array estimated to exceed `contextWindow -
+ * responseReserve`, except in the degenerate case where the fixed blocks alone
+ * are too large — and in that case the query is truncated visibly rather than
+ * being handed to the backend to cut silently.
+ */
+export function planPrompt(inputs: BudgetInputs): BudgetResult {
+ const ratio = inputs.ratio ?? 1
+ const placement = inputs.ragPlacement ?? 'tail'
+ const contextWindow = inputs.contextWindow
+
+ const responseReserve =
+ inputs.responseReserve ??
+ Math.max(256, Math.min(MAX_RESPONSE_RESERVE, Math.floor(contextWindow * RESPONSE_RESERVE_FRACTION)))
+ const promptBudget = Math.max(0, contextWindow - responseReserve)
+
+ const cost = (messages: BudgetMessage[]) => estimateMessagesTokens(messages, ratio)
+
+ // --- Fixed blocks: system prompts and the question --------------------
+ const systemTokens = cost(inputs.systemBlocks)
+ let query = inputs.query
+ let queryTokens = cost([query])
+ let queryTruncated = false
+
+ // Degenerate case: the system prompts plus the question alone overflow the
+ // window. Truncating here is a bad outcome, but it is a *visible* one — the
+ // alternative is the backend silently cutting the question in half.
+ if (systemTokens + queryTokens > promptBudget) {
+ const available = Math.max(0, promptBudget - systemTokens)
+ if (available > 0) {
+ const perToken = query.content.length / Math.max(1, estimateTokens(query.content, ratio))
+ const keepChars = Math.max(0, Math.floor(available * perToken) - ELISION_MARKER.length)
+ query = {
+ ...query,
+ content:
+ query.content.slice(0, keepChars) +
+ '\n\n[This message was truncated because it exceeds the context window.]',
+ }
+ queryTokens = cost([query])
+ queryTruncated = true
+ }
+ }
+
+ let remaining = promptBudget - systemTokens - queryTokens
+
+ // --- Retrieved context: whole chunks, best first ----------------------
+ //
+ // Whole chunks, deliberately. The previous behaviour capped on a running
+ // character count, which could hand the model a chunk cut off mid-sentence —
+ // the worst of both worlds, since it costs tokens without carrying a complete
+ // fact. A chunk that doesn't fit is dropped, not shortened.
+ const ragBudget = Math.max(0, Math.floor(Math.max(0, remaining) * (inputs.ragShare ?? DEFAULT_RAG_SHARE)))
+ const keptChunks: BudgetChunk[] = []
+ let ragTokens = 0
+
+ if (inputs.ragChunks.length > 0 && ragBudget > 0) {
+ for (const chunk of inputs.ragChunks) {
+ const candidate = [...keptChunks, chunk]
+ const rendered = inputs.renderRagBlock(candidate)
+ const candidateTokens = cost([{ role: 'system', content: rendered }])
+ if (candidateTokens <= ragBudget) {
+ keptChunks.push(chunk)
+ ragTokens = candidateTokens
+ }
+ // Keep scanning: a later, smaller chunk may still fit where this one didn't.
+ }
+ }
+
+ const ragMessage: BudgetMessage | null =
+ keptChunks.length > 0
+ ? { role: 'system', content: inputs.renderRagBlock(keptChunks) }
+ : null
+
+ remaining -= ragTokens
+
+ // --- History: whole turns, newest first -------------------------------
+ const turns = groupIntoTurns(inputs.history)
+ const keptTurns: BudgetMessage[][] = []
+ let historyTokens = 0
+
+ for (let i = turns.length - 1; i >= 0; i--) {
+ const turnTokens = cost(turns[i])
+ if (historyTokens + turnTokens > remaining) break
+ keptTurns.unshift(turns[i])
+ historyTokens += turnTokens
+ }
+
+ const turnsDropped = turns.length - keptTurns.length
+
+ // Chunky eviction: once we're dropping anything, drop a whole block, so the
+ // surviving prefix stays put for the next several turns instead of shifting
+ // on every request and forcing a re-prefill each time.
+ if (turnsDropped > 0 && keptTurns.length > 0) {
+ const overshoot = turnsDropped % HISTORY_EVICTION_BLOCK
+ const extra = overshoot === 0 ? 0 : HISTORY_EVICTION_BLOCK - overshoot
+ for (let i = 0; i < extra && keptTurns.length > 1; i++) {
+ const removed = keptTurns.shift()!
+ historyTokens -= cost(removed)
+ }
+ }
+
+ const historyElided = keptTurns.length < turns.length
+ const historyMessages = keptTurns.flat()
+
+ // --- Assemble, stability-descending -----------------------------------
+ const messages: BudgetMessage[] = []
+ messages.push(...inputs.systemBlocks)
+ if (historyElided) {
+ messages.push({ role: 'system', content: ELISION_MARKER })
+ }
+
+ if (placement === 'front' && ragMessage) {
+ // Historical ordering, kept only so the two can be compared under eval.
+ messages.splice(inputs.systemBlocks.length, 0, ragMessage)
+ messages.push(...historyMessages)
+ } else {
+ messages.push(...historyMessages)
+ if (ragMessage) messages.push(ragMessage)
+ }
+ messages.push(query)
+
+ const estimatedPromptTokens = cost(messages)
+
+ return {
+ messages,
+ numPredict: responseReserve,
+ trace: {
+ contextWindow,
+ responseReserve,
+ promptBudget,
+ estimatedPromptTokens,
+ systemTokens,
+ queryTokens,
+ ragTokens,
+ historyTokens,
+ chunksKept: keptChunks.length,
+ chunksDropped: inputs.ragChunks.length - keptChunks.length,
+ turnsKept: keptTurns.length,
+ turnsDropped: turns.length - keptTurns.length,
+ historyElided,
+ queryTruncated,
+ ragPlacement: placement,
+ },
+ }
+}
diff --git a/admin/app/utils/context_window.ts b/admin/app/utils/context_window.ts
new file mode 100644
index 00000000..668a4aa9
--- /dev/null
+++ b/admin/app/utils/context_window.ts
@@ -0,0 +1,282 @@
+/**
+ * Pure helpers for deciding how large a model's context window should be.
+ *
+ * Kept out of OllamaService so they can run under bare `node --test` with no
+ * Ollama, Docker, or MySQL — the same convention as `rag_prompt.ts`. The service
+ * supplies the facts (what /api/show said, how much memory is free); everything
+ * here is a function of its arguments.
+ */
+
+/**
+ * Ladder of context sizes we're willing to request.
+ *
+ * Snapping to a ladder is not cosmetic. Ollama unloads and reloads a model
+ * whenever a request asks for a different `num_ctx` than the loaded instance, so
+ * a continuously-varying window would stall a turn and throw away the KV cache
+ * every time the conversation grew. Snapping (plus per-model memoization in the
+ * resolver) keeps the value constant for a session.
+ */
+export const CONTEXT_LADDER = [4096, 8192, 16384, 32768, 65536, 131072] as const
+
+/** Never go below this: it is roughly a system prompt plus a few real turns. */
+export const MIN_CONTEXT = 4096
+
+/**
+ * Fallback window when the backend can't tell us anything — a non-Ollama
+ * OpenAI-compatible server, or /api/show unavailable. Deliberately modest:
+ * budgeting too small wastes capacity, budgeting too large silently truncates.
+ */
+export const UNKNOWN_BACKEND_CONTEXT = 8192
+
+/**
+ * Read the trained context length out of an /api/show `model_info` blob.
+ *
+ * Keys are architecture-prefixed (`llama.context_length`, `qwen3.context_length`,
+ * `gemma3.context_length`, ...), so prefer the prefix named by
+ * `general.architecture` and fall back to any `*.context_length` key. Ollama's
+ * client types this as a Map but the wire format is a plain object; handle both.
+ */
+export function readContextLength(modelInfo: unknown): number | undefined {
+ if (!modelInfo) return undefined
+ const entries: [string, any][] =
+ modelInfo instanceof Map
+ ? [...modelInfo.entries()]
+ : typeof modelInfo === 'object'
+ ? Object.entries(modelInfo as Record)
+ : []
+ if (entries.length === 0) return undefined
+
+ const lookup = new Map(entries)
+ const arch = lookup.get('general.architecture')
+ if (typeof arch === 'string') {
+ const exact = lookup.get(`${arch}.context_length`)
+ if (typeof exact === 'number' && exact > 0) return exact
+ }
+
+ for (const [key, value] of entries) {
+ if (key.endsWith('.context_length') && typeof value === 'number' && value > 0) {
+ return value
+ }
+ }
+ return undefined
+}
+
+/**
+ * Pull `num_ctx` out of the `parameters` string /api/show returns (the modelfile
+ * PARAMETER lines, e.g. `num_ctx 8192`). When a model
+ * author baked in a window, that is a deliberate signal worth respecting as a
+ * ceiling hint.
+ */
+export function readModelfileNumCtx(parameters: unknown): number | undefined {
+ if (typeof parameters !== 'string') return undefined
+ const match = parameters.match(/^\s*num_ctx\s+(\d+)\s*$/m)
+ if (!match) return undefined
+ const value = Number.parseInt(match[1], 10)
+ return Number.isFinite(value) && value > 0 ? value : undefined
+}
+
+/**
+ * Parse Ollama's `details.parameter_size` ("8.0B", "1.5B", "70B") into billions.
+ * Falls back to the tag-name regex — which is what the codebase used to rely on
+ * exclusively, and which guesses wrong on names like "phi3" that carry no size.
+ */
+export function parseParameterBillions(
+ parameterSize: string | undefined,
+ modelName?: string
+): number | undefined {
+ const fromDetails = parameterSize?.match(/([\d.]+)\s*B/i)
+ if (fromDetails) {
+ const value = Number.parseFloat(fromDetails[1])
+ if (Number.isFinite(value)) return value
+ }
+ const fromName = modelName?.match(/(\d+\.?\d*)\s*[bB](?![a-zA-Z])/)
+ if (fromName) {
+ const value = Number.parseFloat(fromName[1])
+ if (Number.isFinite(value)) return value
+ }
+ return undefined
+}
+
+/** Round a token count down to the largest ladder rung that fits. */
+export function snapToLadder(tokens: number): number {
+ let chosen = MIN_CONTEXT
+ for (const rung of CONTEXT_LADDER) {
+ if (rung <= tokens) chosen = rung
+ }
+ return chosen
+}
+
+// ---------------------------------------------------------------------------
+// How much memory a context window actually costs
+// ---------------------------------------------------------------------------
+
+/**
+ * Bytes per element in the KV cache.
+ *
+ * Budgeted at f16 (2 bytes) even though the Ollama container is started with
+ * `OLLAMA_KV_CACHE_TYPE=q8_0`. Ollama silently falls back to f16 on
+ * architectures that don't support cache quantization, so treating q8_0 as
+ * guaranteed would double the real cost of every window we hand out. Quantized
+ * cache is headroom, not a budgeting assumption.
+ */
+const KV_BYTES_PER_ELEMENT = 2
+
+/**
+ * Exact KV-cache cost per token, from the model's own GGUF metadata.
+ *
+ * 2 (K and V) x layers x kv_heads x head_dim x bytes_per_element
+ *
+ * For llama3:8b that is 2 x 32 x 8 x 128 x 2 = 128 KiB per token, so an 8k
+ * window costs 1 GiB — which is why "just set num_ctx high" is not free advice
+ * on the hardware NOMAD targets.
+ *
+ * Returns undefined when the metadata is missing (non-Ollama backend, or an
+ * architecture that names its keys differently), leaving the caller to fall back
+ * to `estimateKvBytesPerToken`.
+ */
+export function computeKvBytesPerToken(modelInfo: unknown): number | undefined {
+ if (!modelInfo) return undefined
+ const entries: [string, any][] =
+ modelInfo instanceof Map
+ ? [...modelInfo.entries()]
+ : typeof modelInfo === 'object'
+ ? Object.entries(modelInfo as Record)
+ : []
+ if (entries.length === 0) return undefined
+ const info = new Map(entries)
+
+ const arch = info.get('general.architecture')
+ if (typeof arch !== 'string') return undefined
+ const num = (suffix: string): number | undefined => {
+ const value = info.get(`${arch}.${suffix}`)
+ return typeof value === 'number' && value > 0 ? value : undefined
+ }
+
+ const layers = num('block_count')
+ // Grouped-query attention means KV heads are usually far fewer than attention
+ // heads; using head_count here would overestimate cost several-fold.
+ const kvHeads = num('attention.head_count_kv') ?? num('attention.head_count')
+ if (!layers || !kvHeads) return undefined
+
+ // Prefer explicit key/value lengths; otherwise derive head_dim from the
+ // embedding width and the attention head count.
+ let headDim = num('attention.key_length')
+ if (!headDim) {
+ const embedding = num('embedding_length')
+ const heads = num('attention.head_count')
+ if (embedding && heads) headDim = embedding / heads
+ }
+ if (!headDim) return undefined
+
+ return 2 * layers * kvHeads * headDim * KV_BYTES_PER_ELEMENT
+}
+
+/**
+ * Fallback KV cost per token when GGUF metadata is unavailable, interpolated
+ * from the common architectures at each size. Deliberately pessimistic — an
+ * overestimate costs a smaller window, an underestimate costs an OOM at load.
+ */
+export function estimateKvBytesPerToken(paramBillions: number | undefined): number {
+ const b = paramBillions ?? 8
+ if (b <= 1.5) return 32 * 1024
+ if (b <= 4) return 64 * 1024
+ if (b <= 9) return 128 * 1024
+ if (b <= 15) return 192 * 1024
+ if (b <= 35) return 256 * 1024
+ return 512 * 1024
+}
+
+/**
+ * Fraction of memory left free for weights-adjacent overhead: compute buffers,
+ * activations, the graph, and whatever else shares the device. Without this the
+ * arithmetic says a window fits right up to the byte and the model fails to load.
+ */
+const MEMORY_SAFETY_FRACTION = 0.8
+
+export type ContextWindowInputs = {
+ /** Trained context length from /api/show. The hard ceiling. */
+ modelMaxCtx?: number
+ /** num_ctx baked into the modelfile, treated as an author's ceiling hint. */
+ modelfileNumCtx?: number
+ /** Cost of one token of KV cache, exact or estimated. */
+ kvBytesPerToken: number
+ /** Memory available for the model, in bytes (VRAM, or a share of system RAM). */
+ availableBytes?: number
+ /** On-disk size of the weights, in bytes — they occupy the same pool. */
+ modelBytes?: number
+ /** User cap from `ai.contextWindow`. Only ever lowers the result. */
+ userCap?: number
+}
+
+export type ContextWindowDecision = {
+ contextWindow: number
+ /** Which constraint actually bound the result — surfaced in logs and the UI. */
+ limitedBy: 'user' | 'model' | 'memory' | 'floor' | 'default'
+ affordableTokens?: number
+}
+
+/**
+ * Decide the context window for a model.
+ *
+ * The result must be *stable for the life of the process*: Ollama unloads and
+ * reloads a model whenever a request asks for a different num_ctx than the
+ * loaded instance, which stalls the turn and discards the KV cache. Callers
+ * memoize per model; this function is pure so that memoization is safe.
+ */
+export function resolveContextWindow(inputs: ContextWindowInputs): ContextWindowDecision {
+ const ceilings: Array<{ value: number; source: ContextWindowDecision['limitedBy'] }> = []
+
+ if (inputs.modelMaxCtx && inputs.modelMaxCtx > 0) {
+ ceilings.push({ value: inputs.modelMaxCtx, source: 'model' })
+ }
+ if (inputs.modelfileNumCtx && inputs.modelfileNumCtx > 0) {
+ ceilings.push({ value: inputs.modelfileNumCtx, source: 'model' })
+ }
+ if (inputs.userCap && inputs.userCap > 0) {
+ ceilings.push({ value: inputs.userCap, source: 'user' })
+ }
+
+ let affordableTokens: number | undefined
+ if (inputs.availableBytes && inputs.availableBytes > 0 && inputs.kvBytesPerToken > 0) {
+ const forKv = inputs.availableBytes * MEMORY_SAFETY_FRACTION - (inputs.modelBytes ?? 0)
+ affordableTokens = Math.max(0, Math.floor(forKv / inputs.kvBytesPerToken))
+ ceilings.push({ value: affordableTokens, source: 'memory' })
+ }
+
+ if (ceilings.length === 0) {
+ return { contextWindow: UNKNOWN_BACKEND_CONTEXT, limitedBy: 'default' }
+ }
+
+ const binding = ceilings.reduce((lowest, c) => (c.value < lowest.value ? c : lowest))
+ const snapped = snapToLadder(binding.value)
+
+ // The floor wins over a *memory* shortfall: a window under 4k is not worth
+ // having, and Ollama will offload to host RAM rather than refuse. It must
+ // never win over the model itself, though — asking for more than the trained
+ // context pushes RoPE past what the weights ever saw and degrades output.
+ // tinyllama (trained 2048) is the case that catches this.
+ const hardCeiling = Math.min(
+ ...[inputs.modelMaxCtx, inputs.modelfileNumCtx, inputs.userCap].filter(
+ (v): v is number => typeof v === 'number' && v > 0
+ ),
+ Number.POSITIVE_INFINITY
+ )
+
+ if (snapped <= MIN_CONTEXT) {
+ const floored = Math.min(MIN_CONTEXT, hardCeiling)
+ return {
+ contextWindow: floored,
+ limitedBy: floored < MIN_CONTEXT ? 'model' : 'floor',
+ affordableTokens,
+ }
+ }
+
+ return { contextWindow: snapped, limitedBy: binding.source, affordableTokens }
+}
+
+/** Parse the `ai.contextWindow` setting. "auto"/unset/garbage all mean "no cap". */
+export function parseUserContextCap(raw: string | null | undefined): number | undefined {
+ if (!raw || raw === 'auto') return undefined
+ const value = Number.parseInt(raw, 10)
+ return Number.isFinite(value) && value >= MIN_CONTEXT ? value : undefined
+}
diff --git a/admin/app/utils/eval/report.ts b/admin/app/utils/eval/report.ts
index 40953166..50a5a115 100644
--- a/admin/app/utils/eval/report.ts
+++ b/admin/app/utils/eval/report.ts
@@ -82,6 +82,16 @@ export const METRIC_DIRECTIONS: Record = {
thinkTagLeakRate: 'lower-is-better',
markdownRate: 'higher-is-better',
groundedness: 'higher-is-better',
+ // context budgeting
+ //
+ // These are diagnostics as much as gates. A rise in historyElidedRate means
+ // conversations are being trimmed more often, which explains a correctness
+ // drop that would otherwise look like the model getting worse. promptTokenError
+ // gates the token estimator itself: every budget decision rests on it, so a
+ // regression there quietly degrades everything downstream.
+ historyElidedRate: 'lower-is-better',
+ chunksDroppedRate: 'lower-is-better',
+ promptTokenError: 'lower-is-better',
}
export type MetricDelta = {
diff --git a/admin/app/utils/rag_prompt.ts b/admin/app/utils/rag_prompt.ts
index ca2b61f7..531fa2ed 100644
--- a/admin/app/utils/rag_prompt.ts
+++ b/admin/app/utils/rag_prompt.ts
@@ -1,32 +1,20 @@
/**
- * Pure prompt-budgeting helpers for the chat RAG pipeline.
+ * Pure helpers for shaping the retrieved-context block.
*
* These live here rather than in RagPipelineService so they can be exercised
* under bare `node --experimental-strip-types` with no MySQL, Redis, Qdrant, or
* Ollama — the same reason `kb_ratio_lookup.ts` is shaped this way. The service
* supplies the config; everything below is a function of its arguments.
- */
-
-/**
- * Chars-per-token estimate used when budgeting the *prompt* (context trimming
- * and the num_ctx ladder).
*
- * NOTE: RagService.CHAR_TO_TOKEN_RATIO is 2, not 3.5 — the two halves of the
- * system disagree about what a token costs, and RagService's own doc-comment
- * says 3. That divergence is real and known; reconciling it changes chunk size
- * and therefore retrieval results, so it is deliberately left alone here and
- * tracked as its own measured change rather than folded into a refactor.
+ * Token budgeting used to live here too, as a chars-per-token estimate feeding a
+ * `num_ctx` ladder. Both are gone: the estimate now comes from
+ * `token_estimate.ts` (calibrated against real token counts) and the allocation
+ * from `context_budget.ts`. The ladder in particular was dead code — it fed a
+ * `num_ctx` that the OpenAI-compatible endpoint silently discarded, so no chat
+ * ever ran at the window it computed.
*/
-export const PROMPT_CHARS_PER_TOKEN = 3.5
-/**
- * num_ctx is only requested once the system prompt is large enough to risk
- * overflowing Ollama's silent 2048 default. Below the trigger we send nothing
- * and inherit the server default.
- */
-export const NUM_CTX_TRIGGER_TOKENS = 3000
-export const NUM_CTX_RESPONSE_HEADROOM = 2048
-export const NUM_CTX_LADDER = [8192, 16384, 32768, 65536]
+import { parseParameterBillions } from './context_window.js'
export type ContextLimits = { maxResults: number; maxTokens: number }
export type ContextLimitTier = { maxParams: number; maxResults: number; maxTokens: number }
@@ -36,20 +24,21 @@ export type BudgetableChunk = { text: string; metadata?: Record }
export type BudgetableMessage = { role: 'system' | 'user' | 'assistant'; content: string }
/**
- * Determine RAG context limits from the model size encoded in its name.
- * Parses size indicators like "1b", "3b", "8b", "70b".
+ * Determine RAG context limits from the model's parameter count.
*
- * PRESERVED QUIRK: an unparseable model name is treated as 8B. That is a guess,
- * and for a name like "phi3" or a custom tag it can hand a small model far more
- * context than it can actually use. Faithful to the pre-extraction behaviour.
+ * `parameterSize` is Ollama's own `details.parameter_size` ("8.0B") and is the
+ * honest source. The model name is only a fallback: it used to be the *only*
+ * source, which meant a tag carrying no size — "phi3", "mistral-nemo", anything
+ * custom — was silently assumed to be 8B and handed far more context than a
+ * small model can actually use. When neither source knows, 8B remains the
+ * assumption, but now it is one made explicitly rather than by a failed regex.
*/
export function getContextLimitsForModel(
modelName: string,
- tiers: readonly ContextLimitTier[]
+ tiers: readonly ContextLimitTier[],
+ parameterSize?: string
): ContextLimits {
- // e.g. "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b"
- const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/)
- const paramBillions = sizeMatch ? Number.parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown
+ const paramBillions = parseParameterBillions(parameterSize, modelName) ?? 8
for (const tier of tiers) {
if (paramBillions <= tier.maxParams) {
@@ -60,29 +49,6 @@ export function getContextLimitsForModel(
return { maxResults: 5, maxTokens: 0 }
}
-/**
- * Apply the model-size context budget: cap the result count, then cap total
- * characters.
- *
- * The first (most relevant) result is always kept — the token cap only gates
- * subsequent results, so a single oversized chunk never starves the model of
- * context entirely.
- */
-export function trimToContextBudget(
- docs: T[],
- limits: ContextLimits
-): T[] {
- const byCount = docs.slice(0, limits.maxResults)
- if (limits.maxTokens <= 0) return byCount
-
- const charCap = limits.maxTokens * PROMPT_CHARS_PER_TOKEN
- let totalChars = 0
- return byCount.filter((doc, idx) => {
- totalChars += doc.text.length
- return idx === 0 || totalChars <= charCap
- })
-}
-
/**
* Render retrieved chunks into the block the rag_context prompt wraps.
*
@@ -101,23 +67,3 @@ export function buildContextBlock(docs: BudgetableChunk[]): string {
})
.join('\n\n')
}
-
-/**
- * Request a context window big enough to hold the system messages, but only
- * once they are large enough to be at risk.
- *
- * Ollama respects num_ctx per request; LM Studio ignores it gracefully. Below
- * the trigger we send nothing and inherit the server default — which for Ollama
- * is a silent 2048.
- */
-export function deriveNumCtx(messages: BudgetableMessage[]): number | undefined {
- const systemChars = messages
- .filter((m) => m.role === 'system')
- .reduce((sum, m) => sum + m.content.length, 0)
- const estimatedSystemTokens = Math.ceil(systemChars / PROMPT_CHARS_PER_TOKEN)
-
- if (estimatedSystemTokens <= NUM_CTX_TRIGGER_TOKENS) return undefined
-
- const needed = estimatedSystemTokens + NUM_CTX_RESPONSE_HEADROOM
- return NUM_CTX_LADDER.find((n) => n >= needed) ?? NUM_CTX_LADDER[NUM_CTX_LADDER.length - 1]
-}
diff --git a/admin/app/utils/tasks_model.ts b/admin/app/utils/tasks_model.ts
new file mode 100644
index 00000000..4e79f1a3
--- /dev/null
+++ b/admin/app/utils/tasks_model.ts
@@ -0,0 +1,64 @@
+import logger from '@adonisjs/core/services/logger'
+import KVStore from '#models/kv_store'
+import type { OllamaService } from '#services/ollama_service'
+import { pickTasksModel } from './misc.js'
+
+/**
+ * Resolve the model to use for ancillary work — chat titles, suggestions, and
+ * the per-turn retrieval query rewrite.
+ *
+ * Lifted out of ChatService so the RAG pipeline can use it too. The rewrite is
+ * the ancillary call that matters most and was the one still running on the main
+ * chat model: it happens on *every* turn, on the critical path, and it sends a
+ * completely different prompt. With Ollama's default OLLAMA_NUM_PARALLEL=1 there
+ * is a single slot, so that call evicted the chat model's cached prefix every
+ * turn and guaranteed a full re-prefill of the conversation. Routing it to a
+ * small dedicated model keeps the chat model's KV cache intact.
+ *
+ * Falls back to `fallback` when the setting is unset (preserving the previous
+ * behaviour) or when the configured model is no longer installed. `installed` is
+ * passed by callers that already listed models, to avoid a second round-trip.
+ */
+export async function resolveTasksModel(
+ ollamaService: OllamaService,
+ fallback: string | null,
+ installed?: { name: string }[],
+ logPrefix = '[TasksModel]'
+): Promise {
+ let configured: string | null = null
+ try {
+ configured = await KVStore.getValue('ai.tasksModel')
+ } catch (error) {
+ logger.error(
+ `${logPrefix} Failed to read ai.tasksModel: ${error instanceof Error ? error.message : error}`
+ )
+ return fallback
+ }
+ if (!configured?.trim()) {
+ return fallback
+ }
+
+ let models = installed
+ if (!models) {
+ try {
+ models = await ollamaService.getModels()
+ } catch (error) {
+ logger.error(
+ `${logPrefix} Failed to list models while resolving the tasks model: ${error instanceof Error ? error.message : error}`
+ )
+ return fallback
+ }
+ }
+
+ const { model, staleConfigured } = pickTasksModel(
+ configured,
+ (models ?? []).map((m) => m.name),
+ fallback
+ )
+ if (staleConfigured) {
+ logger.warn(
+ `${logPrefix} Configured tasks model "${staleConfigured}" is not installed; falling back to "${fallback ?? 'none'}"`
+ )
+ }
+ return model
+}
diff --git a/admin/app/utils/think_stream.ts b/admin/app/utils/think_stream.ts
new file mode 100644
index 00000000..7a3b120e
--- /dev/null
+++ b/admin/app/utils/think_stream.ts
@@ -0,0 +1,105 @@
+/**
+ * Stateful splitter for inline `...` reasoning tags.
+ *
+ * Ollama's native `/api/chat` and its `/v1` shim both surface reasoning as a
+ * dedicated field (`message.thinking` / `delta.reasoning`), but other
+ * OpenAI-compatible backends (LM Studio, llama.cpp, plain GGUF servers) emit
+ * the tags inline in the content stream. This splits them back out so the UI
+ * gets a clean `content` channel and a separate `thinking` channel regardless
+ * of which backend answered.
+ *
+ * Lives here rather than inside OllamaService because both transports need it
+ * and because it is the only real logic in the streaming path — as a pure
+ * class it can be exercised under bare `node --test` with no Ollama, the same
+ * reason `rag_prompt.ts` is shaped this way.
+ *
+ * The hard part is that a tag can be split across chunk boundaries: a chunk may
+ * end with `"= 1; len--) {
+ if (text.endsWith(tag.slice(0, len))) return len
+ }
+ return 0
+}
+
+export type ThinkSplit = { content: string; thinking: string }
+
+export class ThinkTagSplitter {
+ private buffer = ''
+ private inThink = false
+
+ /**
+ * Feed the next slice of raw model output. Returns only what could be
+ * resolved from this call; text that might be a partial tag stays buffered.
+ */
+ push(raw: string): ThinkSplit {
+ this.buffer += raw
+ let content = ''
+ let thinking = ''
+
+ while (this.buffer.length > 0) {
+ if (this.inThink) {
+ const closeIdx = this.buffer.indexOf(CLOSE_TAG)
+ if (closeIdx !== -1) {
+ thinking += this.buffer.slice(0, closeIdx)
+ this.buffer = this.buffer.slice(closeIdx + CLOSE_TAG.length)
+ this.inThink = false
+ } else {
+ const hold = partialTagSuffix(CLOSE_TAG, this.buffer)
+ thinking += this.buffer.slice(0, this.buffer.length - hold)
+ this.buffer = this.buffer.slice(this.buffer.length - hold)
+ break
+ }
+ } else {
+ const openIdx = this.buffer.indexOf(OPEN_TAG)
+ if (openIdx !== -1) {
+ content += this.buffer.slice(0, openIdx)
+ this.buffer = this.buffer.slice(openIdx + OPEN_TAG.length)
+ this.inThink = true
+ } else {
+ const hold = partialTagSuffix(OPEN_TAG, this.buffer)
+ content += this.buffer.slice(0, this.buffer.length - hold)
+ this.buffer = this.buffer.slice(this.buffer.length - hold)
+ break
+ }
+ }
+ }
+
+ return { content, thinking }
+ }
+
+ /**
+ * Flush whatever is still buffered at end of stream. A held-back partial tag
+ * that never completed was never a tag, so it belongs to whichever channel
+ * we were in when the stream ended — losing it would silently truncate the
+ * answer.
+ */
+ flush(): ThinkSplit {
+ const pending = this.buffer
+ this.buffer = ''
+ return this.inThink ? { content: '', thinking: pending } : { content: pending, thinking: '' }
+ }
+}
+
+/**
+ * One-shot convenience for non-streaming responses.
+ */
+export function splitThinkTags(raw: string): ThinkSplit {
+ const splitter = new ThinkTagSplitter()
+ const streamed = splitter.push(raw)
+ const tail = splitter.flush()
+ return {
+ content: streamed.content + tail.content,
+ thinking: streamed.thinking + tail.thinking,
+ }
+}
diff --git a/admin/app/utils/token_estimate.ts b/admin/app/utils/token_estimate.ts
new file mode 100644
index 00000000..b8406a02
--- /dev/null
+++ b/admin/app/utils/token_estimate.ts
@@ -0,0 +1,191 @@
+/**
+ * Token estimation for prompt budgeting.
+ *
+ * NOMAD cannot tokenize exactly. The model's real vocabulary lives inside a GGUF
+ * on the Ollama side, Ollama exposes no tokenize endpoint (ollama#12030 is still
+ * open), and bundling a BPE vocabulary would mean shipping the *wrong* vocabulary
+ * for whatever Llama/Qwen/Gemma/Granite model the user actually pulled. Fetching
+ * the right one per model needs network access, which an offline-first appliance
+ * does not have.
+ *
+ * So this estimates — but it estimates in a way that corrects itself. Two parts:
+ *
+ * 1. A structural segmenter (below). BPE vocabularies are built from natural text,
+ * so in practice a token is roughly "a word, or a punctuation mark, or a chunk
+ * of a long/rare word". Counting those directly tracks a real tokenizer far
+ * better than dividing character count by a constant, because the constant is
+ * only right for the kind of text it was tuned on. Prose runs ~4 chars/token;
+ * dense JSON, markdown tables and code run closer to 2.5. Those are exactly the
+ * payloads that overflow a window, so that is exactly where a fixed ratio is
+ * most wrong and most dangerous.
+ *
+ * 2. A per-model correction factor learned from ground truth. Every Ollama
+ * response reports `prompt_eval_count` — the real token count of the prompt we
+ * just sent. TokenCalibrationService folds the observed error into a per-model
+ * EWMA and feeds it back here as `ratio`. Costs nothing: no extra inference, no
+ * extra dependency, and it converges on the actual tokenizer within a couple of
+ * turns.
+ *
+ * Measured against real `prompt_eval_count` over a six-fixture set (prose,
+ * markdown table, TypeScript, JSON, bullet list, mixed long-form), mean absolute
+ * error after per-model calibration:
+ *
+ * llama3:8b segmenter 5.7% chars/3.5 18.0%
+ * qwen2.5:0.5b segmenter 6.2% chars/3.5 20.0%
+ * granite4.1:8b segmenter 6.7% chars/3.5 18.4%
+ *
+ * The learned factors were k=1.02, k=1.26 and k=1.00 respectively — which is the
+ * argument for making it per-model rather than a global constant. Qwen tokenizes
+ * noticeably finer than Llama, and no single tuned divisor can serve both.
+ *
+ * Kept pure and dependency-free so it runs under bare `node --test`, the same
+ * convention as `rag_prompt.ts`.
+ */
+
+/**
+ * Chars-per-token used by the *ingestion* path when sizing chunks for the
+ * embedding model.
+ *
+ * Deliberately unchanged from RagService's historical value. Chunk size is baked
+ * into the eval corpus fingerprint, so touching it re-chunks the corpus and
+ * invalidates every committed retrieval baseline. Re-tuning it is a legitimate
+ * change, but it is a *retrieval* change and belongs to its own measured PR — not
+ * folded silently into a context-management refactor. Re-exported here only so
+ * there is one obvious place to find every token constant.
+ */
+export const INGEST_CHARS_PER_TOKEN = 2
+
+/**
+ * Per-message overhead from the chat template: role marker, turn delimiters, and
+ * the trailing newline the template adds. Real templates cost roughly 3-5 tokens
+ * a message. The old char-based math ignored this entirely, which quietly
+ * under-counted a 40-turn conversation by well over a hundred tokens.
+ */
+export const PER_MESSAGE_OVERHEAD_TOKENS = 4
+
+/** Tokens the template adds once, for the generation prompt that follows the messages. */
+export const CONVERSATION_OVERHEAD_TOKENS = 3
+
+/**
+ * Safety margin applied when a budget decision must not overflow.
+ *
+ * Under-estimating is strictly worse than over-estimating: an over-estimate wastes
+ * a little window, an under-estimate overflows it and hands the backend a silent
+ * truncation — the exact failure this whole subsystem exists to prevent.
+ */
+export const ESTIMATE_SAFETY_MARGIN = 1.1
+
+/**
+ * Character-class fragments, kept as strings so the CJK ranges appear exactly
+ * once and the regexes below stay readable.
+ */
+const CJK = '\\u3040-\\u30ff\\u3400-\\u4dbf\\u4e00-\\u9fff\\uf900-\\ufaff\\uac00-\\ud7af'
+const WORD = 'A-Za-z0-9\\u00c0-\\u024f'
+
+/**
+ * Segments that a BPE tokenizer tends to split on. Order matters: the first
+ * alternative that matches wins.
+ *
+ * - CJK/Hangul codepoints are usually one token each (often more), so they are
+ * matched singly rather than as words.
+ * - Runs of letters/digits are word-ish; long ones get sub-split below.
+ * - Runs of punctuation/symbols are matched as a *run*, not per character. BPE
+ * vocabularies contain merged punctuation pieces (`":`, `",`, `);`, `],`), so
+ * charging one token per punctuation character over-counts structured text
+ * badly -- measured at +48% on a JSON payload before this was split out.
+ */
+const SEGMENT_RE = new RegExp(`[${CJK}]|[${WORD}]+|[^\\s${WORD}${CJK}]+`, 'gu')
+
+/** Identifies the punctuation-run alternative when walking matches. */
+const PUNCT_RUN_RE = new RegExp(`^[^\\s${WORD}${CJK}]+$`, 'u')
+
+/**
+ * Above this length, a run of letters/digits is a rare word, an identifier, or a
+ * hash, and a real tokenizer will break it into several pieces.
+ */
+const LONG_WORD_CHARS = 7
+
+/** Roughly how many characters of a punctuation run one merged token covers. */
+const PUNCT_CHARS_PER_TOKEN = 2
+
+/**
+ * Estimate the token count of a string.
+ *
+ * `ratio` is the learned per-model correction (1.0 = uncalibrated). Pass the value
+ * from TokenCalibrationService.ratioFor(model).
+ */
+export function estimateTokens(text: string, ratio = 1): number {
+ if (!text) return 0
+
+ let tokens = 0
+ const matches = text.match(SEGMENT_RE)
+ if (matches) {
+ for (const segment of matches) {
+ if (PUNCT_RUN_RE.test(segment)) {
+ // Punctuation merges: `":` and `},` are usually single tokens.
+ tokens += Math.ceil(segment.length / PUNCT_CHARS_PER_TOKEN)
+ } else if (segment.length > LONG_WORD_CHARS) {
+ // Long words split into roughly one piece per LONG_WORD_CHARS characters.
+ tokens += Math.ceil(segment.length / LONG_WORD_CHARS)
+ } else {
+ tokens += 1
+ }
+ }
+ }
+
+ // Leading whitespace is folded into the following token by most BPE
+ // vocabularies, but newlines are usually tokens in their own right and matter
+ // for the markdown-heavy content NOMAD deals in.
+ const newlines = (text.match(/\n/g) || []).length
+ tokens += newlines
+
+ return Math.max(1, Math.ceil(tokens * ratio))
+}
+
+export type EstimatableMessage = { role: string; content: string }
+
+/**
+ * Estimate the token cost of a full message array as the backend will see it,
+ * including chat-template overhead.
+ */
+export function estimateMessagesTokens(messages: EstimatableMessage[], ratio = 1): number {
+ if (messages.length === 0) return 0
+ let total = CONVERSATION_OVERHEAD_TOKENS
+ for (const message of messages) {
+ total += estimateTokens(message.content, ratio) + PER_MESSAGE_OVERHEAD_TOKENS
+ }
+ return total
+}
+
+/**
+ * Estimate with the safety margin applied — use this wherever exceeding the
+ * budget causes silent truncation rather than merely wasting space.
+ */
+export function estimateTokensConservative(text: string, ratio = 1): number {
+ return Math.ceil(estimateTokens(text, ratio) * ESTIMATE_SAFETY_MARGIN)
+}
+
+/**
+ * Fold a fresh observation into an exponentially-weighted moving average.
+ *
+ * `alpha` is intentionally low: a single odd turn (a huge code paste, a burst of
+ * CJK) should nudge the ratio, not redefine it.
+ */
+export function updateEwma(previous: number | null, observed: number, alpha = 0.25): number {
+ if (previous === null || !Number.isFinite(previous) || previous <= 0) return observed
+ return previous * (1 - alpha) + observed * alpha
+}
+
+/**
+ * Guard rail on the learned ratio. A calibration ratio far outside this range
+ * means something other than tokenization differs — a backend injecting a hidden
+ * system prompt, a tool schema we didn't count, a reasoning preamble. Clamping
+ * keeps one bad reading from destroying the budget for every later turn.
+ */
+export const MIN_CALIBRATION_RATIO = 0.5
+export const MAX_CALIBRATION_RATIO = 2.5
+
+export function clampRatio(ratio: number): number {
+ if (!Number.isFinite(ratio) || ratio <= 0) return 1
+ return Math.min(MAX_CALIBRATION_RATIO, Math.max(MIN_CALIBRATION_RATIO, ratio))
+}
diff --git a/admin/app/validators/settings.ts b/admin/app/validators/settings.ts
index 089e9221..8c28583e 100644
--- a/admin/app/validators/settings.ts
+++ b/admin/app/validators/settings.ts
@@ -1,6 +1,7 @@
import vine from "@vinejs/vine";
import { SETTINGS_KEYS } from "../../constants/kv_store.js";
import type { KVStoreKey } from "../../types/kv_store.js";
+import { CONTEXT_LADDER } from "../utils/context_window.js";
export const getSettingSchema = vine.compile(vine.object({
key: vine.enum(SETTINGS_KEYS),
@@ -62,6 +63,30 @@ export function validateSettingValue(key: KVStoreKey, value: unknown): string |
}
return null
}
+ case 'ai.contextWindow': {
+ // "auto" (or empty) hands sizing to ContextWindowService. An explicit
+ // value is a *cap*, so it only ever lowers the resolved window — but it
+ // still has to be a real ladder rung, since a value the backend can't
+ // honour would silently fall back to the default and confuse the user.
+ if (value === '' || value === 'auto' || value === undefined || value === null) {
+ return null
+ }
+ const num = Number(value)
+ if (!CONTEXT_LADDER.includes(num as (typeof CONTEXT_LADDER)[number])) {
+ return `Context window must be "auto" or one of: ${CONTEXT_LADDER.join(', ')}.`
+ }
+ return null
+ }
+ case 'ai.keepAlive': {
+ // Ollama duration format: "10m", "1h", "30s", "-1" (forever), "0" (evict).
+ if (value === '' || value === undefined || value === null) {
+ return null
+ }
+ if (typeof value !== 'string' || !/^(-?\d+|\d+(\.\d+)?(ms|s|m|h))$/.test(value)) {
+ return 'Keep-alive must be a duration like "10m", "1h", or "-1" to keep the model loaded indefinitely.'
+ }
+ return null
+ }
default:
return null
}
diff --git a/admin/commands/eval/corpus.ts b/admin/commands/eval/corpus.ts
index 1f12d195..b01aa57e 100644
--- a/admin/commands/eval/corpus.ts
+++ b/admin/commands/eval/corpus.ts
@@ -75,7 +75,7 @@ export default class EvalCorpus extends BaseCommand {
const [fingerprint, chunks, goldens] = await Promise.all([
service.fingerprint(),
service.count(),
- service.loadGoldens(),
+ service.loadGoldens(['*']),
])
this.logger.info(`Fingerprint: ${fingerprint}`)
this.logger.info(`Chunks in KB: ${chunks}`)
@@ -93,7 +93,7 @@ export default class EvalCorpus extends BaseCommand {
private async runCheck(service: any) {
try {
const corpus = await service.loadCorpus()
- const goldens = await service.loadGoldens()
+ const goldens = await service.loadGoldens(['*'])
const fingerprint = await service.fingerprint()
const tagCounts = new Map()
@@ -131,7 +131,7 @@ export default class EvalCorpus extends BaseCommand {
this.logger.info('Rebuilding the eval corpus (user content is never touched)...')
// Validate before destroying anything — a typo in a golden should not cost
// you the ingest you were about to run.
- await service.loadGoldens()
+ await service.loadGoldens(['*'])
const summary = await service.ingest((docId: string, index: number, total: number) => {
this.logger.info(` [${index}/${total}] ${docId}`)
diff --git a/admin/commands/eval/generation.ts b/admin/commands/eval/generation.ts
index 2de609df..7075e5ec 100644
--- a/admin/commands/eval/generation.ts
+++ b/admin/commands/eval/generation.ts
@@ -39,6 +39,13 @@ export default class EvalGeneration extends BaseCommand {
@flags.string({ description: 'Repeats per question, for stability (default: 3)' })
declare repeats: string
+ @flags.string({
+ description:
+ 'Comma-separated golden suites to run, by filename without .jsonl (default: core). ' +
+ 'Extra suites are opt-in so they cannot move the numbers a baseline was recorded against.',
+ })
+ declare suite: string
+
@flags.string({ description: 'Only run goldens carrying this tag' })
declare tag: string
@@ -82,7 +89,7 @@ export default class EvalGeneration extends BaseCommand {
}
}
- let goldens = await corpusService.loadGoldens()
+ let goldens = await corpusService.loadGoldens(this.suite ? this.suite.split(',').map((s) => s.trim()) : undefined)
if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag))
if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10))
if (goldens.length === 0) {
@@ -161,6 +168,29 @@ export default class EvalGeneration extends BaseCommand {
this.logger.info(
` mean answer length ${agg.meanAnswerLength === null ? 'n/a' : Math.round(agg.meanAnswerLength)} chars`
)
+ // Context budgeting. These explain *why* a correctness number moved: an
+ // answer the model never had the material for is a different failure from
+ // an answer it got wrong, and only these two lines tell them apart.
+ if (agg.meanPromptTokens !== null) {
+ this.logger.info(` mean prompt tokens ${Math.round(agg.meanPromptTokens)}`)
+ }
+ if (agg.historyElidedRate !== null) {
+ this.logger.info(
+ ` history elided ${fmt(agg.historyElidedRate)} (share of cases needing a trim)`
+ )
+ }
+ if (agg.chunksDroppedRate !== null) {
+ this.logger.info(
+ ` context dropped ${fmt(agg.chunksDroppedRate)} (share of cases where a chunk didn't fit)`
+ )
+ }
+ if (agg.promptTokenError) {
+ // Every budget decision rests on the estimate; if this drifts, the trims
+ // above are being made against a number that is no longer true.
+ this.logger.info(
+ ` token estimate err ${(agg.promptTokenError.mean * 100).toFixed(1)}% (vs the backend's real count)`
+ )
+ }
if (agg.unstable > 0) {
this.logger.warning(
` ${agg.unstable} question(s) flipped between repeats — excluded from gating, do not read them as a regression.`
diff --git a/admin/commands/eval/retrieval.ts b/admin/commands/eval/retrieval.ts
index 6ca6fb53..fa6202f5 100644
--- a/admin/commands/eval/retrieval.ts
+++ b/admin/commands/eval/retrieval.ts
@@ -28,6 +28,13 @@ export default class EvalRetrieval extends BaseCommand {
@flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' })
declare ablate: boolean
+ @flags.string({
+ description:
+ 'Comma-separated golden suites to run, by filename without .jsonl (default: core). ' +
+ 'Extra suites are opt-in so they cannot move the numbers a baseline was recorded against.',
+ })
+ declare suite: string
+
@flags.string({ description: 'Only run goldens carrying this tag' })
declare tag: string
@@ -58,7 +65,7 @@ export default class EvalRetrieval extends BaseCommand {
try {
const chunks = await retrievalService.assertCorpusReady()
const fingerprint = await corpusService.fingerprint()
- let goldens = await corpusService.loadGoldens()
+ let goldens = await corpusService.loadGoldens(this.suite ? this.suite.split(',').map((s) => s.trim()) : undefined)
if (this.tag) {
goldens = goldens.filter((g) => g.tags.includes(this.tag))
diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts
index 09d99d92..625a6341 100644
--- a/admin/constants/kv_store.ts
+++ b/admin/constants/kv_store.ts
@@ -12,6 +12,8 @@ export const SETTINGS_KEYS: KVStoreKey[] = [
'ai.ollamaFlashAttention',
'ai.autoThinking',
'ai.tasksModel',
+ 'ai.contextWindow',
+ 'ai.keepAlive',
'rag.defaultIngestPolicy',
'rag.enabled',
'autoUpdate.enabled',
diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts
index 54d2ea70..43f5abd1 100644
--- a/admin/constants/ollama.ts
+++ b/admin/constants/ollama.ts
@@ -64,6 +64,23 @@ export const FALLBACK_RECOMMENDED_OLLAMA_MODELS: NomadOllamaModel[] = [
export const EMBEDDING_MODEL_NAME = 'nomic-embed-text:v1.5'
+/**
+ * Server-side context floor set as `OLLAMA_CONTEXT_LENGTH` on the nomad_ollama
+ * container.
+ *
+ * Ollama defaults to 4096 tokens on machines under 24GB VRAM and silently
+ * truncates anything longer — no error, no warning, the model simply never sees
+ * the start of the conversation. This raises the floor for every request that
+ * can't carry its own `num_ctx`, which notably includes the whole
+ * OpenAI-compatible path (that endpoint has no context-size field at all).
+ *
+ * Deliberately conservative rather than clever: it is allocated at model load on
+ * whatever hardware NOMAD happens to be running, so it has to be affordable on a
+ * small box. Per-model, hardware-aware sizing happens per request via
+ * ContextWindowResolver, which can go well above this.
+ */
+export const DEFAULT_OLLAMA_CONTEXT_LENGTH = 8192
+
/**
* Adaptive RAG context limits based on model size.
* Smaller models get overwhelmed with too much context, so we cap it.
@@ -83,6 +100,60 @@ export const RAG_CONTEXT_LIMITS: { maxParams: number; maxResults: number; maxTok
export const RAG_DEFAULT_TOP_K = 5
export const RAG_DEFAULT_SCORE_THRESHOLD = 0.3
+/**
+ * Where the per-turn retrieved-context block sits in the prompt.
+ *
+ * `tail` places it immediately before the current question, so
+ * [system][history] stays byte-identical from turn to turn and the backend can
+ * reuse its KV cache for the whole conversation. `front` is the historical
+ * placement — a system message ahead of all history — which changes content
+ * every turn and therefore invalidates the cached prefix behind it, making
+ * follow-ups progressively slower as the conversation grows.
+ *
+ * Measured on llama3:8b over a 14-turn conversation (~400-token retrieved block
+ * per turn), prefill time for the same prompt:
+ *
+ * turn 1 front 214 ms tail 214 ms
+ * turn 7 front 347 ms tail 225 ms
+ * turn 14 front 539 ms tail 231 ms
+ *
+ * front grows about 25 ms per turn; tail grows about 1 ms. By turn 14 that is
+ * 2.3x the time-to-first-token, and the gap keeps widening — the familiar
+ * "first answer fast, follow-ups get slower" complaint.
+ *
+ * Caveat worth knowing before trusting this: the saving is proportional to how
+ * much of the prompt is *stable*. With a very large retrieved block and a short
+ * history the block dominates, must be reprocessed either way, and the two
+ * placements measure the same. Tail placement never loses, but it only clearly
+ * wins once a conversation has accumulated history — which is exactly when
+ * responsiveness starts to matter.
+ *
+ * Both are kept so `eval:generation` can measure the answer-quality difference
+ * rather than the choice resting on the cache argument alone. Flip to 'front'
+ * to reproduce pre-change behaviour.
+ */
+export const RAG_PLACEMENT: 'tail' | 'front' = 'tail'
+
+/**
+ * Token cap on the query-rewrite call. A rewrite is one short sentence; the
+ * prompt already asks for under 150 words. Without a cap a small model can
+ * ramble, and every one of those tokens is latency the user waits through
+ * before retrieval even starts.
+ */
+export const QUERY_REWRITE_MAX_TOKENS = 120
+
+/**
+ * How long Ollama keeps a chat model — and its KV cache — resident after a
+ * request.
+ *
+ * Ollama's default is 5 minutes, which is comfortably shorter than the time a
+ * user spends reading an answer and typing a follow-up. Evicting in that gap
+ * throws away the cached prefix and forces a cold reload plus a full re-prefill
+ * on the next message, which is the single most obvious "why is it slow again?"
+ * moment in a conversation. Overridable via the `ai.keepAlive` setting.
+ */
+export const DEFAULT_KEEP_ALIVE = '15m'
+
export const SYSTEM_PROMPTS = {
default: `
Format all responses using markdown for better readability. Vanilla markdown or GitHub-flavored markdown is preferred.
diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx
index 462958ac..d7f24c21 100644
--- a/admin/inertia/pages/settings/models.tsx
+++ b/admin/inertia/pages/settings/models.tsx
@@ -27,7 +27,9 @@ export default function ModelsPage(props: {
models: {
availableModels: NomadOllamaModel[]
installedModels: NomadInstalledModel[]
- settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean }
+ settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string }
+ /** Effective window per installed model, as resolved by ContextWindowService. */
+ resolvedContextWindows?: Record
}
}) {
const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props
@@ -103,6 +105,7 @@ export default function ModelsPage(props: {
const [autoThinking, setAutoThinking] = useState(props.models.settings.autoThinking)
const [ragEnabled, setRagEnabled] = useState(props.models.settings.ragEnabled)
const [tasksModel, setTasksModel] = useState(props.models.settings.tasksModel)
+ const [contextWindow, setContextWindow] = useState(props.models.settings.contextWindow)
const [aiAssistantCustomName, setAiAssistantCustomName] = useState(
props.models.settings.aiAssistantCustomName
)
@@ -248,6 +251,23 @@ export default function ModelsPage(props: {
// A model can be deleted after being picked here. Surface the stale name as a
// disabled option instead of letting the select silently render empty — the
// backend already falls back to the chat model at call time.
+ // "Auto" sizes each model's window from its own trained context and what the
+ // hardware can afford. An explicit choice is a cap, never a boost — asking for
+ // more than a model or a GPU can support just degrades or fails to load.
+ const contextWindowOptions = [
+ { value: 'auto', label: 'Auto (recommended)' },
+ { value: '4096', label: '4K tokens' },
+ { value: '8192', label: '8K tokens' },
+ { value: '16384', label: '16K tokens' },
+ { value: '32768', label: '32K tokens' },
+ { value: '65536', label: '64K tokens' },
+ { value: '131072', label: '128K tokens' },
+ ]
+
+ const resolvedWindows = props.models.resolvedContextWindows ?? {}
+ const formatWindow = (tokens: number) =>
+ tokens >= 1024 ? `${Math.round(tokens / 1024)}K` : String(tokens)
+
const tasksModelOptions = [
{ value: '', label: 'Use the chat model' },
...props.models.installedModels.map((model) => ({ value: model.name, label: model.name })),
@@ -383,6 +403,26 @@ export default function ModelsPage(props: {
updateSettingMutation.mutate({ key: 'ai.tasksModel', value: newVal })
}}
/>
+