diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 858d2c74..f0feca4f 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -13,6 +13,8 @@ import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import { SERVICE_NAMES } from '../../constants/service_names.js' import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js' +import { resolveResponseStyle } from '../utils/response_style.js' +import { resolveSamplerProfile } from '../utils/sampler.js' import type { PipelineTrace } from '../../types/rag.js' import { buildCitations } from '../utils/rag_prompt.js' import logger from '@adonisjs/core/services/logger' @@ -175,7 +177,10 @@ export default class OllamaController { // Thinking is only enabled when the model supports it AND the user wants it: the explicit // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF). // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat - const thinkingCapability = modelCapabilities.thinking + // One /api/show read, memoized per model, for two decisions: whether the + // model can think, and what sampler values its author baked in. + const modelInfo = await this.ollamaService.getModelInfo(reqData.model) + const thinkingCapability = modelInfo.hasThinking let thinkingEnabled = false if (thinkingCapability) { thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false) @@ -183,6 +188,16 @@ export default class OllamaController { const think: boolean | 'medium' = thinkingEnabled ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false + // Sampler settings for this turn. Chat used to send none of these and + // inherit the backend's defaults, which on a heavily quantized small model + // is the wandering, repetitive output people read as "the local model is + // bad". Resolved here rather than in OllamaService so the ancillary + // structured calls, which want temperature 0 and a grammar, stay untouched. + const sampler = resolveSamplerProfile( + await resolveResponseStyle(), + modelInfo.modelfileSamplers + ) + // Separate sessionId and the resolved thinking preference from the Ollama request payload — // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw). const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData @@ -219,6 +234,7 @@ export default class OllamaController { numCtx, numPredict, keepAlive, + sampler, signal: abortController.signal, }) for await (const chunk of stream) { @@ -272,6 +288,7 @@ export default class OllamaController { numCtx, numPredict, keepAlive, + sampler, }) } catch (err) { // A backend that never advertised vision support rejecting a request that diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index cfd426dc..930a870a 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -9,6 +9,7 @@ import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import env from '#start/env' import { parseMinRelevance } from '../utils/misc.js' +import { parseResponseStyle } from '../utils/sampler.js' import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js' @inject() @@ -79,6 +80,7 @@ export default class SettingsController { const ragEnabled = await KVStore.getValue('rag.enabled') const contextWindow = await KVStore.getValue('ai.contextWindow') const minRelevance = await KVStore.getValue('rag.minRelevance') + const responseStyle = await KVStore.getValue('ai.responseStyle') // Resolved window per installed model, so the setting shows what "Auto" // actually produced rather than leaving the user to guess. Best-effort: // a model whose metadata can't be read simply doesn't get a badge. @@ -108,6 +110,9 @@ export default class SettingsController { // Sent as the resolved number so the select can match an option // without duplicating the "unset means the default" rule in the UI. minRelevance: parseMinRelevance(minRelevance, RAG_MIN_FINAL_SCORE), + // Resolved rather than raw, for the same reason: unset means 'auto', + // and the select shouldn't have to know that. + responseStyle: parseResponseStyle(responseStyle), }, resolvedContextWindows, }, diff --git a/admin/app/services/eval_generation_service.ts b/admin/app/services/eval_generation_service.ts index fbcb98f9..98db0a28 100644 --- a/admin/app/services/eval_generation_service.ts +++ b/admin/app/services/eval_generation_service.ts @@ -5,7 +5,8 @@ import { inject } from '@adonisjs/core' import logger from '@adonisjs/core/services/logger' import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js' -import type { OllamaChatMessage } from '../../types/ollama.js' +import { resolveSamplerProfile } from '../utils/sampler.js' +import type { OllamaChatMessage, ResponseStyle, SamplerProfile } from '../../types/ollama.js' import type { PipelineOptions, RetrievedChunk } from '../../types/rag.js' import { docIdFromSource } from '../utils/eval/corpus_source.js' import type { Golden } from '../utils/eval/golden_set.js' @@ -48,10 +49,37 @@ export type GenerationMode = 'oracle' | 'e2e' | 'noretrieval' */ export const MOCK_MODEL = 'mock' -/** Fixed sampling for eval runs. Never used by production chat. */ +/** + * Fixed sampling for eval runs. Never used by production chat. + * + * Temperature 0 is the default because a repeatable number is worth more than a + * realistic one for most of what this harness gates. But greedy decoding makes + * top_p, top_k and min_p inert — the highest-probability token wins whatever + * they say — so a run left at 0 cannot see a Response Style change at all. That + * is what `--style` is for: it sets `samplerStyle`, which replaces this + * temperature with the style's and sends the real truncation settings, so the + * chat path can actually be measured. The seed stays fixed either way, which + * makes a sampled run approximately rather than exactly reproducible; `--repeats` + * and the `unstable` counter are how that gets accounted for. + */ export const EVAL_TEMPERATURE = 0 export const EVAL_SEED = 42 +/** + * What a run sends to the decoder: a fixed temperature, a sampler profile, or + * neither. Exactly one of the three, which is why it is a shape rather than two + * loose optional arguments threaded through runCase. + */ +type EvalSampling = { temperature?: number; sampler?: SamplerProfile } + +function resolveEvalSampling(style: ResponseStyle | undefined): EvalSampling { + if (!style) return { temperature: EVAL_TEMPERATURE } + const sampler = resolveSamplerProfile(style) + // `off` resolves to no profile, and sends no temperature either — that is the + // pre-setting behaviour this harness needs to be able to reproduce. + return sampler ? { sampler } : {} +} + export type GenerationRunOptions = { mode: GenerationMode model: string @@ -62,6 +90,23 @@ export type GenerationRunOptions = { minFinalScore?: number /** Skip the history-aware rewrite even on multi-turn goldens. */ skipQueryRewrite?: boolean + /** + * Sample the way real chat does, under this Response Style, instead of at + * temperature 0. + * + * Three states, not two. Unset is the deterministic default and the right + * choice for anything being gated on. A style is the chat path as a user + * experiences it. `off` sends no sampler settings *and* no temperature, which + * is the pre-setting behaviour and therefore the honest control to compare a + * style against: `--style=auto` against `--style=off` measures the change, + * where `--style=auto` against a temperature-0 run measures the change plus + * the difference between greedy and sampled decoding. + * + * Model-declared samplers are deliberately not merged in even under `auto`: + * the harness scores a style, and a number that moved because one machine's + * model shipped a different modelfile is not a number worth committing. + */ + samplerStyle?: ResponseStyle onProgress?: (id: string, index: number, total: number) => void } @@ -129,8 +174,11 @@ export type GenerationRunResult = { mode: GenerationMode model: string repeats: number - temperature: number + /** Null when the run sent no temperature at all, under `--style=off`. */ + temperature: number | null seed: number + /** The Response Style sampled under; null for the deterministic default. */ + samplerStyle: ResponseStyle | null topK?: number scoreThreshold?: number minFinalScore: number @@ -153,6 +201,7 @@ export class EvalGenerationService { const repeats = Math.max(1, options.repeats ?? 3) const started = Date.now() const isMock = options.model === MOCK_MODEL + const sampling = resolveEvalSampling(options.samplerStyle) // Oracle mode needs the corpus text on hand to synthesize perfect context. const corpusText = options.mode === 'oracle' ? await this.loadCorpusText() : null @@ -162,7 +211,7 @@ export class EvalGenerationService { const cases: GenerationCaseResult[] = [] for (const [index, golden] of goldens.entries()) { options.onProgress?.(golden.id, index + 1, goldens.length) - cases.push(await this.runCase(golden, options, repeats, corpusText, isMock)) + cases.push(await this.runCase(golden, options, repeats, corpusText, isMock, sampling)) } return { @@ -170,8 +219,9 @@ export class EvalGenerationService { mode: options.mode, model: options.model, repeats, - temperature: EVAL_TEMPERATURE, + temperature: sampling.sampler?.temperature ?? sampling.temperature ?? null, seed: EVAL_SEED, + samplerStyle: options.samplerStyle ?? null, topK: options.topK, scoreThreshold: options.scoreThreshold, minFinalScore: options.minFinalScore ?? RAG_MIN_FINAL_SCORE, @@ -222,7 +272,8 @@ export class EvalGenerationService { options: GenerationRunOptions, repeats: number, corpusText: Map | null, - isMock: boolean + isMock: boolean, + sampling: EvalSampling ): Promise { const messages: OllamaChatMessage[] = [ ...golden.turns.map((t) => ({ role: t.role, content: t.content })), @@ -290,7 +341,7 @@ export class EvalGenerationService { const generated = isMock ? { answer: mockAnswer(context), usage: undefined } - : await this.generate(options.model, trace.messages, trace.numCtx, trace.numPredict) + : await this.generate(options.model, trace.messages, trace.numCtx, trace.numPredict, sampling) const answer = generated.answer // The backend just reported the prompt's real token count. Comparing it @@ -340,14 +391,19 @@ export class EvalGenerationService { model: string, messages: OllamaChatMessage[], numCtx?: number, - numPredict?: number + numPredict?: number, + sampling: EvalSampling = { temperature: EVAL_TEMPERATURE } ): Promise<{ answer: string; usage?: NomadChatUsage }> { const response = await this.ollamaService.chat({ model, messages, numCtx, numPredict, - temperature: EVAL_TEMPERATURE, + // Spread rather than passed as fields: an explicit temperature outranks a + // profile's inside OllamaService, so under a style it has to be absent + // entirely or the run would carry the style's truncation settings at + // temperature 0 and measure a decoder nobody uses. + ...sampling, seed: EVAL_SEED, }) return { answer: response.message.content.trim(), usage: response.usage } diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 584da008..eeec1298 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -5,7 +5,8 @@ import type { Stream } from 'openai/streaming.js' import { Ollama } from 'ollama' import { ThinkTagSplitter, normalizeNonStreamed } from '../utils/think_stream.js' import { readContextLength, readModelfileNumCtx } from '../utils/context_window.js' -import { NomadOllamaModel } from '../../types/ollama.js' +import { compatSamplerParams, nativeSamplerOptions, readModelfileSamplers } from '../utils/sampler.js' +import { NomadOllamaModel, SamplerProfile } from '../../types/ollama.js' import { EMBEDDING_MODEL_NAME, FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js' import fs from 'node:fs/promises' import path from 'node:path' @@ -49,6 +50,12 @@ export type NomadModelInfo = { quantizationLevel?: string /** num_ctx baked into the modelfile, if the author set one. */ modelfileNumCtx?: number + /** + * Sampler values baked into the modelfile, if the author set any. Under the + * `auto` response style these beat NOMAD's own preset key by key, so a model + * tuned by its author keeps its tuning and only gains the settings it lacked. + */ + modelfileSamplers?: SamplerProfile /** * The raw `model_info` blob. Carries the GGUF architecture keys * (block_count, attention.head_count_kv, embedding_length) that make an exact @@ -130,6 +137,14 @@ type ChatInput = { // (batching and GPU non-determinism still move outputs, hence --repeats). temperature?: number seed?: number + // Resolved sampler settings for this request: top_p, top_k, min_p and + // repeat_penalty. Set from the user's Response Style on the chat path only. + // The ancillary structured calls deliberately leave it unset — they pin + // temperature to 0 and want a grammar honoured, not a style applied. + // + // `temperature` above still wins over `sampler.temperature`, so a caller that + // asks for determinism gets it whatever style the user has chosen. + sampler?: SamplerProfile // JSON Schema (or the string 'json') constraining the decoder, for the structured // ancillary calls — title, suggestion chips, query rewrite. Native-only: it is a // top-level field on /api/chat, not an `options` member, and the OpenAI-compat @@ -569,6 +584,11 @@ export class OllamaService { const options: Record = {} if (chatRequest.numCtx) options.num_ctx = chatRequest.numCtx if (chatRequest.numPredict) options.num_predict = chatRequest.numPredict + // Spread first so an explicit temperature below overrides the style's. + Object.assign(options, nativeSamplerOptions(chatRequest.sampler)) + if (chatRequest.sampler?.temperature !== undefined) { + options.temperature = chatRequest.sampler.temperature + } if (chatRequest.temperature !== undefined) options.temperature = chatRequest.temperature if (chatRequest.seed !== undefined) options.seed = chatRequest.seed return options @@ -605,6 +625,12 @@ export class OllamaService { if (chatRequest.numPredict) { params.max_tokens = chatRequest.numPredict } + // min_p, top_k and repeat_penalty have no equivalent here, so the style + // reaches this transport as temperature plus a nucleus value and nothing else. + Object.assign(params, compatSamplerParams(chatRequest.sampler)) + if (chatRequest.sampler?.temperature !== undefined) { + params.temperature = chatRequest.sampler.temperature + } if (chatRequest.temperature !== undefined) { params.temperature = chatRequest.temperature } @@ -869,6 +895,7 @@ export class OllamaService { parameterSize: data.details?.parameter_size, quantizationLevel: data.details?.quantization_level, modelfileNumCtx: readModelfileNumCtx(data.parameters), + modelfileSamplers: readModelfileSamplers(data.parameters), rawModelInfo: data.model_info, } this.modelInfoCache.set(modelName, info) diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 56371ece..694fe8bb 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -24,6 +24,7 @@ import { isNewerVersion } from '../utils/version.js' import { isUnresolvedGpuModel } from '../utils/gpu_model.js' import { invalidateAssistantNameCache } from '../../config/inertia.js' import { invalidateMinRelevanceCache } from '../utils/rag_relevance.js' +import { invalidateResponseStyleCache } from '../utils/response_style.js' import { KiwixLibraryService } from '#services/kiwix_library_service' @inject() @@ -948,6 +949,9 @@ export class SystemService { if (key === 'rag.minRelevance') { invalidateMinRelevanceCache() } + if (key === 'ai.responseStyle') { + invalidateResponseStyleCache() + } // Re-enabling auto-update after a backoff-driven auto-disable clears the // failure state so it gets a fresh start instead of immediately re-tripping. if (key === 'autoUpdate.enabled' && (value === true || value === 'true')) { diff --git a/admin/app/utils/response_style.ts b/admin/app/utils/response_style.ts new file mode 100644 index 00000000..6cebbfa5 --- /dev/null +++ b/admin/app/utils/response_style.ts @@ -0,0 +1,47 @@ +import logger from '@adonisjs/core/services/logger' +import KVStore from '#models/kv_store' +import { DEFAULT_RESPONSE_STYLE } from '../../constants/ollama.js' +import type { ResponseStyle } from '../../types/ollama.js' +import { parseResponseStyle } from './sampler.js' + +/** + * Resolves the user's Response Style (`ai.responseStyle`), cached. + * + * Same shape and same reasoning as the relevance-floor cache next door: read on + * every chat turn, changed roughly never, and NOMAD's services are `@inject()` + * but not registered as singletons, so caching on a service instance would give + * a cache that lives and dies inside one request. + * + * The TTL bounds how stale this gets if an invalidation is ever missed; + * SystemService.updateSetting calls invalidateResponseStyleCache() so the common + * case takes effect on the next turn rather than a minute later. + */ +const CACHE_TTL_MS = 60_000 + +let cache: { value: ResponseStyle; expiresAt: number } | null = null + +export function invalidateResponseStyleCache(): void { + cache = null +} + +/** The response style to sample this turn under. */ +export async function resolveResponseStyle(): Promise { + const now = Date.now() + if (cache && now < cache.expiresAt) return cache.value + + let raw: string | null = null + try { + raw = await KVStore.getValue('ai.responseStyle') + } catch (error) { + // A KV read failure must not take chat out, and must not be cached — fall + // back to the default and let the next turn try again. + logger.warn( + `[Ollama] Failed to read ai.responseStyle, using ${DEFAULT_RESPONSE_STYLE}: ${error instanceof Error ? error.message : error}` + ) + return DEFAULT_RESPONSE_STYLE + } + + const value = parseResponseStyle(raw) + cache = { value, expiresAt: now + CACHE_TTL_MS } + return value +} diff --git a/admin/app/utils/sampler.ts b/admin/app/utils/sampler.ts new file mode 100644 index 00000000..fc5b6bc6 --- /dev/null +++ b/admin/app/utils/sampler.ts @@ -0,0 +1,133 @@ +/** + * Pure helpers for deciding which sampler settings a chat request carries. + * + * Kept out of OllamaService for the same reason `context_window.ts` is: every + * interesting case here is a bad stored value or a model that declares its own + * parameters, and none of them need Ollama, Docker or MySQL to exercise. The + * service supplies the facts (what the stored setting says, what /api/show + * returned); everything here is a function of its arguments. + */ +import { DEFAULT_RESPONSE_STYLE, SAMPLER_PRESETS } from '../../constants/ollama.js' +import type { ResponseStyle, SamplerProfile } from '../../types/ollama.js' + +const RESPONSE_STYLES: ResponseStyle[] = ['auto', 'focused', 'creative', 'off'] + +/** + * Interpret a stored `ai.responseStyle` value. + * + * Unset, empty and unrecognised all mean the default. A corrupt row must not + * silently drop chat back to unsampled backend defaults, because that is the + * failure this setting exists to fix and it would be invisible from the UI. + * `off` is only ever reached by someone choosing it. + */ +export function parseResponseStyle(raw: string | null | undefined): ResponseStyle { + if (raw === null || raw === undefined) return DEFAULT_RESPONSE_STYLE + const trimmed = String(raw).trim().toLowerCase() + if (!trimmed) return DEFAULT_RESPONSE_STYLE + return (RESPONSE_STYLES as string[]).includes(trimmed) + ? (trimmed as ResponseStyle) + : DEFAULT_RESPONSE_STYLE +} + +/** + * Pull sampler values out of the `parameters` string /api/show returns (the + * modelfile PARAMETER lines, e.g. `temperature 0.6`). + * + * Sibling of `readModelfileNumCtx`, and there for a sharper reason than it: a + * good number of Ollama library models ship author-tuned samplers (qwen3 sets + * 0.6/0.95/20, deepseek-r1 sets 0.6), and stamping a generic house profile over + * those would make exactly the models someone tuned carefully worse. Under the + * `auto` style these values win, key by key. + * + * Repeated PARAMETER lines are legal in a modelfile and the last one wins in + * Ollama, so the scan does not stop at the first match. + */ +export function readModelfileSamplers(parameters: unknown): SamplerProfile { + const profile: SamplerProfile = {} + if (typeof parameters !== 'string') return profile + + const keys: [RegExp, keyof SamplerProfile][] = [ + [/^\s*temperature\s+(-?[\d.]+)\s*$/gm, 'temperature'], + [/^\s*top_p\s+(-?[\d.]+)\s*$/gm, 'topP'], + [/^\s*top_k\s+(-?[\d.]+)\s*$/gm, 'topK'], + [/^\s*min_p\s+(-?[\d.]+)\s*$/gm, 'minP'], + [/^\s*repeat_penalty\s+(-?[\d.]+)\s*$/gm, 'repeatPenalty'], + ] + + for (const [pattern, field] of keys) { + let match: RegExpExecArray | null + while ((match = pattern.exec(parameters)) !== null) { + const value = Number.parseFloat(match[1]) + if (Number.isFinite(value) && value >= 0) profile[field] = value + } + } + return profile +} + +/** + * The sampler settings to send for one request. + * + * `off` sends nothing. `auto` is NOMAD's baseline with the model author's own + * values layered over it, which in practice means the model keeps its tuned + * temperature and top_p and gains the min_p it was missing. `focused` and + * `creative` are explicit user choices and replace the model's values outright, + * because a style the user picked by name that quietly did nothing on half + * their models would be worse than not offering it. + * + * A model that declares its own `top_p` under `auto` also gets it on the compat + * transport: an author-set nucleus is a better substitute for the missing min_p + * than the preset's generic fallback. + */ +export function resolveSamplerProfile( + style: ResponseStyle, + modelfileSamplers?: SamplerProfile +): SamplerProfile | undefined { + if (style === 'off') return undefined + + const preset = SAMPLER_PRESETS[style] + if (style !== 'auto') return { ...preset } + + const declared = modelfileSamplers ?? {} + const merged: SamplerProfile = { ...preset, ...declared } + if (declared.topP !== undefined) merged.compatTopP = declared.topP + return merged +} + +/** + * Map a profile onto Ollama's native `options` keys. + * + * `temperature` is deliberately absent: it is carried on ChatInput itself, so an + * explicit caller value (eval runs and the grammar-constrained task calls, both + * of which pin it to 0) keeps beating whatever style the user has chosen. + */ +export function nativeSamplerOptions(profile: SamplerProfile | undefined): Record { + if (!profile) return {} + const options: Record = {} + if (profile.topP !== undefined) options.top_p = profile.topP + if (profile.topK !== undefined) options.top_k = profile.topK + if (profile.minP !== undefined) options.min_p = profile.minP + if (profile.repeatPenalty !== undefined) options.repeat_penalty = profile.repeatPenalty + return options +} + +/** + * Map a profile onto the OpenAI-compatible endpoint's params. + * + * Only `top_p` survives the trip. min_p, top_k and repeat_penalty have no + * equivalent there, and presence/frequency_penalty are not stand-ins for + * repeat_penalty — they accumulate over the whole completion rather than a + * 64-token window, so borrowing one to fake the other would punish a grounded + * answer for citing the same identifier twice. + * + * `compatTopP` rather than `topP` is the value sent, for the reason documented + * on the type: the presets set `topP: 1.0` and truncate with min_p, and passing + * that through unchanged to a backend that drops min_p would leave the raw + * distribution untruncated, which is worse than the default it replaced. + */ +export function compatSamplerParams(profile: SamplerProfile | undefined): Record { + if (!profile) return {} + const params: Record = {} + const topP = profile.compatTopP ?? profile.topP + if (topP !== undefined) params.top_p = topP + return params +} diff --git a/admin/app/validators/settings.ts b/admin/app/validators/settings.ts index b46a1bb7..96d12600 100644 --- a/admin/app/validators/settings.ts +++ b/admin/app/validators/settings.ts @@ -2,6 +2,7 @@ import vine from "@vinejs/vine"; import { SETTINGS_KEYS } from "../../constants/kv_store.js"; import type { KVStoreKey } from "../../types/kv_store.js"; import { CONTEXT_LADDER } from "../utils/context_window.js"; +import { RESPONSE_STYLE_PRESETS } from "../../constants/ollama.js"; export const getSettingSchema = vine.compile(vine.object({ key: vine.enum(SETTINGS_KEYS), @@ -94,6 +95,19 @@ export function validateSettingValue(key: KVStoreKey, value: unknown): string | } return null } + case 'ai.responseStyle': { + // Empty/unset means 'auto'. Anything else has to be a style we know, + // since an unrecognised value silently resolves back to the default + // and would leave the select showing a choice nothing honours. + if (value === '' || value === undefined || value === null) { + return null + } + const styles = RESPONSE_STYLE_PRESETS.map((preset) => preset.value) + if (typeof value !== 'string' || !styles.includes(value as (typeof styles)[number])) { + return `Response style must be one of: ${styles.join(', ')}.` + } + return null + } case 'ai.keepAlive': { // Ollama duration format: "10m", "1h", "30s", "-1" (forever), "0" (evict). if (value === '' || value === undefined || value === null) { diff --git a/admin/commands/eval/generation.ts b/admin/commands/eval/generation.ts index 7075e5ec..dfb22d47 100644 --- a/admin/commands/eval/generation.ts +++ b/admin/commands/eval/generation.ts @@ -4,8 +4,10 @@ import type { GenerationAggregate, GenerationMode, } from '../../app/services/eval_generation_service.js' +import type { ResponseStyle } from '../../types/ollama.js' const MODES: GenerationMode[] = ['oracle', 'e2e', 'noretrieval'] +const STYLES: ResponseStyle[] = ['auto', 'focused', 'creative', 'off'] /** * Score generated answers against the golden set. @@ -46,6 +48,16 @@ export default class EvalGeneration extends BaseCommand { }) declare suite: string + @flags.string({ + description: + `Sample the way real chat does, under one of: ${STYLES.join(', ')}. ` + + 'Omit for the deterministic temperature-0 default. Greedy decoding makes ' + + 'min_p, top_p and top_k inert, so a Response Style is only visible with ' + + 'this flag set, and "off" (no samplers, no temperature) is the control to ' + + 'compare a style against.', + }) + declare style: string + @flags.string({ description: 'Only run goldens carrying this tag' }) declare tag: string @@ -89,6 +101,13 @@ export default class EvalGeneration extends BaseCommand { } } + if (this.style && !STYLES.includes(this.style as ResponseStyle)) { + this.logger.error(`Unknown style "${this.style}". Expected one of: ${STYLES.join(', ')}`) + this.exitCode = 1 + return + } + const samplerStyle = this.style ? (this.style as ResponseStyle) : undefined + let goldens = await corpusService.loadGoldens(this.suite ? this.suite.split(',').map((s) => s.trim()) : undefined) if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag)) if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10)) @@ -110,8 +129,14 @@ export default class EvalGeneration extends BaseCommand { const fingerprint = await corpusService.fingerprint() const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 3 this.logger.info( - `Corpus ${fingerprint} · ${goldens.length} goldens · model=${this.model} · repeats=${repeats}` + `Corpus ${fingerprint} · ${goldens.length} goldens · model=${this.model} · repeats=${repeats}` + + (samplerStyle ? ` · style=${samplerStyle}` : ' · style=deterministic (temp 0)') ) + if (samplerStyle) { + this.logger.info( + 'Sampling is on: the seed is fixed but the run is only approximately reproducible. Watch the unstable count.' + ) + } if (this.model === MOCK_MODEL) { this.logger.info( 'Mock model: answers are the injected context verbatim — this is the extractive ceiling for the current retrieval, not a real model.' @@ -126,6 +151,7 @@ export default class EvalGeneration extends BaseCommand { mode, model: this.model, repeats, + samplerStyle, onProgress: (id, index, total) => { if (this.verbose) this.logger.info(` [${index}/${total}] ${mode}: ${id}`) }, diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 2a3a1826..47c904c4 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -14,6 +14,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [ 'ai.tasksModel', 'ai.contextWindow', 'ai.keepAlive', + 'ai.responseStyle', 'rag.defaultIngestPolicy', 'rag.enabled', 'rag.minRelevance', diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts index aa1aa092..e35513e5 100644 --- a/admin/constants/ollama.ts +++ b/admin/constants/ollama.ts @@ -1,4 +1,4 @@ -import { NomadOllamaModel } from '../types/ollama.js' +import { NomadOllamaModel, ResponseStyle, SamplerProfile } from '../types/ollama.js' /** * Fallback basic recommended Ollama models in case fetching from the service fails. @@ -223,6 +223,71 @@ export const QUERY_REWRITE_MAX_TOKENS = 160 */ export const DEFAULT_KEEP_ALIVE = '15m' +/** + * Sampler settings per Response Style. + * + * Chat used to send none of these, so every turn ran at whatever the backend + * picked: temperature 0.8, top_p 0.9, top_k 40 and no min_p on a bare Ollama + * model. On a heavily quantized 1-3B model, which is NOMAD's realistic target, + * those defaults are what produces the wandering, repetitive output that reads + * as "the local model is bad". + * + * Three things are worth knowing before retuning these numbers. + * + * min_p is the actual missing lever, not temperature. It is a floor relative to + * the top token's probability, so it truncates hard when the model is confident + * and barely at all when it isn't. top_p can't do that: a fixed nucleus keeps + * the same share of mass either way, which is why a flat distribution at the end + * of a long generation is where small models start to wander. With min_p doing + * the work, top_p is left at 1.0 rather than stacked on top of it, and a higher + * temperature stays safe. + * + * repeat_penalty moves with the style instead of sitting at Ollama's 1.1. The + * penalty applies over the last 64 tokens with no idea what a token means, and + * a grounded answer out of this corpus legitimately repeats a part number, a + * band designator or a drug name inside that window. So the factual style + * relaxes it and the creative style, which is where genuine loops happen, + * tightens it. + * + * `off` is not a preset, it is the absence of one: no options are sent and the + * backend decides, exactly as it did before this setting existed. + */ +export const SAMPLER_PRESETS: Record, SamplerProfile> = { + auto: { temperature: 0.6, topP: 1.0, topK: 40, minP: 0.05, repeatPenalty: 1.1, compatTopP: 0.9 }, + focused: { + temperature: 0.2, + topP: 1.0, + topK: 40, + minP: 0.1, + repeatPenalty: 1.05, + compatTopP: 0.85, + }, + creative: { + temperature: 1.0, + topP: 1.0, + topK: 100, + minP: 0.03, + repeatPenalty: 1.15, + compatTopP: 0.95, + }, +} + +/** Unset `ai.responseStyle` resolves to this; see app/utils/response_style.ts. */ +export const DEFAULT_RESPONSE_STYLE: ResponseStyle = 'auto' + +/** + * The Response Style options offered in Settings > Models, and their labels. + * + * Ordered least to most adventurous, with the default first, matching how the + * Context Window and Knowledge Base Relevance selects above it are ordered. + */ +export const RESPONSE_STYLE_PRESETS: { value: ResponseStyle; label: string }[] = [ + { value: 'auto', label: 'Auto (recommended)' }, + { value: 'focused', label: 'Focused (factual, repeatable)' }, + { value: 'creative', label: 'Creative (varied, exploratory)' }, + { value: 'off', label: 'Backend defaults (send nothing)' }, +] + export const SYSTEM_PROMPTS = { default: ` Format all responses using markdown for better readability. Vanilla markdown or GitHub-flavored markdown is preferred. diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index 09e8c016..1c4258d6 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -12,7 +12,7 @@ import { useModals } from '~/context/ModalContext' import StyledModal from '~/components/StyledModal' import type { NomadInstalledModel } from '../../../types/ollama' import { SERVICE_NAMES } from '../../../constants/service_names' -import { RAG_MIN_RELEVANCE_PRESETS } from '../../../constants/ollama' +import { RAG_MIN_RELEVANCE_PRESETS, RESPONSE_STYLE_PRESETS } from '../../../constants/ollama' import Switch from '~/components/inputs/Switch' import Select from '~/components/inputs/Select' import StyledSectionHeader from '~/components/StyledSectionHeader' @@ -28,7 +28,7 @@ export default function ModelsPage(props: { models: { availableModels: NomadOllamaModel[] installedModels: NomadInstalledModel[] - settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string; minRelevance: number } + settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string; minRelevance: number; responseStyle: string } /** Effective window per installed model, as resolved by ContextWindowService. */ resolvedContextWindows?: Record } @@ -108,6 +108,7 @@ export default function ModelsPage(props: { const [tasksModel, setTasksModel] = useState(props.models.settings.tasksModel) const [contextWindow, setContextWindow] = useState(props.models.settings.contextWindow) const [minRelevance, setMinRelevance] = useState(String(props.models.settings.minRelevance)) + const [responseStyle, setResponseStyle] = useState(props.models.settings.responseStyle) const [aiAssistantCustomName, setAiAssistantCustomName] = useState( props.models.settings.aiAssistantCustomName ) @@ -284,6 +285,11 @@ export default function ModelsPage(props: { : [{ value: minRelevance, label: `Custom (${minRelevance})`, disabled: true }]), ] + const responseStyleOptions = RESPONSE_STYLE_PRESETS.map((preset) => ({ + value: preset.value, + label: preset.label, + })) + const resolvedWindows = props.models.resolvedContextWindows ?? {} const formatWindow = (tokens: number) => tokens >= 1024 ? `${Math.round(tokens / 1024)}K` : String(tokens) @@ -435,6 +441,17 @@ export default function ModelsPage(props: { updateSettingMutation.mutate({ key: 'rag.minRelevance', value: newVal }) }} /> + { + // The whole point of the setting is that the default samples. Falling back to + // "send nothing" would restore the bug and look identical in the UI. + assert.equal(parseResponseStyle(null), 'auto') + assert.equal(parseResponseStyle(undefined), 'auto') + assert.equal(parseResponseStyle(''), 'auto') + assert.equal(parseResponseStyle(' '), 'auto') +}) + +test('a stored style round-trips', () => { + assert.equal(parseResponseStyle('auto'), 'auto') + assert.equal(parseResponseStyle('focused'), 'focused') + assert.equal(parseResponseStyle('creative'), 'creative') + assert.equal(parseResponseStyle('off'), 'off') +}) + +test('case and surrounding whitespace are tolerated', () => { + assert.equal(parseResponseStyle(' Focused '), 'focused') +}) + +test('an unrecognised value falls back to auto rather than disabling sampling', () => { + assert.equal(parseResponseStyle('turbo'), 'auto') + assert.equal(parseResponseStyle('0.7'), 'auto') +}) + +// --- readModelfileSamplers ---------------------------------------------------- + +const QWEN_PARAMETERS = [ + 'repeat_penalty 1', + 'stop "<|im_start|>"', + 'stop "<|im_end|>"', + 'temperature 0.6', + 'top_k 20', + 'top_p 0.95', +].join('\n') + +test('reads the sampler parameters a model author baked in', () => { + assert.deepEqual(readModelfileSamplers(QWEN_PARAMETERS), { + repeatPenalty: 1, + temperature: 0.6, + topK: 20, + topP: 0.95, + }) +}) + +test('a model that declares nothing yields an empty profile', () => { + assert.deepEqual(readModelfileSamplers('num_ctx 8192'), {}) + assert.deepEqual(readModelfileSamplers(undefined), {}) + assert.deepEqual(readModelfileSamplers(null), {}) + assert.deepEqual(readModelfileSamplers(42), {}) +}) + +test('a repeated PARAMETER line takes its last value, as Ollama does', () => { + assert.deepEqual(readModelfileSamplers('temperature 0.2\ntemperature 0.9'), { temperature: 0.9 }) +}) + +test('a non-numeric or negative value is ignored rather than propagated', () => { + assert.deepEqual(readModelfileSamplers('temperature abc\ntop_k -5'), {}) +}) + +test('a key embedded in another key name is not matched', () => { + // `min_p` must not be read out of a hypothetical `xmin_p` line, and the + // trailing-anchor is what stops `top_p 0.9 # comment` scoring as 0.9. + assert.deepEqual(readModelfileSamplers('xtop_p 0.5'), {}) + assert.deepEqual(readModelfileSamplers('top_p 0.9 trailing'), {}) +}) + +// --- resolveSamplerProfile ---------------------------------------------------- + +test('off resolves to no profile at all', () => { + assert.equal(resolveSamplerProfile('off'), undefined) + assert.equal(resolveSamplerProfile('off', { temperature: 0.6 }), undefined) +}) + +test('auto on a model that declares nothing is the baseline preset', () => { + assert.deepEqual(resolveSamplerProfile('auto'), SAMPLER_PRESETS.auto) + assert.deepEqual(resolveSamplerProfile('auto', {}), SAMPLER_PRESETS.auto) +}) + +test('auto lets the model author win key by key, and still supplies min_p', () => { + const resolved = resolveSamplerProfile('auto', readModelfileSamplers(QWEN_PARAMETERS)) + assert.equal(resolved?.temperature, 0.6) + assert.equal(resolved?.topK, 20) + assert.equal(resolved?.topP, 0.95) + assert.equal(resolved?.repeatPenalty, 1) + // The one value no modelfile in the wild sets, and the reason this exists. + assert.equal(resolved?.minP, SAMPLER_PRESETS.auto.minP) +}) + +test("auto sends the author's nucleus on the compat transport too", () => { + // A declared top_p is a better stand-in for the missing min_p than the + // preset's generic fallback. + const resolved = resolveSamplerProfile('auto', { topP: 0.95 }) + assert.equal(resolved?.compatTopP, 0.95) +}) + +test('a style the user picked by name overrides the model, or it would do nothing', () => { + const declared = readModelfileSamplers(QWEN_PARAMETERS) + assert.deepEqual(resolveSamplerProfile('focused', declared), SAMPLER_PRESETS.focused) + assert.deepEqual(resolveSamplerProfile('creative', declared), SAMPLER_PRESETS.creative) +}) + +test('resolving does not mutate the shared preset', () => { + resolveSamplerProfile('auto', { temperature: 0.1, topP: 0.5 }) + assert.equal(SAMPLER_PRESETS.auto.temperature, 0.6) + assert.equal(SAMPLER_PRESETS.auto.compatTopP, 0.9) +}) + +// --- transport mapping -------------------------------------------------------- + +test('the native path carries the truncation settings', () => { + const options = nativeSamplerOptions(SAMPLER_PRESETS.auto) + assert.deepEqual(options, { + top_p: 1, + top_k: 40, + min_p: 0.05, + repeat_penalty: 1.1, + }) +}) + +test('temperature is not a sampler option, so an explicit caller value still wins', () => { + // OllamaService applies chatRequest.temperature after this, and the eval + // harness and the grammar-constrained task calls both depend on that ordering. + assert.equal('temperature' in nativeSamplerOptions(SAMPLER_PRESETS.focused), false) +}) + +test('no profile means no options, which is what off has to produce', () => { + assert.deepEqual(nativeSamplerOptions(undefined), {}) + assert.deepEqual(compatSamplerParams(undefined), {}) +}) + +test('the compat path substitutes a real nucleus for the min_p it cannot send', () => { + // Passing topP straight through would be top_p 1.0 with no min_p behind it: + // the raw distribution, which is worse than the 0.9 default it replaced. + assert.deepEqual(compatSamplerParams(SAMPLER_PRESETS.auto), { top_p: 0.9 }) + assert.deepEqual(compatSamplerParams(SAMPLER_PRESETS.focused), { top_p: 0.85 }) + assert.deepEqual(compatSamplerParams(SAMPLER_PRESETS.creative), { top_p: 0.95 }) +}) + +test('the compat path sends nothing it has no equivalent for', () => { + const params = compatSamplerParams(SAMPLER_PRESETS.auto) + assert.deepEqual(Object.keys(params), ['top_p']) +}) + +test('a profile with no compat fallback falls back to its own top_p', () => { + assert.deepEqual(compatSamplerParams({ topP: 0.8 }), { top_p: 0.8 }) +}) + +test('every preset supplies min_p and a compat nucleus below 1', () => { + // Two invariants worth failing a retune over: the preset that forgets min_p + // is the bug this setting exists to fix, and a compat nucleus of 1 is the + // untruncated distribution. + for (const [name, preset] of Object.entries(SAMPLER_PRESETS)) { + assert.ok(preset.minP !== undefined && preset.minP > 0, `${name} is missing min_p`) + assert.ok( + preset.compatTopP !== undefined && preset.compatTopP < 1, + `${name} would sample the raw distribution on a compat backend` + ) + } +}) diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index b6fed876..8207ba3b 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -66,6 +66,11 @@ export const KV_STORE_SCHEMA = { // How long Ollama keeps a chat model (and its KV cache) resident after a // request, in Ollama's duration format ("10m"). Unset inherits Ollama's 5m. 'ai.keepAlive': 'string', + // Sampler profile for chat replies: 'auto' | 'focused' | 'creative' | 'off'. + // Unset resolves to 'auto', which is NOMAD's baseline with any sampler values + // the model's author baked into their modelfile layered over it. 'off' sends + // no sampler settings at all, the behaviour from before the setting existed. + 'ai.responseStyle': 'string', 'ai.amdGpuAcceleration': 'boolean', 'ai.amdHsaOverride': 'string', 'ai.autoFixGpuPassthrough': 'boolean', diff --git a/admin/types/ollama.ts b/admin/types/ollama.ts index 20b90aa3..66fc03e1 100644 --- a/admin/types/ollama.ts +++ b/admin/types/ollama.ts @@ -67,3 +67,46 @@ export type NomadChatResponse = { done: boolean model: string } + +/** + * The four sampler profiles offered as "Response Style" in Settings > Models. + * + * `auto` is the default and the only one that defers to the model: a value the + * model author baked into their modelfile wins over NOMAD's baseline, key by + * key. `focused` and `creative` are deliberate user overrides and replace those + * values wholesale. `off` sends no sampler settings at all, reproducing the + * pre-setting behaviour and giving a bug report somewhere clean to start. + */ +export type ResponseStyle = 'auto' | 'focused' | 'creative' | 'off' + +/** + * Resolved sampler settings for one chat request. + * + * Every field is optional because "unset" is meaningful: it means the backend + * (or the model's own modelfile) decides, which is exactly what `off` wants and + * what a missing key under `auto` falls back to. + */ +export type SamplerProfile = { + temperature?: number + /** + * Nucleus cut for the native transport. The presets leave this at 1.0 and let + * min_p do the truncating: min_p is relative to the top token's probability, + * so it tightens on a confident distribution and loosens on an uncertain one, + * where a fixed nucleus does neither. Stacking both mostly means the weaker + * cut never fires. + */ + topP?: number + topK?: number + /** The lever this whole setting exists for. Ollama's default is 0 (disabled). */ + minP?: number + repeatPenalty?: number + /** + * `top_p` to send on the OpenAI-compatible transport, which has no min_p. + * + * Without this the compat path would inherit `topP: 1.0` from a profile whose + * truncation lives entirely in min_p, and end up sampling the raw distribution + * — strictly worse than the 0.9 Ollama's /v1 endpoint defaults to. So the + * compat path substitutes a real nucleus value instead of dropping to nothing. + */ + compatTopP?: number +}