diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index f3e19a17..cfd426dc 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -8,6 +8,8 @@ import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#va import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import env from '#start/env' +import { parseMinRelevance } from '../utils/misc.js' +import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js' @inject() export default class SettingsController { @@ -76,6 +78,7 @@ export default class SettingsController { const tasksModel = await KVStore.getValue('ai.tasksModel') const ragEnabled = await KVStore.getValue('rag.enabled') const contextWindow = await KVStore.getValue('ai.contextWindow') + const minRelevance = await KVStore.getValue('rag.minRelevance') // Resolved window per installed model, so the setting shows what "Auto" // actually produced rather than leaving the user to guess. Best-effort: // a model whose metadata can't be read simply doesn't get a badge. @@ -102,6 +105,9 @@ export default class SettingsController { tasksModel: tasksModel ?? '', ragEnabled: ragEnabled ?? true, contextWindow: contextWindow ?? 'auto', + // Sent as the resolved number so the select can match an option + // without duplicating the "unset means the default" rule in the UI. + minRelevance: parseMinRelevance(minRelevance, RAG_MIN_FINAL_SCORE), }, resolvedContextWindows, }, diff --git a/admin/app/services/eval_generation_service.ts b/admin/app/services/eval_generation_service.ts index 560657a3..fbcb98f9 100644 --- a/admin/app/services/eval_generation_service.ts +++ b/admin/app/services/eval_generation_service.ts @@ -4,6 +4,7 @@ import { RagPipelineService } from '#services/rag_pipeline_service' import { inject } from '@adonisjs/core' import logger from '@adonisjs/core/services/logger' import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' +import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js' import type { OllamaChatMessage } from '../../types/ollama.js' import type { PipelineOptions, RetrievedChunk } from '../../types/rag.js' import { docIdFromSource } from '../utils/eval/corpus_source.js' @@ -57,6 +58,8 @@ export type GenerationRunOptions = { repeats?: number topK?: number scoreThreshold?: number + /** Post-rerank relevance floor. Defaults to the constant, never to the setting. */ + minFinalScore?: number /** Skip the history-aware rewrite even on multi-turn goldens. */ skipQueryRewrite?: boolean onProgress?: (id: string, index: number, total: number) => void @@ -130,6 +133,7 @@ export type GenerationRunResult = { seed: number topK?: number scoreThreshold?: number + minFinalScore: number } overall: GenerationAggregate byTag: Record @@ -170,6 +174,7 @@ export class EvalGenerationService { seed: EVAL_SEED, topK: options.topK, scoreThreshold: options.scoreThreshold, + minFinalScore: options.minFinalScore ?? RAG_MIN_FINAL_SCORE, }, overall: aggregateGeneration(cases), byTag: aggregateGenerationByTag(cases), @@ -228,6 +233,11 @@ export class EvalGenerationService { collection: KB_EVAL_COLLECTION, topK: options.topK, scoreThreshold: options.scoreThreshold, + // Pinned to the constant, not left to fall through to resolveMinFinalScore() + // — otherwise a `rag.minRelevance` slider set on one developer's machine + // would silently move every generation score. Same reasoning as the + // retrieval tier; see EvalRetrievalService.run. + minFinalScore: options.minFinalScore ?? RAG_MIN_FINAL_SCORE, // The mock run must not touch Ollama at all — that is what makes it // usable with no models installed. The rewrite is a chat-model call, so // it is always skipped there (it would 404 and silently fall back, which diff --git a/admin/app/services/eval_retrieval_service.ts b/admin/app/services/eval_retrieval_service.ts index e3a86859..112026e4 100644 --- a/admin/app/services/eval_retrieval_service.ts +++ b/admin/app/services/eval_retrieval_service.ts @@ -2,7 +2,11 @@ import { EvalCorpusService } from '#services/eval_corpus_service' import { RagService } from '#services/rag_service' import { inject } from '@adonisjs/core' import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' -import { RAG_DEFAULT_SCORE_THRESHOLD, RAG_DEFAULT_TOP_K } from '../../constants/ollama.js' +import { + RAG_DEFAULT_SCORE_THRESHOLD, + RAG_DEFAULT_TOP_K, + RAG_MIN_FINAL_SCORE, +} from '../../constants/ollama.js' import type { RetrievalStages } from '../../types/rag.js' import { docIdFromSource } from '../utils/eval/corpus_source.js' import type { Golden } from '../utils/eval/golden_set.js' @@ -20,6 +24,8 @@ import { export type RetrievalRunOptions = { topK?: number scoreThreshold?: number + /** Post-rerank relevance floor. Defaults to the constant, never to the setting. */ + minFinalScore?: number kValues?: number[] /** Score the raw dense / reranked / diversified orderings separately. */ ablate?: boolean @@ -32,7 +38,7 @@ export type StageAblation = { } export type RetrievalRunResult = { - params: { topK: number; scoreThreshold: number; kValues: number[] } + params: { topK: number; scoreThreshold: number; minFinalScore: number; kValues: number[] } overall: RetrievalAggregate byTag: Record cases: RetrievalCaseResult[] @@ -69,6 +75,12 @@ export class EvalRetrievalService { async run(goldens: Golden[], options: RetrievalRunOptions = {}): Promise { const topK = options.topK ?? RAG_DEFAULT_TOP_K const scoreThreshold = options.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD + // RAG_MIN_FINAL_SCORE, deliberately — NOT resolveMinFinalScore(). The chat + // path reads the user's `rag.minRelevance` setting; this tier must not, or a + // slider position on one developer's machine would silently move the numbers + // and the committed baseline would stop being reproducible anywhere else. + // Same reasoning as the harness omitting `skipRetrieval`. + const minFinalScore = options.minFinalScore ?? RAG_MIN_FINAL_SCORE const kValues = options.kValues ?? DEFAULT_K_VALUES const cases: RetrievalCase[] = [] @@ -84,7 +96,8 @@ export class EvalRetrievalService { topK, scoreThreshold, KB_EVAL_COLLECTION, - options.ablate ? stages : undefined + options.ablate ? stages : undefined, + minFinalScore ) const retrieved: ScoredChunk[] = docs.map((d) => { @@ -108,7 +121,7 @@ export class EvalRetrievalService { aggregate(stageCases, stageCases.map((c) => scoreCase(c, kValues)), kValues) return { - params: { topK, scoreThreshold, kValues }, + params: { topK, scoreThreshold, minFinalScore, kValues }, overall: aggregate(cases, results, kValues), byTag: aggregateByTag(cases, results, kValues), cases: results, diff --git a/admin/app/services/rag_pipeline_service.ts b/admin/app/services/rag_pipeline_service.ts index 6707b44a..61ee1b8a 100644 --- a/admin/app/services/rag_pipeline_service.ts +++ b/admin/app/services/rag_pipeline_service.ts @@ -14,9 +14,10 @@ import { SYSTEM_PROMPTS, } from '../../constants/ollama.js' import type { OllamaChatMessage } from '../../types/ollama.js' -import type { PipelineOptions, PipelineTrace, RetrievedChunk } from '../../types/rag.js' +import type { PipelineOptions, PipelineTrace, RetrievalFloorStats, RetrievedChunk } from '../../types/rag.js' import { planPrompt } from '../utils/context_budget.js' import { estimateMessagesTokens } from '../utils/token_estimate.js' +import { resolveMinFinalScore } from '../utils/rag_relevance.js' import { resolveTasksModel } from '../utils/tasks_model.js' import { buildContextBlock, getContextLimitsForModel } from '../utils/rag_prompt.js' import { QUERIES_SCHEMA, pickQueries, resolveStructured } from '../utils/structured_output.js' @@ -104,6 +105,8 @@ export class RagPipelineService { numPredict: undefined, contextLimits: { maxResults: RAG_DEFAULT_TOP_K, maxTokens: 0 }, timings: { rewriteMs: 0, retrievalMs: 0 }, + minFinalScore: 0, + chunksBelowFloor: 0, } // --- Retrieval ------------------------------------------------------- @@ -131,16 +134,28 @@ export class RagPipelineService { if (retrievalQuery) { const retrievalStart = Date.now() + // An explicit option always wins, so the eval harness never inherits + // whatever this machine's `rag.minRelevance` slider happens to be set to. + const minFinalScore = opts.minFinalScore ?? (await resolveMinFinalScore()) + const floor: RetrievalFloorStats = { candidates: 0, belowFloor: 0 } relevantDocs = await this.ragService.searchSimilarDocuments( retrievalQuery, opts.topK ?? RAG_DEFAULT_TOP_K, opts.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD, - opts.collection + opts.collection, + undefined, + minFinalScore, + floor ) trace.timings.retrievalMs = Date.now() - retrievalStart trace.retrieved = relevantDocs + trace.minFinalScore = minFinalScore + trace.chunksBelowFloor = floor.belowFloor logger.debug( - `[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${retrievalQuery}"` + `[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${retrievalQuery}"` + + (floor.belowFloor > 0 + ? ` (${floor.belowFloor} of ${floor.candidates} dropped below the ${minFinalScore} relevance floor)` + : '') ) } } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 858b58a1..cb1fd228 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -22,7 +22,8 @@ import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision import { decideContentReindex, type ReindexOutcome } from '../utils/content_reindex_decision.js' import KbRatioRegistry from '#models/kb_ratio_registry' import { decideWarnings } from '../utils/kb_warning_decision.js' -import type { FileWarning, FileWarningsResult, RetrievalStages, StoredFileInfo } from '../../types/rag.js' +import type { FileWarning, FileWarningsResult, RetrievalFloorStats, RetrievalStages, StoredFileInfo } from '../../types/rag.js' +import { applyRelevanceFloor } from '../utils/misc.js' import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' /** @@ -849,6 +850,7 @@ export class RagService { * @param query - The search query text * @param limit - Maximum number of results to return (default: 5) * @param scoreThreshold - Minimum similarity score threshold (default: 0.3, much lower than before) + * @param minFinalScore - Post-rerank relevance floor; below it a chunk is dropped * @returns Array of relevant text chunks with their scores */ public async searchSimilarDocuments( @@ -863,7 +865,15 @@ export class RagService { * changes — and it is what lets the eval harness answer "is the reranking * heuristic actually helping?" without a second implementation of search. */ - stagesOut?: RetrievalStages + stagesOut?: RetrievalStages, + /** + * Relevance floor on the post-rerank score. Defaults to 0 so the parameter + * is purely additive for callers that predate it; the chat pipeline and the + * eval harness both pass a real value. + */ + minFinalScore: number = 0, + /** Optional sink for the floor's counts; see RetrievalFloorStats. */ + floorOut?: RetrievalFloorStats ): Promise }>> { try { logger.debug(`[RAG] Starting similarity search for query: "${query}"`) @@ -967,12 +977,44 @@ export class RagService { ) }) + // Relevance floor, applied here — after reranking, before diversity. + // + // The ordering is deliberate. The floor is a judgement about relevance, + // and the reranked score is where that judgement is best informed. The + // diversity penalty that follows is about redundancy, not relevance: it + // multiplies by 0.85^n, so flooring afterwards would drop the fourth chunk + // of the one document that actually answers the question and blame a knob + // labelled "relevance" for it. + const { survivors, belowFloor } = applyRelevanceFloor(rerankedResults, minFinalScore) + if (floorOut) { + floorOut.candidates = rerankedResults.length + floorOut.belowFloor = belowFloor + } + if (belowFloor > 0) { + logger.debug( + `[RAG] Relevance floor ${minFinalScore}: dropped ${belowFloor} of ${rerankedResults.length} chunk(s)` + ) + } + if (survivors.length === 0 && rerankedResults.length > 0) { + // Nothing cleared the bar. Returning empty is the point: the pipeline + // then injects no context block at all, rather than handing the model + // passages it has to be talked out of using. + logger.debug( + `[RAG] Nothing cleared the relevance floor (best was ${rerankedResults[0].finalScore.toFixed(4)}) — injecting no context` + ) + } + // Apply source diversity penalty to avoid all results from the same document - const diverseResults = this.applySourceDiversity(rerankedResults) + const diverseResults = this.applySourceDiversity(survivors) // Record the three ranked lists for the eval harness's stage ablation. // Sliced to `limit` so each stage is compared on the window that would // actually have been injected, not on the wider rerank candidate pool. + // + // `reranked` is recorded pre-floor on purpose: the ablation asks whether + // each heuristic *orders* better, and a filter applied to two of the three + // lists would answer a different question. The floor's own effect is + // measured by sweeping --min-final-score against the headline metrics. if (stagesOut) { const summarize = (rows: Array<{ text: string; score: number; source?: string }>) => rows.slice(0, limit).map((r) => ({ source: r.source, score: r.score })) diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 47a950fd..d9ab50d7 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -22,6 +22,7 @@ import KVStore from '#models/kv_store' import { KV_STORE_SCHEMA, KVStoreKey } from '../../types/kv_store.js' import { isNewerVersion } from '../utils/version.js' import { invalidateAssistantNameCache } from '../../config/inertia.js' +import { invalidateMinRelevanceCache } from '../utils/rag_relevance.js' import { KiwixLibraryService } from '#services/kiwix_library_service' @inject() @@ -920,6 +921,9 @@ export class SystemService { if (key === 'ai.assistantCustomName') { invalidateAssistantNameCache() } + if (key === 'rag.minRelevance') { + invalidateMinRelevanceCache() + } // Re-enabling auto-update after a backoff-driven auto-disable clears the // failure state so it gets a fresh start instead of immediately re-tripping. if (key === 'autoUpdate.enabled' && (value === true || value === 'true')) { diff --git a/admin/app/utils/eval/retrieval_metrics.ts b/admin/app/utils/eval/retrieval_metrics.ts index 32a57173..29f5d59f 100644 --- a/admin/app/utils/eval/retrieval_metrics.ts +++ b/admin/app/utils/eval/retrieval_metrics.ts @@ -159,6 +159,20 @@ export function ndcgAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: n * intuition. If the relevant p10 sits below the irrelevant p90, no threshold * can cleanly separate them and the honest conclusion is that the *retriever* * needs work, not the cutoff. + * + * ## Two axes, because there are two cutoffs + * + * A chunk carries two scores and they are not interchangeable: + * + * - the **semantic** score is raw cosine, and it is what Qdrant's + * `score_threshold` filters on — before reranking ever runs; + * - the **final** score is what reranking and source diversity produced, and it + * is what `RAG_MIN_FINAL_SCORE` filters on. + * + * Reranking's boosts are score-scaled and additive (up to ~1.275x), so the two + * distributions sit at different heights. Calibrating one cutoff against the + * other axis' percentiles is off by exactly that factor, which is why both are + * reported separately rather than collapsed into one number. */ export type ScoreDistribution = { count: number @@ -234,8 +248,12 @@ export type RetrievalAggregate = { * confident, wrong reply. */ nonEmptyRateOnRefusal: number | null + /** Semantic (raw cosine) scores — the axis Qdrant's score_threshold filters on. */ relevantScores: ScoreDistribution | null irrelevantScores: ScoreDistribution | null + /** Post-rerank scores — the axis RAG_MIN_FINAL_SCORE filters on. */ + relevantFinalScores: ScoreDistribution | null + irrelevantFinalScores: ScoreDistribution | null } export const DEFAULT_K_VALUES = [1, 3, 5, 10] @@ -277,11 +295,14 @@ export function aggregate( // threshold needs to exclude. const relevantScores: number[] = [] const irrelevantScores: number[] = [] + const relevantFinalScores: number[] = [] + const irrelevantFinalScores: number[] = [] for (const c of cases) { const relevant = new Set(c.relevantDocIds) for (const chunk of c.retrieved) { - const bucket = chunk.docId && relevant.has(chunk.docId) ? relevantScores : irrelevantScores - bucket.push(chunk.semanticScore ?? chunk.score) + const isRelevant = Boolean(chunk.docId && relevant.has(chunk.docId)) + ;(isRelevant ? relevantScores : irrelevantScores).push(chunk.semanticScore ?? chunk.score) + ;(isRelevant ? relevantFinalScores : irrelevantFinalScores).push(chunk.score) } } @@ -299,6 +320,8 @@ export function aggregate( refusals.length === 0 ? null : refusals.filter((r) => !r.empty).length / refusals.length, relevantScores: describeScores(relevantScores), irrelevantScores: describeScores(irrelevantScores), + relevantFinalScores: describeScores(relevantFinalScores), + irrelevantFinalScores: describeScores(irrelevantFinalScores), } } diff --git a/admin/app/utils/misc.ts b/admin/app/utils/misc.ts index 46faca23..2e32dba2 100644 --- a/admin/app/utils/misc.ts +++ b/admin/app/utils/misc.ts @@ -46,4 +46,48 @@ export function parseBoolean(value: any): boolean { return value === 1 } return false -} \ No newline at end of file +} + +/** + * Interpret a stored `rag.minRelevance` value as a retrieval relevance floor. + * + * Pure, and separate from the cached KV read in rag_relevance.ts, for the same + * reason pickTasksModel is separate from resolveTasksModel: every interesting + * case here is a bad or absent value, and none of them need a database. + * + * Unset, empty, 'auto' and unparseable all mean "use the recommended default" — + * a corrupt row must not silently disable filtering. An explicit 0 is different: + * it is a real choice, and it turns the floor off. + */ +export function parseMinRelevance(raw: string | null | undefined, fallback: number): number { + if (raw === null || raw === undefined) return fallback + const trimmed = String(raw).trim() + if (trimmed === '' || trimmed === 'auto') return fallback + const parsed = Number(trimmed) + if (!Number.isFinite(parsed)) return fallback + return Math.min(1, Math.max(0, parsed)) +} + +/** + * Apply the retrieval relevance floor to a reranked candidate list. + * + * Pure, and called from RagService.searchSimilarDocuments between reranking and + * the source-diversity penalty. The ordering is the interesting part: the floor + * is a judgement about *relevance*, and the reranked score is where that + * judgement is best informed. Diversity is about *redundancy* — it multiplies by + * 0.85^n per repeated source — so flooring afterwards would drop the fourth + * chunk of the one document that actually answers the question and blame a knob + * labelled "relevance" for it. + * + * An empty result is a real answer, not a failure: it means nothing retrieved + * was relevant enough, and the caller should inject no context block at all + * rather than hand the model passages it has to be talked out of using. + */ +export function applyRelevanceFloor( + results: T[], + minFinalScore: number +): { survivors: T[]; belowFloor: number } { + if (!(minFinalScore > 0)) return { survivors: results, belowFloor: 0 } + const survivors = results.filter((r) => r.finalScore >= minFinalScore) + return { survivors, belowFloor: results.length - survivors.length } +} diff --git a/admin/app/utils/rag_relevance.ts b/admin/app/utils/rag_relevance.ts new file mode 100644 index 00000000..ad025f7c --- /dev/null +++ b/admin/app/utils/rag_relevance.ts @@ -0,0 +1,49 @@ +import logger from '@adonisjs/core/services/logger' +import KVStore from '#models/kv_store' +import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js' +import { parseMinRelevance } from './misc.js' + +/** + * Resolves the user's retrieval relevance floor (`rag.minRelevance`), cached. + * + * Read on every chat turn, changed roughly never, so a KV round-trip per turn + * would be pure overhead on the critical path. Cached at module scope rather + * than on a service instance: NOMAD's services are `@inject()` but are not + * registered as singletons, so instance state lives and dies with a single + * request — which is why ContextWindowService's "memoized for the process + * lifetime" memo isn't, and why its invalidate() never had a caller to matter to. + * + * Belt and braces, mirroring the assistant-name cache in config/inertia.ts: the + * TTL bounds how stale this can get if an invalidation is ever missed, and + * SystemService.updateSetting calls invalidateMinRelevanceCache() so the common + * case takes effect immediately. + */ +const CACHE_TTL_MS = 60_000 + +let cache: { value: number; expiresAt: number } | null = null + +export function invalidateMinRelevanceCache(): void { + cache = null +} + +/** The relevance floor to apply to this turn's retrieval. */ +export async function resolveMinFinalScore(): Promise { + const now = Date.now() + if (cache && now < cache.expiresAt) return cache.value + + let raw: string | null = null + try { + raw = await KVStore.getValue('rag.minRelevance') + } catch (error) { + // A KV read failure must not take retrieval out, and must not be cached — + // fall back to the default and let the next turn try again. + logger.warn( + `[RAG] Failed to read rag.minRelevance, using ${RAG_MIN_FINAL_SCORE}: ${error instanceof Error ? error.message : error}` + ) + return RAG_MIN_FINAL_SCORE + } + + const value = parseMinRelevance(raw, RAG_MIN_FINAL_SCORE) + cache = { value, expiresAt: now + CACHE_TTL_MS } + return value +} diff --git a/admin/app/validators/settings.ts b/admin/app/validators/settings.ts index 8c28583e..b46a1bb7 100644 --- a/admin/app/validators/settings.ts +++ b/admin/app/validators/settings.ts @@ -77,6 +77,23 @@ export function validateSettingValue(key: KVStoreKey, value: unknown): string | } return null } + case 'rag.minRelevance': { + // Empty/'auto' clears the setting and reverts to RAG_MIN_FINAL_SCORE. + // An explicit 0 is a real choice — it turns the floor off — so it is + // deliberately not treated as "unset". + // Trimmed first so the accepted set matches parseMinRelevance's: + // Number(' ') is 0, which would otherwise validate as "floor off" + // for a value the parser reads as unset. + const raw = typeof value === 'string' ? value.trim() : value + if (raw === '' || raw === 'auto' || raw === undefined || raw === null) { + return null + } + const num = Number(raw) + if (!Number.isFinite(num) || num < 0 || num > 1) { + return 'Relevance threshold must be a number between 0 and 1, or empty to use the recommended default.' + } + return null + } case 'ai.keepAlive': { // Ollama duration format: "10m", "1h", "30s", "-1" (forever), "0" (evict). if (value === '' || value === undefined || value === null) { diff --git a/admin/commands/eval/retrieval.ts b/admin/commands/eval/retrieval.ts index fa6202f5..60307eb1 100644 --- a/admin/commands/eval/retrieval.ts +++ b/admin/commands/eval/retrieval.ts @@ -11,9 +11,10 @@ import type { RetrievalAggregate } from '../../app/utils/eval/retrieval_metrics. * thresholds, or reranking. * * node ace eval:retrieval - * node ace eval:retrieval --ablate # is the reranker helping? - * node ace eval:retrieval --threshold=0.5 # sweep the cutoff - * node ace eval:retrieval --tag=multi-hop # one slice only + * node ace eval:retrieval --ablate # is the reranker helping? + * node ace eval:retrieval --threshold=0.5 # sweep the Qdrant cutoff + * node ace eval:retrieval --min-final-score=0.6 # sweep the post-rerank floor + * node ace eval:retrieval --tag=multi-hop # one slice only */ export default class EvalRetrieval extends BaseCommand { static commandName = 'eval:retrieval' @@ -25,6 +26,13 @@ export default class EvalRetrieval extends BaseCommand { @flags.string({ description: 'Minimum similarity score (default: the production value)' }) declare threshold: string + @flags.string({ + description: + 'Post-rerank relevance floor; chunks below it are dropped (default: the production value). ' + + 'Reads RAG_MIN_FINAL_SCORE, never the rag.minRelevance setting — see EvalRetrievalService.', + }) + declare minFinalScore: string + @flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' }) declare ablate: boolean @@ -83,12 +91,14 @@ export default class EvalRetrieval extends BaseCommand { const result = await retrievalService.run(goldens, { topK: this.topK ? Number.parseInt(this.topK, 10) : undefined, scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined, + minFinalScore: this.minFinalScore ? Number.parseFloat(this.minFinalScore) : undefined, ablate: this.ablate, }) const elapsed = ((Date.now() - started) / 1000).toFixed(1) this.logger.info( - `Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} (${elapsed}s)` + `Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} ` + + `minFinalScore=${result.params.minFinalScore} (${elapsed}s)` ) this.logger.info('') this.printAggregate('OVERALL', result.overall, result.params.kValues) @@ -181,36 +191,25 @@ export default class EvalRetrieval extends BaseCommand { * Turn the score distributions into an actual recommendation. The raw * percentiles are the evidence; this is the reading of them, which is what * the threshold constants have never had. + * + * Printed per axis, because the two cutoffs filter different numbers: + * `--threshold` is applied by Qdrant to the raw cosine score, and + * `--min-final-score` is applied after reranking. Reading one off the other's + * percentiles is wrong by the reranker's boost factor. */ private printThresholdGuidance(agg: RetrievalAggregate) { - const rel = agg.relevantScores - const irr = agg.irrelevantScores this.logger.info('') - this.logger.info('=== Score separation (how to calibrate the threshold) ===') - if (rel) { - this.logger.info( - ` relevant chunks n=${rel.count} min=${rel.min.toFixed(3)} p10=${rel.p10.toFixed(3)} median=${rel.median.toFixed(3)} p90=${rel.p90.toFixed(3)}` - ) - } - if (irr) { - this.logger.info( - ` irrelevant chunks n=${irr.count} min=${irr.min.toFixed(3)} p10=${irr.p10.toFixed(3)} median=${irr.median.toFixed(3)} p90=${irr.p90.toFixed(3)}` - ) - } - if (rel && irr) { - if (rel.p10 > irr.p90) { - this.logger.success( - ` Clean separation: a threshold between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.` - ) - } else { - this.logger.warning( - ` Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}).` - ) - this.logger.warning( - ' No cutoff separates these cleanly — the retriever needs work, not the threshold.' - ) - } - } + this.logger.info('=== Score separation (how to calibrate the cutoffs) ===') + this.printAxis( + 'semantic, pre-rerank — calibrates --threshold', + agg.relevantScores, + agg.irrelevantScores + ) + this.printAxis( + 'final, post-rerank — calibrates --min-final-score', + agg.relevantFinalScores, + agg.irrelevantFinalScores + ) if (agg.emptyRateOnAnswerable !== null && agg.emptyRateOnAnswerable > 0) { this.logger.warning( ` ${pct(agg.emptyRateOnAnswerable)} of answerable questions retrieved nothing — threshold may be too high.` @@ -223,6 +222,34 @@ export default class EvalRetrieval extends BaseCommand { } } + /** One axis of the separation table, plus the reading of it. */ + private printAxis( + label: string, + rel: RetrievalAggregate['relevantScores'], + irr: RetrievalAggregate['irrelevantScores'] + ) { + if (!rel && !irr) return + this.logger.info(` ${label}`) + const row = (name: string, d: NonNullable) => + this.logger.info( + ` ${name.padEnd(18)} n=${d.count} min=${d.min.toFixed(3)} p10=${d.p10.toFixed(3)} median=${d.median.toFixed(3)} p90=${d.p90.toFixed(3)}` + ) + if (rel) row('relevant chunks', rel) + if (irr) row('irrelevant chunks', irr) + if (rel && irr) { + if (rel.p10 > irr.p90) { + this.logger.success( + ` Clean separation: a cutoff between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.` + ) + } else { + this.logger.warning( + ` Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}); ` + + `a cutoff near ${rel.min.toFixed(3)} keeps every relevant chunk seen here.` + ) + } + } + } + private explainAblation(ablation: { dense: RetrievalAggregate; reranked: RetrievalAggregate; diversified: RetrievalAggregate }, k: number) { const verdict = (name: string, before: number | null, after: number | null) => { if (before === null || after === null) return @@ -253,7 +280,9 @@ function renderRetrievalMarkdown(doc: any, result: any, misses: any[]): string { lines.push(`- corpus: \`${doc.meta.corpusFingerprint}\``) lines.push(`- commit: \`${doc.meta.gitSha ?? 'unknown'}\`${doc.meta.gitDirty ? ' (dirty tree)' : ''}`) lines.push(`- when: ${doc.meta.createdAt}`) - lines.push(`- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold}`) + lines.push( + `- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} minFinalScore=${result.params.minFinalScore}` + ) lines.push('') lines.push('## Metrics', '') lines.push('| metric | value |', '|---|---:|') diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 625a6341..2a3a1826 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -16,6 +16,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [ 'ai.keepAlive', 'rag.defaultIngestPolicy', 'rag.enabled', + 'rag.minRelevance', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts index e655e91c..aa1aa092 100644 --- a/admin/constants/ollama.ts +++ b/admin/constants/ollama.ts @@ -100,6 +100,73 @@ export const RAG_CONTEXT_LIMITS: { maxParams: number; maxResults: number; maxTok export const RAG_DEFAULT_TOP_K = 5 export const RAG_DEFAULT_SCORE_THRESHOLD = 0.3 +/** + * Relevance floor applied to the post-rerank score. Chunks below it are + * dropped, and when nothing clears it no context block is injected at all. + * + * Retrieval used to have exactly one cutoff — RAG_DEFAULT_SCORE_THRESHOLD, + * applied by Qdrant to the raw cosine score. nomic scores unrelated text well + * above 0.3, so every question retrieved something and the system prompt was + * left to compensate in prose: "silently judge whether the context genuinely + * addresses the question". That is a relevance classification, handed to a 3B + * model, that a number should have decided. + * + * RAG_DEFAULT_SCORE_THRESHOLD deliberately stays low. It is the candidate net, + * and pruning it would starve the reranker (worth +0.0125 ndcg@5 on the golden + * set) and, later, the sparse leg of hybrid retrieval. This is the decisive + * cutoff instead, applied once at the end where the score means the most. + * + * Calibrated with `node ace eval:retrieval --min-final-score=` against the + * 99-golden set at corpus fingerprint 51e642964facf251: + * + * floor recall@5 precision@5 emptyOnAnswerable nonEmptyOnRefusal + * 0.00 0.991 0.221 0% 100% + * 0.55 0.991 0.289 0% 80% + * 0.62 0.991 0.524 0% 60% + * 0.65 0.991 0.659 0% 60% + * 0.66 0.991 0.695 0% 60% + * 0.67 0.972 0.704 1% 60% + * 0.72 0.888 0.796 6% 60% + * + * 0.66 is the last value that costs no recall, and 0.62 is where the refusal + * metric reaches its floor. Everything between them buys precision only, so the + * default sits at 0.62 rather than at the edge: this corpus is 28 documents and + * a real ZIM will not score identically, and the two failure modes are not + * symmetric. A dropped relevant chunk means no answer to a question the corpus + * *can* answer; an extra irrelevant chunk means noise the model mostly ignores + * — measured correctness was fine at 0.221 precision. Buy margin, not precision. + * + * Note this is a higher cutoff than the same sweep supports on the *semantic* + * axis, where recall starts falling at 0.60. That gap is the reranker earning + * its place: it separates relevant from irrelevant better than raw cosine does, + * so the floor can sit higher without cutting real answers. + * + * Note the honest ceiling on the refusal metric: three of the five refusal + * goldens are tagged `adversarial` and are near-corpus by construction ("what + * is the TR-88 pump's warranty period?" against a corpus that documents the + * TR-88 but not its warranty). Retrieving that document is *correct*; declining + * to answer from it is the generation tier's job, scored by refusalCorrectness. + * So nonEmptyRateOnRefusal bottoms out at 0.6 here, and a cutoff tuned to drive + * it lower is cutting real documents. + * + * Unset `rag.minRelevance` resolves to this value; see app/utils/rag_relevance.ts. + */ +export const RAG_MIN_FINAL_SCORE = 0.62 + +/** + * Preset relevance floors offered in Settings > Models, and the labels for them. + * The stored setting is the number, not the label, so retuning these presets + * later cannot invalidate a value someone has already saved. + */ +export const RAG_MIN_RELEVANCE_PRESETS = [ + { value: 0, label: 'Off — use every passage retrieved' }, + { value: 0.55, label: 'Lenient' }, + { value: RAG_MIN_FINAL_SCORE, label: 'Balanced (recommended)' }, + // The last floor that costs no recall on the golden set. Stricter than this + // starts dropping answers, which is a choice to offer, not one to default to. + { value: 0.66, label: 'Strict' }, +] as const + /** * Where the per-turn retrieved-context block sits in the prompt. * @@ -166,27 +233,24 @@ export const SYSTEM_PROMPTS = { - Use tables when presenting structured data. `, rag_context: (context: string) => ` -Information has been retrieved from the NOMAD knowledge base that MAY be relevant to the -user's question. It was selected by automated similarity search, which is imperfect — some -or all of it may be unrelated to what the user actually asked. +Information has been retrieved from the NOMAD knowledge base for the user's question. It +cleared a relevance filter before reaching you, so it is likely on topic — but it was +selected by automated search, so it may still not contain the specific detail asked for. [Knowledge Base Context] ${context} HOW TO ANSWER: -1. First, silently judge whether the context genuinely addresses the user's question. Use - it ONLY when it really contains relevant information. Do not force a connection that - isn't there: poetic, narrative, tangential, or topically-unrelated passages are NOT - relevant just because they share a word with the question — ignore them. -2. When the context is relevant, base your answer on it and answer directly and specifically. -3. When the context does not actually address the question, ignore it completely and answer - from your own general knowledge. Do this silently — do not mention the knowledge base, - the context, or the fact that it lacked an answer, and do not apologize. -4. Never narrate your retrieval or reasoning process. Do not write "according to Context 1", +1. When the context contains what was asked for, base your answer on it and answer + directly and specifically. +2. When it does not, ignore it and answer from your own general knowledge. Do this + silently — do not mention the knowledge base, the context, or the fact that it lacked + an answer, and do not apologize. +3. Never narrate your retrieval or reasoning process. Do not write "according to Context 1", "the context is unrelated, but", "I couldn't find specific context", or similar. Just give the answer as if you simply knew it. -5. Do not fabricate specifics (numbers, names, procedures) that are neither supported by - genuinely relevant context nor part of your reliable knowledge. +4. Do not fabricate specifics (numbers, names, procedures) that are neither supported by + the context nor part of your reliable knowledge. Format your response using markdown for readability. `, diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index d7f24c21..09e8c016 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -12,6 +12,7 @@ import { useModals } from '~/context/ModalContext' import StyledModal from '~/components/StyledModal' import type { NomadInstalledModel } from '../../../types/ollama' import { SERVICE_NAMES } from '../../../constants/service_names' +import { RAG_MIN_RELEVANCE_PRESETS } from '../../../constants/ollama' import Switch from '~/components/inputs/Switch' import Select from '~/components/inputs/Select' import StyledSectionHeader from '~/components/StyledSectionHeader' @@ -27,7 +28,7 @@ export default function ModelsPage(props: { models: { availableModels: NomadOllamaModel[] installedModels: NomadInstalledModel[] - settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string } + settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string; minRelevance: number } /** Effective window per installed model, as resolved by ContextWindowService. */ resolvedContextWindows?: Record } @@ -106,6 +107,7 @@ export default function ModelsPage(props: { const [ragEnabled, setRagEnabled] = useState(props.models.settings.ragEnabled) const [tasksModel, setTasksModel] = useState(props.models.settings.tasksModel) const [contextWindow, setContextWindow] = useState(props.models.settings.contextWindow) + const [minRelevance, setMinRelevance] = useState(String(props.models.settings.minRelevance)) const [aiAssistantCustomName, setAiAssistantCustomName] = useState( props.models.settings.aiAssistantCustomName ) @@ -264,6 +266,24 @@ export default function ModelsPage(props: { { value: '131072', label: '128K tokens' }, ] + // Presets rather than a raw 0-1 number: the value is a cosine-similarity + // floor, which is not a thing anyone can reason about directly. The stored + // setting is still the number, so retuning these labels later cannot orphan a + // saved value. + const minRelevanceOptions = [ + ...RAG_MIN_RELEVANCE_PRESETS.map((preset) => ({ + value: String(preset.value), + label: preset.label, + })), + // The setting is API-writable to any value in [0,1], so a value off the + // preset ladder is reachable. Surface it as a disabled option rather than + // letting the select render empty — same treatment the tasks model gets when + // the chosen model has since been deleted. + ...(RAG_MIN_RELEVANCE_PRESETS.some((p) => String(p.value) === minRelevance) + ? [] + : [{ value: minRelevance, label: `Custom (${minRelevance})`, disabled: true }]), + ] + const resolvedWindows = props.models.resolvedContextWindows ?? {} const formatWindow = (tokens: number) => tokens >= 1024 ? `${Math.round(tokens / 1024)}K` : String(tokens) @@ -403,6 +423,18 @@ export default function ModelsPage(props: { updateSettingMutation.mutate({ key: 'ai.tasksModel', value: newVal }) }} /> + { + // A chunk carries two scores, and the two cutoffs filter different ones: + // Qdrant's score_threshold sees the semantic score, RAG_MIN_FINAL_SCORE sees + // the post-rerank score. Collapsing them would make one cutoff calibrate + // against the other's percentiles, off by the reranker's boost factor. + const cases: RetrievalCase[] = [ + { + id: 'c1', + tags: [], + relevantDocIds: ['doc-a'], + expectRefusal: false, + retrieved: [ + { docId: 'doc-a', score: 0.9, semanticScore: 0.7 }, + { docId: 'doc-b', score: 0.5, semanticScore: 0.4 }, + ], + }, + ] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + + assert.equal(agg.relevantScores!.median, 0.7) + assert.equal(agg.irrelevantScores!.median, 0.4) + assert.equal(agg.relevantFinalScores!.median, 0.9) + assert.equal(agg.irrelevantFinalScores!.median, 0.5) +}) + +test('aggregate falls back to the final score when no semantic score was recorded', () => { + // The ablation stages record only one number per row, so the semantic axis + // has to degrade to it rather than reporting an empty distribution. + const cases: RetrievalCase[] = [ + { + id: 'c1', + tags: [], + relevantDocIds: ['doc-a'], + expectRefusal: false, + retrieved: [{ docId: 'doc-a', score: 0.8 }], + }, + ] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + + assert.equal(agg.relevantScores!.median, 0.8) + assert.equal(agg.relevantFinalScores!.median, 0.8) +}) diff --git a/admin/tests/unit/rag_relevance.spec.ts b/admin/tests/unit/rag_relevance.spec.ts new file mode 100644 index 00000000..9665f1cb --- /dev/null +++ b/admin/tests/unit/rag_relevance.spec.ts @@ -0,0 +1,95 @@ +/** + * Interpretation of the `rag.minRelevance` setting. + * + * Every case that matters here is a value that should never have been stored — + * a cleared row, a hand-edited string, a half-written migration. The rule these + * lock in is that none of them may silently switch the relevance floor *off*: + * losing the setting has to mean "use the recommended default", because the + * failure is invisible (irrelevant passages quietly return to every answer) + * rather than loud. The one value that does turn it off is an explicit 0, and + * that has to keep working — it is a real choice offered in the UI. + * + * Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed: + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { applyRelevanceFloor, parseMinRelevance } from '../../app/utils/misc.js' + +const DEFAULT = 0.6 + +test('min relevance: an unset setting uses the recommended default', () => { + assert.equal(parseMinRelevance(null, DEFAULT), DEFAULT) + assert.equal(parseMinRelevance(undefined, DEFAULT), DEFAULT) +}) + +test('min relevance: empty, whitespace and "auto" all count as unset', () => { + // SystemService.updateSetting clears the row on an empty string, but a value + // written before that behaviour (or by hand) must not read as 0. + assert.equal(parseMinRelevance('', DEFAULT), DEFAULT) + assert.equal(parseMinRelevance(' ', DEFAULT), DEFAULT) + assert.equal(parseMinRelevance('auto', DEFAULT), DEFAULT) +}) + +test('min relevance: an unparseable value uses the default rather than disabling the floor', () => { + assert.equal(parseMinRelevance('balanced', DEFAULT), DEFAULT) + assert.equal(parseMinRelevance('NaN', DEFAULT), DEFAULT) + assert.equal(parseMinRelevance('0.6.1', DEFAULT), DEFAULT) +}) + +test('min relevance: an explicit 0 turns the floor off and is not treated as unset', () => { + assert.equal(parseMinRelevance('0', DEFAULT), 0) + assert.equal(parseMinRelevance('0.0', DEFAULT), 0) +}) + +test('min relevance: a valid value is used as written', () => { + assert.equal(parseMinRelevance('0.5', DEFAULT), 0.5) + assert.equal(parseMinRelevance('0.68', DEFAULT), 0.68) + assert.equal(parseMinRelevance(' 0.55 ', DEFAULT), 0.55) + assert.equal(parseMinRelevance('1', DEFAULT), 1) +}) + +test('min relevance: out-of-range values clamp instead of being rejected', () => { + // The validator rejects these at the API, so reaching here means the row was + // written some other way. Clamping keeps retrieval working on a value that is + // at least meaningful, where a raw -5 would floor nothing and a raw 40 would + // floor everything. + assert.equal(parseMinRelevance('-5', DEFAULT), 0) + assert.equal(parseMinRelevance('40', DEFAULT), 1) +}) + +const ranked = (...scores: number[]) => scores.map((finalScore, i) => ({ finalScore, id: `c${i}` })) + +test('relevance floor: keeps chunks at or above the floor', () => { + // Boundary inclusive: a chunk scoring exactly the floor is on the right side + // of it. The presets are round numbers, so this edge is reachable. + const { survivors, belowFloor } = applyRelevanceFloor(ranked(0.8, 0.6, 0.59), 0.6) + assert.deepEqual(survivors.map((s) => s.id), ['c0', 'c1']) + assert.equal(belowFloor, 1) +}) + +test('relevance floor: nothing clearing it returns an empty list, not the best of a bad set', () => { + // The whole point of the item. Retrieval declining is what lets the prompt + // stop asking a 3B model to silently judge relevance for us. + const { survivors, belowFloor } = applyRelevanceFloor(ranked(0.55, 0.5, 0.42), 0.6) + assert.deepEqual(survivors, []) + assert.equal(belowFloor, 3) +}) + +test('relevance floor: a floor of 0 is a pass-through', () => { + // The "Off" preset, and the default for callers that predate the parameter. + const results = ranked(0.9, 0.1) + const { survivors, belowFloor } = applyRelevanceFloor(results, 0) + assert.equal(survivors, results) + assert.equal(belowFloor, 0) +}) + +test('relevance floor: an empty candidate list stays empty and reports nothing dropped', () => { + // "The knowledge base had no match above the Qdrant threshold" and "the floor + // rejected everything" are different states, and the retrieval-status UX needs + // to tell them apart — so an empty input must not report phantom drops. + const { survivors, belowFloor } = applyRelevanceFloor(ranked(), 0.6) + assert.deepEqual(survivors, []) + assert.equal(belowFloor, 0) +}) diff --git a/admin/tests/unit/rag_retrieval_toggle.spec.ts b/admin/tests/unit/rag_retrieval_toggle.spec.ts index 79e63b88..e8a09d03 100644 --- a/admin/tests/unit/rag_retrieval_toggle.spec.ts +++ b/admin/tests/unit/rag_retrieval_toggle.spec.ts @@ -19,19 +19,38 @@ import { test } from '@japa/runner' import { RagPipelineService } from '#services/rag_pipeline_service' import { SYSTEM_PROMPTS } from '../../constants/ollama.js' import type { OllamaChatMessage } from '../../types/ollama.js' +import type { RetrievedChunk } from '../../types/rag.js' /** Records every call so the tests can assert on what was *not* run. */ -function makeFakes(opts: { nomadMd?: string | null } = {}) { +function makeFakes(opts: { nomadMd?: string | null; searchResults?: RetrievedChunk[] } = {}) { const calls = { hasDocuments: 0, search: 0, chat: 0 } + const args = { minFinalScore: undefined as number | undefined } const ragService = { async hasDocuments() { calls.hasDocuments++ return true }, - async searchSimilarDocuments() { + async searchSimilarDocuments( + _query: string, + _limit: number, + _scoreThreshold: number, + _collection: string | undefined, + _stagesOut: unknown, + minFinalScore: number, + floorOut?: { candidates: number; belowFloor: number } + ) { calls.search++ - return [{ text: 'retrieved body', score: 0.9, metadata: { full_title: 'A Doc' } }] + args.minFinalScore = minFinalScore + const results = + opts.searchResults ?? [{ text: 'retrieved body', score: 0.9, metadata: { full_title: 'A Doc' } }] + // Stand in for the real floor: whatever the fake was told to return is + // what survived, and anything it was told to withhold is what did not. + if (floorOut) { + floorOut.candidates = 3 + floorOut.belowFloor = 3 - results.length + } + return results }, } @@ -76,7 +95,7 @@ function makeFakes(opts: { nomadMd?: string | null } = {}) { tokenCalibration as any ) - return { service, calls } + return { service, calls, args } } const userTurn: OllamaChatMessage[] = [{ role: 'user', content: 'how do I purify water?' }] @@ -145,3 +164,45 @@ test.group('buildPrompt | skipRetrieval', () => { assert.lengthOf(trace.injected, 1) }) }) + +test.group('buildPrompt | relevance floor', () => { + test('injects no context block when nothing clears the floor', async ({ assert }) => { + // The point of the floor. Retrieval ran, found candidates, and declined them + // all — so the model must be handed nothing rather than the best of a bad + // set, which is what the rag_context prompt used to have to talk it out of. + const { service, calls } = makeFakes({ searchResults: [] }) + + const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', { minFinalScore: 0.62 }) + + assert.equal(calls.search, 1) + assert.deepEqual(trace.retrieved, []) + assert.deepEqual(trace.injected, []) + assert.isFalse(systemContents(trace.messages).some((c) => c.includes('[Context 1'))) + // ...and the default system prompt is untouched: declining to retrieve is + // not the same as turning the assistant off. + assert.include(systemContents(trace.messages), SYSTEM_PROMPTS.default) + }) + + test('records the floor and the drop count on the trace', async ({ assert }) => { + // "The knowledge base had no match at all" and "everything found was judged + // irrelevant" are different things to tell a user, so the trace has to keep + // them apart rather than both collapsing to an empty retrieved list. + const { service } = makeFakes({ searchResults: [] }) + + const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', { minFinalScore: 0.62 }) + + assert.equal(trace.minFinalScore, 0.62) + assert.equal(trace.chunksBelowFloor, 3) + }) + + test('an explicit option overrides the rag.minRelevance setting', async ({ assert }) => { + // What keeps the eval harness reproducible: it always passes a value, so a + // slider set on one machine cannot move the numbers a baseline was recorded + // against. + const { service, args } = makeFakes() + + await service.buildPrompt(userTurn, 'llama3.1:8b', { minFinalScore: 0.9 }) + + assert.equal(args.minFinalScore, 0.9) + }) +}) diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 3b040eb0..b6fed876 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -9,6 +9,11 @@ export const KV_STORE_SCHEMA = { // pipeline (hasDocuments, the query-rewrite LLM call, and the Qdrant search), // which matters on small hardware and when the KB is small or empty. 'rag.enabled': 'boolean', + // Relevance floor for retrieved chunks, as a stringified number in [0,1] + // ("0.6"). Unset means "use RAG_MIN_FINAL_SCORE"; "0" explicitly means off. + // Stored as the number rather than a preset name so retuning the presets in + // Settings > Models cannot invalidate a value someone already saved. + 'rag.minRelevance': 'string', 'system.updateAvailable': 'boolean', 'system.latestVersion': 'string', 'system.earlyAccess': 'boolean', diff --git a/admin/types/rag.ts b/admin/types/rag.ts index 06e3a501..a75fff56 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -62,6 +62,19 @@ export type RetrievalStages = { diversified?: StageEntry[] } +/** + * Counts from the relevance floor, for callers that want to say *why* nothing + * came back. Written whether or not stage ablation is on — it is two integers, + * and "we searched and found nothing relevant enough" is worth being able to + * say out loud. + */ +export type RetrievalFloorStats = { + /** Candidates the floor was applied to (post-rerank, pre-diversity). */ + candidates: number + /** How many of those fell below it. */ + belowFloor: number +} + /** * A chunk as returned by `RagService.searchSimilarDocuments` — the shape the * chat pipeline consumes and the eval harness scores. @@ -98,6 +111,10 @@ export type PipelineOptions = { /** Override where the retrieved-context block sits. Defaults to RAG_PLACEMENT; * the eval harness sets it explicitly to compare the two orderings. */ ragPlacement?: RagPlacement + /** Post-rerank relevance floor. Unset resolves the user's `rag.minRelevance` + * setting; the eval harness passes an explicit value so its numbers cannot + * depend on how one machine's slider happens to be set. */ + minFinalScore?: number } /** @@ -121,6 +138,14 @@ export type PipelineTrace = { numPredict: number | undefined contextLimits: { maxResults: number; maxTokens: number } timings: { rewriteMs: number; retrievalMs: number } + /** + * The relevance floor this turn was retrieved under, and how many candidates + * fell below it. `chunksBelowFloor > 0` with `retrieved.length === 0` is the + * "we searched and nothing was relevant enough" case — which is the honest + * thing to tell the user, and the data the retrieval-status UX needs to say it. + */ + minFinalScore: number + chunksBelowFloor: number /** * What the budget planner decided: how the window was spent and what was left * out. Undefined only when planning was bypassed. The eval harness reads this