mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 05:14:19 +02:00
There was one cutoff, Qdrant's score_threshold at 0.3, and nomic scores unrelated text well above it, so every question retrieved *something* (nonEmptyRateOnRefusal 1.0) and rag_context wasted four lines asking a 3B model to "silently judge" relevance. That is a classification a number should decide, not a micro model. Adds `RAG_MIN_FINAL_SCORE`, a floor on the post-rerank score, applied before source diversity. The floor judges relevance, the 0.85^n penalty judges redundancy. Nothing clears it and nothing is injected. `score_threshold` stays at 0.3 as the candidate net. Exposed as `rag.minRelevance` with a module- scoped TTL cache invalidated from `updateSetting`. The floor is calibrated and the harness was describing the wrong number: aggregate() bucketed semanticScore, so its percentiles measured raw cosine while the new knob filters the reranked score. Both axes now, and the gap matters: recall falls at 0.60 on the semantic axis, 0.66 on the final one. floor recall@5 precision@5 emptyOnAnswerable nonEmptyOnRefusal 0.00 0.991 0.221 0% 100% 0.62 0.991 0.524 0% 60% 0.66 0.991 0.695 0% 60% 0.67 0.972 0.704 1% 60% Default 0.62, not the last-lossless 0.66: the refusal win is complete at 0.62 and 28 documents is a thin basis for spending margin. 0.60 is the honest floor. Three of five refusal goldens are near-corpus by construction, where declining is the generation tier's job. Both eval tiers pin the floor to the constant and ignore the setting, for the same reason the harness never sets skipRetrieval. 4 metrics improved against baseline, no regressions.
110 lines
4.8 KiB
TypeScript
110 lines
4.8 KiB
TypeScript
import vine from "@vinejs/vine";
|
|
import { SETTINGS_KEYS } from "../../constants/kv_store.js";
|
|
import type { KVStoreKey } from "../../types/kv_store.js";
|
|
import { CONTEXT_LADDER } from "../utils/context_window.js";
|
|
|
|
export const getSettingSchema = vine.compile(vine.object({
|
|
key: vine.enum(SETTINGS_KEYS),
|
|
}))
|
|
|
|
export const updateSettingSchema = vine.compile(vine.object({
|
|
key: vine.enum(SETTINGS_KEYS),
|
|
value: vine.any().optional(),
|
|
}))
|
|
|
|
const HHMM_PATTERN = /^([01]\d|2[0-3]):[0-5]\d$/
|
|
|
|
/**
|
|
* Validate the *value* for keys that have format constraints beyond the generic
|
|
* enum/any check (the generic validator only constrains the key). Returns an
|
|
* error message string when invalid, or null when the value is acceptable.
|
|
*/
|
|
export function validateSettingValue(key: KVStoreKey, value: unknown): string | null {
|
|
switch (key) {
|
|
case 'autoUpdate.windowStart':
|
|
case 'autoUpdate.windowEnd':
|
|
case 'contentAutoUpdate.windowStart':
|
|
case 'contentAutoUpdate.windowEnd':
|
|
if (typeof value !== 'string' || !HHMM_PATTERN.test(value)) {
|
|
return 'Time window values must be in 24-hour HH:MM format (e.g. "20:00").'
|
|
}
|
|
return null
|
|
case 'autoUpdate.cooloffHours':
|
|
case 'contentAutoUpdate.cooloffHours': {
|
|
const num = Number(value)
|
|
if (!Number.isInteger(num) || num < 0 || num > 8760) {
|
|
return 'Cool-off must be a whole number of hours between 0 and 8760.'
|
|
}
|
|
return null
|
|
}
|
|
case 'system.internetStatusTestUrl': {
|
|
// Empty clears the setting (reverts to env var / built-in defaults).
|
|
if (value === '' || value === undefined || value === null) {
|
|
return null
|
|
}
|
|
if (typeof value !== 'string') {
|
|
return 'Test URL must be a string.'
|
|
}
|
|
try {
|
|
const url = new URL(value)
|
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
return 'Test URL must use http or https.'
|
|
}
|
|
} catch {
|
|
return 'Test URL must be a valid URL (e.g. "https://example.com").'
|
|
}
|
|
return null
|
|
}
|
|
case 'contentAutoUpdate.maxBytesPerWindow': {
|
|
// Per-window download budget in bytes. 0 = unlimited.
|
|
const num = Number(value)
|
|
if (!Number.isInteger(num) || num < 0) {
|
|
return 'The per-window data cap must be a whole number of bytes (0 = unlimited).'
|
|
}
|
|
return null
|
|
}
|
|
case 'ai.contextWindow': {
|
|
// "auto" (or empty) hands sizing to ContextWindowService. An explicit
|
|
// value is a *cap*, so it only ever lowers the resolved window — but it
|
|
// still has to be a real ladder rung, since a value the backend can't
|
|
// honour would silently fall back to the default and confuse the user.
|
|
if (value === '' || value === 'auto' || value === undefined || value === null) {
|
|
return null
|
|
}
|
|
const num = Number(value)
|
|
if (!CONTEXT_LADDER.includes(num as (typeof CONTEXT_LADDER)[number])) {
|
|
return `Context window must be "auto" or one of: ${CONTEXT_LADDER.join(', ')}.`
|
|
}
|
|
return null
|
|
}
|
|
case 'rag.minRelevance': {
|
|
// Empty/'auto' clears the setting and reverts to RAG_MIN_FINAL_SCORE.
|
|
// An explicit 0 is a real choice — it turns the floor off — so it is
|
|
// deliberately not treated as "unset".
|
|
// Trimmed first so the accepted set matches parseMinRelevance's:
|
|
// Number(' ') is 0, which would otherwise validate as "floor off"
|
|
// for a value the parser reads as unset.
|
|
const raw = typeof value === 'string' ? value.trim() : value
|
|
if (raw === '' || raw === 'auto' || raw === undefined || raw === null) {
|
|
return null
|
|
}
|
|
const num = Number(raw)
|
|
if (!Number.isFinite(num) || num < 0 || num > 1) {
|
|
return 'Relevance threshold must be a number between 0 and 1, or empty to use the recommended default.'
|
|
}
|
|
return null
|
|
}
|
|
case 'ai.keepAlive': {
|
|
// Ollama duration format: "10m", "1h", "30s", "-1" (forever), "0" (evict).
|
|
if (value === '' || value === undefined || value === null) {
|
|
return null
|
|
}
|
|
if (typeof value !== 'string' || !/^(-?\d+|\d+(\.\d+)?(ms|s|m|h))$/.test(value)) {
|
|
return 'Keep-alive must be a duration like "10m", "1h", or "-1" to keep the model loaded indefinitely.'
|
|
}
|
|
return null
|
|
}
|
|
default:
|
|
return null
|
|
}
|
|
} |