mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 21:29:20 +02:00
There was one cutoff, Qdrant's score_threshold at 0.3, and nomic scores unrelated text well above it, so every question retrieved *something* (nonEmptyRateOnRefusal 1.0) and rag_context wasted four lines asking a 3B model to "silently judge" relevance. That is a classification a number should decide, not a micro model. Adds `RAG_MIN_FINAL_SCORE`, a floor on the post-rerank score, applied before source diversity. The floor judges relevance, the 0.85^n penalty judges redundancy. Nothing clears it and nothing is injected. `score_threshold` stays at 0.3 as the candidate net. Exposed as `rag.minRelevance` with a module- scoped TTL cache invalidated from `updateSetting`. The floor is calibrated and the harness was describing the wrong number: aggregate() bucketed semanticScore, so its percentiles measured raw cosine while the new knob filters the reranked score. Both axes now, and the gap matters: recall falls at 0.60 on the semantic axis, 0.66 on the final one. floor recall@5 precision@5 emptyOnAnswerable nonEmptyOnRefusal 0.00 0.991 0.221 0% 100% 0.62 0.991 0.524 0% 60% 0.66 0.991 0.695 0% 60% 0.67 0.972 0.704 1% 60% Default 0.62, not the last-lossless 0.66: the refusal win is complete at 0.62 and 28 documents is a thin basis for spending margin. 0.60 is the honest floor. Three of five refusal goldens are near-corpus by construction, where declining is the generation tier's job. Both eval tiers pin the floor to the constant and ignore the setting, for the same reason the harness never sets skipRetrieval. 4 metrics improved against baseline, no regressions.
181 lines
6.6 KiB
TypeScript
181 lines
6.6 KiB
TypeScript
import KVStore from '#models/kv_store'
|
|
import { BenchmarkService } from '#services/benchmark_service'
|
|
import { ContextWindowService } from '#services/context_window_service'
|
|
import { MapService } from '#services/map_service'
|
|
import { OllamaService } from '#services/ollama_service'
|
|
import { SystemService } from '#services/system_service'
|
|
import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#validators/settings'
|
|
import { inject } from '@adonisjs/core'
|
|
import type { HttpContext } from '@adonisjs/core/http'
|
|
import env from '#start/env'
|
|
import { parseMinRelevance } from '../utils/misc.js'
|
|
import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js'
|
|
|
|
@inject()
|
|
export default class SettingsController {
|
|
constructor(
|
|
private systemService: SystemService,
|
|
private mapService: MapService,
|
|
private benchmarkService: BenchmarkService,
|
|
private ollamaService: OllamaService,
|
|
private contextWindowService: ContextWindowService
|
|
) {}
|
|
|
|
async system({ inertia }: HttpContext) {
|
|
const systemInfo = await this.systemService.getSystemInfo()
|
|
return inertia.render('settings/system', {
|
|
system: {
|
|
info: systemInfo,
|
|
},
|
|
})
|
|
}
|
|
|
|
async apps({ inertia }: HttpContext) {
|
|
const services = await this.systemService.getServices({ installedOnly: false })
|
|
return inertia.render('settings/apps', {
|
|
system: {
|
|
services,
|
|
},
|
|
})
|
|
}
|
|
|
|
async legal({ inertia }: HttpContext) {
|
|
return inertia.render('settings/legal')
|
|
}
|
|
|
|
async support({ inertia }: HttpContext) {
|
|
return inertia.render('settings/support')
|
|
}
|
|
|
|
async maps({ inertia }: HttpContext) {
|
|
const baseAssetsCheck = await this.mapService.ensureBaseAssets()
|
|
const [regionFiles, worldBasemapExists] = await Promise.all([
|
|
this.mapService.listRegions(),
|
|
this.mapService.checkWorldBasemapExists(),
|
|
])
|
|
return inertia.render('settings/maps', {
|
|
maps: {
|
|
baseAssetsExist: baseAssetsCheck,
|
|
worldBasemapExists,
|
|
regionFiles: regionFiles.files,
|
|
},
|
|
})
|
|
}
|
|
|
|
async models({ inertia }: HttpContext) {
|
|
const availableModels = await this.ollamaService.getAvailableModels({
|
|
sort: 'pulls',
|
|
recommendedOnly: false,
|
|
query: null,
|
|
limit: 15,
|
|
})
|
|
const installedModels = await this.ollamaService.getModels().catch(() => [])
|
|
const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
|
|
const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName')
|
|
const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
|
|
const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention')
|
|
const autoThinking = await KVStore.getValue('ai.autoThinking')
|
|
const tasksModel = await KVStore.getValue('ai.tasksModel')
|
|
const ragEnabled = await KVStore.getValue('rag.enabled')
|
|
const contextWindow = await KVStore.getValue('ai.contextWindow')
|
|
const minRelevance = await KVStore.getValue('rag.minRelevance')
|
|
// Resolved window per installed model, so the setting shows what "Auto"
|
|
// actually produced rather than leaving the user to guess. Best-effort:
|
|
// a model whose metadata can't be read simply doesn't get a badge.
|
|
const resolvedContextWindows: Record<string, number> = {}
|
|
await Promise.all(
|
|
(installedModels || []).map(async (model) => {
|
|
try {
|
|
resolvedContextWindows[model.name] = await this.contextWindowService.windowFor(model.name)
|
|
} catch {
|
|
/* leave unset */
|
|
}
|
|
})
|
|
)
|
|
return inertia.render('settings/models', {
|
|
models: {
|
|
availableModels: availableModels?.models || [],
|
|
installedModels: installedModels || [],
|
|
settings: {
|
|
chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
|
|
aiAssistantCustomName: aiAssistantCustomName ?? '',
|
|
remoteOllamaUrl: remoteOllamaUrl ?? '',
|
|
ollamaFlashAttention: ollamaFlashAttention ?? true,
|
|
autoThinking: autoThinking ?? false,
|
|
tasksModel: tasksModel ?? '',
|
|
ragEnabled: ragEnabled ?? true,
|
|
contextWindow: contextWindow ?? 'auto',
|
|
// Sent as the resolved number so the select can match an option
|
|
// without duplicating the "unset means the default" rule in the UI.
|
|
minRelevance: parseMinRelevance(minRelevance, RAG_MIN_FINAL_SCORE),
|
|
},
|
|
resolvedContextWindows,
|
|
},
|
|
})
|
|
}
|
|
|
|
async update({ inertia }: HttpContext) {
|
|
const updateInfo = await this.systemService.checkLatestVersion()
|
|
return inertia.render('settings/update', {
|
|
system: {
|
|
updateAvailable: updateInfo.updateAvailable,
|
|
latestVersion: updateInfo.latestVersion,
|
|
currentVersion: updateInfo.currentVersion,
|
|
},
|
|
})
|
|
}
|
|
|
|
async zim({ inertia }: HttpContext) {
|
|
return inertia.render('settings/zim/index')
|
|
}
|
|
|
|
async zimRemote({ inertia }: HttpContext) {
|
|
return inertia.render('settings/zim/remote-explorer')
|
|
}
|
|
|
|
async creatorPacks({ inertia }: HttpContext) {
|
|
return inertia.render('settings/creator-packs')
|
|
}
|
|
|
|
async benchmark({ inertia }: HttpContext) {
|
|
const latestResult = await this.benchmarkService.getLatestResult()
|
|
const status = this.benchmarkService.getStatus()
|
|
return inertia.render('settings/benchmark', {
|
|
benchmark: {
|
|
latestResult,
|
|
status: status.status,
|
|
currentBenchmarkId: status.benchmarkId,
|
|
},
|
|
})
|
|
}
|
|
|
|
async advanced({ inertia }: HttpContext) {
|
|
// When the env var is set it always takes precedence over the stored value,
|
|
// so surface that to the UI to disable the field and explain the override.
|
|
const envOverride = Boolean(env.get('INTERNET_STATUS_TEST_URL')?.trim())
|
|
const internetStatusTestUrl = await KVStore.getValue('system.internetStatusTestUrl')
|
|
return inertia.render('settings/advanced', {
|
|
advanced: {
|
|
internetStatusTestUrl: internetStatusTestUrl ?? '',
|
|
internetStatusTestUrlEnvOverride: envOverride,
|
|
},
|
|
})
|
|
}
|
|
|
|
async getSetting({ request, response }: HttpContext) {
|
|
const { key } = await getSettingSchema.validate({ key: request.qs().key });
|
|
const value = await KVStore.getValue(key);
|
|
return response.status(200).send({ key, value });
|
|
}
|
|
|
|
async updateSetting({ request, response }: HttpContext) {
|
|
const reqData = await request.validateUsing(updateSettingSchema)
|
|
const valueError = validateSettingValue(reqData.key, reqData.value)
|
|
if (valueError) {
|
|
return response.status(422).send({ success: false, message: valueError })
|
|
}
|
|
await this.systemService.updateSetting(reqData.key, reqData.value)
|
|
return response.status(200).send({ success: true, message: 'Setting updated successfully' })
|
|
}
|
|
}
|