diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 3d01ca4..01479f4 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -68,6 +68,11 @@ export default class RagController { return response.status(200).json({ files }) } + public async getFileWarnings({ response }: HttpContext) { + const warnings = await this.ragService.computeFileWarnings() + return response.status(200).json({ warnings }) + } + public async deleteFile({ request, response }: HttpContext) { const { source } = await request.validateUsing(deleteFileSchema) const result = await this.ragService.deleteFileBySource(source) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 3775818..91f1658 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -18,6 +18,8 @@ import { join, resolve, sep } from 'node:path' import KVStore from '#models/kv_store' import KbIngestState from '#models/kb_ingest_state' import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' +import KbRatioRegistry from '#models/kb_ratio_registry' +import { decideWarnings, type FileWarning } from '../utils/kb_warning_decision.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1086,6 +1088,90 @@ export class RagService { } } + /** + * Compute conditional warnings (RFC #883 §6) for every source the scanner + * sees on disk. Returns a map from source path → list of warnings, with + * sources that have no warnings omitted entirely (so the frontend can + * `warningsBySource[source] ?? []` for clean defaults). + * + * Per-source chunk counts come from a single Qdrant scroll over the + * collection's points; expected-chunk estimates come from the ratio + * registry. Files in the scanner's directories that have no qdrant points + * at all show up with `chunksInQdrant: 0` so Warning A can fire. + */ + public async computeFileWarnings(): Promise> { + try { + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Per-source chunk count from a single scroll. We deliberately don't + // assume `kb_ingest_state.chunks_embedded` here so this PR stays + // independent of the state-machine PR (#888) — but a future cleanup can + // read from there for efficiency once both have landed. + const chunksBySource = new Map() + let offset: string | number | null | Record = null + const batchSize = 100 + do { + const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, { + limit: batchSize, + offset, + with_payload: ['source'], + with_vector: false, + }) + for (const point of scrollResult.points) { + const source = point.payload?.source + if (source && typeof source === 'string') { + chunksBySource.set(source, (chunksBySource.get(source) ?? 0) + 1) + } + } + offset = scrollResult.next_page_offset || null + } while (offset !== null) + + // Scan the filesystem the same way scanAndSyncStorage does so Warning A + // can fire on files with zero qdrant points (the headline "video-only + // ZIM" case). + const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) + const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) + const allSources = new Set(chunksBySource.keys()) + const sizeByPath = new Map() + + for (const dir of [KB_UPLOADS_PATH, ZIM_PATH]) { + try { + const entries = await listDirectoryContentsRecursive(dir) + for (const entry of entries) { + if (entry.type !== 'file') continue + allSources.add(entry.key) + const stat = await getFileStatsIfExists(entry.key) + if (stat) sizeByPath.set(entry.key, Number(stat.size)) + } + } catch (error: any) { + if (error?.code !== 'ENOENT') throw error + } + } + + const out: Record = {} + for (const source of allSources) { + const fileSizeBytes = sizeByPath.get(source) ?? 0 + const chunksInQdrant = chunksBySource.get(source) ?? 0 + const fileName = source.split(/[/\\]/).pop() ?? source + const expectedChunks = + fileSizeBytes > 0 + ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes) + : null + + const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks }) + if (warnings.length > 0) out[source] = warnings + } + + return out + } catch (error) { + logger.error('[RAG] Error computing file warnings:', error) + return {} + } + } + /** * Delete all Qdrant points associated with a given source path and remove * the corresponding file from disk if it lives under the uploads directory. diff --git a/admin/app/utils/kb_warning_decision.ts b/admin/app/utils/kb_warning_decision.ts new file mode 100644 index 0000000..cec59fa --- /dev/null +++ b/admin/app/utils/kb_warning_decision.ts @@ -0,0 +1,70 @@ +/** + * Conditional warnings surfaced on Stored Files rows in the KB panel. + * See RFC #883 §6 — these warnings appear ONLY when their triggering condition + * is met, never on healthy files, to keep the panel silent in the common case. + * + * - `zero_chunks` — a non-trivial file produced 0 embedding chunks. Common + * cause: video-only or image-only ZIMs that the pipeline + * completes "successfully" with no extractable text. + * AI Assistant cannot reference this content. + * - `partial_stall` — the file has embedded chunks but well below the count + * expected from the ratio registry. Likely a mid-batch + * stall (which the binary "any chunks ⇒ embedded" check + * used to mask). Surfaces a Retry affordance. + */ +export type FileWarning = + | { kind: 'zero_chunks'; fileSizeBytes: number } + | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } + +/** Files smaller than this are too small to flag as suspicious zero-chunk + * cases — a 5 KB upload that produces 0 chunks is much more likely to be a + * legitimate edge case (placeholder file) than the gigabyte-scale video ZIM + * problem this warning targets. */ +export const ZERO_CHUNKS_MIN_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB + +/** Fraction of expected chunks below which we consider a file partially + * stalled. 0.5 (50%) matches the threshold described in RFC #883 §6 Warning B. */ +export const PARTIAL_STALL_RATIO_THRESHOLD = 0.5 + +export interface WarningInputs { + /** Source file size on disk in bytes. */ + fileSizeBytes: number + /** Distinct chunks present in Qdrant for this source. */ + chunksInQdrant: number + /** Best estimate of chunks the file should produce, from the ratio + * registry. `null` when no registry pattern matches and no fallback is + * configured — Warning B is suppressed in that case (we'd rather be silent + * than wrong). */ + expectedChunks: number | null +} + +export function decideWarnings(inputs: WarningInputs): FileWarning[] { + const warnings: FileWarning[] = [] + + // Warning A: file is large but produced nothing. Almost always a video-only + // or image-only ZIM; AI Assistant literally cannot reference this content. + if ( + inputs.chunksInQdrant === 0 && + inputs.fileSizeBytes > ZERO_CHUNKS_MIN_SIZE_BYTES + ) { + warnings.push({ kind: 'zero_chunks', fileSizeBytes: inputs.fileSizeBytes }) + } + + // Warning B: chunks present but far below expectation. Suppresses when we + // have no expectation (registry miss) since the comparison would be + // meaningless and we'd rather under-warn than mislead. + if ( + inputs.expectedChunks !== null && + inputs.expectedChunks > 0 && + inputs.chunksInQdrant > 0 && + inputs.chunksInQdrant < inputs.expectedChunks * PARTIAL_STALL_RATIO_THRESHOLD + ) { + warnings.push({ + kind: 'partial_stall', + chunksEmbedded: inputs.chunksInQdrant, + chunksExpected: inputs.expectedChunks, + }) + } + + return warnings +} diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 8b41524..9311718 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -51,6 +51,16 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) + // Per-file conditional warnings (RFC #883 §6). Only sources with at least + // one triggered warning are returned, so an empty map means everything is + // healthy. Polled at the same idle cadence as health for low overhead. + const { data: fileWarnings = {} } = useQuery({ + queryKey: ['kbFileWarnings'], + queryFn: () => api.getKbFileWarnings(), + select: (data) => data ?? {}, + refetchInterval: 30_000, + }) + // Global auto-index policy. KVStore returns `null` for an unset key, which // we treat as 'Always' for backward compatibility with installs that predate // this UI. The user can opt into Manual mode from the toggle below. @@ -442,8 +452,34 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o accessor: 'source', title: 'File Name', render(record) { + const warnings = fileWarnings[record.source] ?? [] return ( - {record.displayName} +
+ + {sourceToDisplayName(record.source)} + + {warnings.map((w, i) => ( + + + {w.kind === 'zero_chunks' && ( + + Embedded 0 chunks — this file has no text content. + AI Assistant cannot reference it. + + )} + {w.kind === 'partial_stall' && ( + + Only {w.chunksEmbedded.toLocaleString()} of est.{' '} + {w.chunksExpected.toLocaleString()} chunks embedded — + ingestion may have stalled. + + )} + + ))} +
) }, }, diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 0b3359f..628f4a0 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -6,6 +6,7 @@ import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' import { EmbedJobWithProgress } from '../../types/rag' +import type { FileWarning } from '../../app/utils/kb_warning_decision.js' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -475,6 +476,15 @@ class API { })() } + async getKbFileWarnings() { + return catchInternal(async () => { + const response = await this.client.get<{ warnings: Record }>( + '/rag/file-warnings' + ) + return response.data.warnings + })() + } + async deleteRAGFile(source: string) { return catchInternal(async () => { const response = await this.client.delete<{ message: string }>('/rag/files', { data: { source } }) diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 2038cb7..d30dc61 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -141,6 +141,7 @@ router .group(() => { router.post('/upload', [RagController, 'upload']) router.get('/files', [RagController, 'getStoredFiles']) + router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) router.get('/failed-jobs', [RagController, 'getFailedJobs']) diff --git a/admin/tests/unit/kb_warning_decision.spec.ts b/admin/tests/unit/kb_warning_decision.spec.ts new file mode 100644 index 0000000..4ca6a00 --- /dev/null +++ b/admin/tests/unit/kb_warning_decision.spec.ts @@ -0,0 +1,125 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { decideWarnings } from '../../app/utils/kb_warning_decision.js' + +const MB = 1024 * 1024 + +test('healthy file: chunks present and on-target → no warnings', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 11_000, + expectedChunks: 11_000, + }), + [] + ) +}) + +test('healthy file: chunks slightly above expectation → no warnings', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 12_000, + expectedChunks: 11_000, + }), + [] + ) +}) + +test('Warning A: large file with 0 chunks (video-only ZIM)', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 5 * 1024 * MB, + chunksInQdrant: 0, + expectedChunks: 0, + }), + [{ kind: 'zero_chunks', fileSizeBytes: 5 * 1024 * MB }] + ) +}) + +test('Warning A: small empty file is silently ignored (under 100 MB threshold)', () => { + // A user uploads a 5 KB placeholder.txt that produces nothing → not worth a banner + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 5 * 1024, // 5 KB + chunksInQdrant: 0, + expectedChunks: null, + }), + [] + ) +}) + +test('Warning B: partial stall — chunks well below expectation', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 1000 * MB, + chunksInQdrant: 266, + expectedChunks: 600_000, + }), + [{ kind: 'partial_stall', chunksEmbedded: 266, chunksExpected: 600_000 }] + ) +}) + +test('Warning B: chunks just under 50% of expected → triggers', () => { + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 4_999, + expectedChunks: 10_000, + }), + [{ kind: 'partial_stall', chunksEmbedded: 4_999, chunksExpected: 10_000 }] + ) +}) + +test('Warning B: chunks at exactly 50% of expected → does NOT trigger', () => { + // Strict less-than threshold leaves room for the boundary + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 5_000, + expectedChunks: 10_000, + }), + [] + ) +}) + +test('Warning B suppressed when expectedChunks is null (registry miss)', () => { + // Better to be silent than show a meaningless "266 of unknown" comparison + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 100 * MB, + chunksInQdrant: 266, + expectedChunks: null, + }), + [] + ) +}) + +test('Warning B suppressed when expectedChunks is 0 (video-only registry entry)', () => { + // A `lrnselfreliance_` row in the registry says "expect 0 chunks". A real + // file matching it correctly producing 0 chunks must not trigger Warning B. + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 500 * MB, + chunksInQdrant: 0, + expectedChunks: 0, + }), + // Note: Warning A triggers here because file > 100 MB and chunks = 0 + [{ kind: 'zero_chunks', fileSizeBytes: 500 * MB }] + ) +}) + +test('Both warnings can fire on the same file in principle', () => { + // Edge case: huge file, 0 chunks, but ratio registry expected 100k. + // Warning A fires (large + zero), Warning B suppressed (chunksInQdrant must be > 0). + // This documents the chunksInQdrant > 0 guard on Warning B. + assert.deepEqual( + decideWarnings({ + fileSizeBytes: 1000 * MB, + chunksInQdrant: 0, + expectedChunks: 100_000, + }), + [{ kind: 'zero_chunks', fileSizeBytes: 1000 * MB }] + ) +})