diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 0930cb56..61c15f59 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -14,6 +14,7 @@ import type { HttpContext } from '@adonisjs/core/http' import { SERVICE_NAMES } from '../../constants/service_names.js' import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js' import type { PipelineTrace } from '../../types/rag.js' +import { buildCitations } from '../utils/rag_prompt.js' import logger from '@adonisjs/core/services/logger' @inject() @@ -79,6 +80,11 @@ export default class OllamaController { reqData.messages = trace.messages const numCtx = trace.numCtx const numPredict = trace.numPredict + // Provenance for the answer about to be generated (#1179). Built from + // trace.injected -- what the model actually read -- and surfaced under the + // answer as "Sources". Empty whenever retrieval was skipped or declined, + // which is the honest result: no context, no citations. + const sources = buildCitations(trace.injected) // Keeping the model resident is what makes the KV cache worth building: // Ollama's default evicts after 5 minutes, which is well inside the time a // user spends reading an answer and typing the next question. @@ -145,11 +151,18 @@ export default class OllamaController { } throw err } + // Trailing citation event, written before end(). It carries no `message` + // key, which is how the client tells it apart from Ollama's own chunks. + if (sources.length > 0) { + response.response.write(`data: ${JSON.stringify({ sources })} + +`) + } response.response.end() // Save assistant message and optionally generate title if (sessionId && fullContent) { - await this.chatService.addMessage(sessionId, 'assistant', fullContent) + await this.chatService.addMessage(sessionId, 'assistant', fullContent, sources) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { this.chatService.generateTitle(sessionId, userContent, fullContent, reqData.model).catch((err) => { @@ -174,7 +187,7 @@ export default class OllamaController { } if (sessionId && result?.message?.content) { - await this.chatService.addMessage(sessionId, 'assistant', result.message.content) + await this.chatService.addMessage(sessionId, 'assistant', result.message.content, sources) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { this.chatService.generateTitle(sessionId, userContent, result.message.content, reqData.model).catch((err) => { @@ -183,7 +196,7 @@ export default class OllamaController { } } - return result + return { ...result, sources } } catch (error) { if (reqData.stream) { response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`) diff --git a/admin/app/models/chat_message.ts b/admin/app/models/chat_message.ts index f0633ffe..b7602ac7 100644 --- a/admin/app/models/chat_message.ts +++ b/admin/app/models/chat_message.ts @@ -18,6 +18,9 @@ export default class ChatMessage extends BaseModel { @column() declare content: string + @column() + declare sources: string | null + @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' }) declare session: BelongsTo diff --git a/admin/app/services/chat_service.ts b/admin/app/services/chat_service.ts index 7cc46cf2..e85fa344 100644 --- a/admin/app/services/chat_service.ts +++ b/admin/app/services/chat_service.ts @@ -15,6 +15,7 @@ import { pickTitle, resolveStructured, } from '../utils/structured_output.js' +import type { ChatSource } from '../../types/chat.js' /** Sidebar width, near enough. Applied once, to whichever candidate title won. */ const TITLE_MAX_LENGTH = 57 @@ -191,6 +192,7 @@ export class ChatService { role: msg.role, content: msg.content, timestamp: msg.created_at.toJSDate(), + sources: msg.sources ? JSON.parse(msg.sources) : undefined, })), } } catch (error) { @@ -253,12 +255,18 @@ export class ChatService { } } - async addMessage(sessionId: number, role: 'system' | 'user' | 'assistant', content: string) { + async addMessage( + sessionId: number, + role: 'system' | 'user' | 'assistant', + content: string, + sources?: ChatSource[] + ) { try { const message = await ChatMessage.create({ session_id: sessionId, role, content, + sources: sources && sources.length > 0 ? JSON.stringify(sources) : null, }) // Update session's updated_at timestamp @@ -271,6 +279,7 @@ export class ChatService { role: message.role, content: message.content, timestamp: message.created_at.toJSDate(), + sources: sources && sources.length > 0 ? sources : undefined, } } catch (error) { logger.error( diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 2c295219..65190b5a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -979,6 +979,11 @@ export class RagService { document_id: result.payload?.document_id as string | undefined, content_type: result.payload?.content_type as string | undefined, source: result.payload?.source as string | undefined, + // Citation metadata (#1179) -- date/title of the archive a chunk was + // extracted from, when known. Undefined for non-ZIM content, which + // carries no equivalent embedded metadata. + archive_title: result.payload?.archive_title as string | undefined, + archive_date: result.payload?.archive_date as string | undefined, })) const rerankedResults = this.rerankResults(resultsWithMetadata, keywords, query) @@ -1059,6 +1064,9 @@ export class RagService { hierarchy: result.hierarchy, document_id: result.document_id, content_type: result.content_type, + // Citation metadata (#1179) + archive_title: result.archive_title, + archive_date: result.archive_date, }, })) } catch (error) { diff --git a/admin/app/utils/rag_prompt.ts b/admin/app/utils/rag_prompt.ts index 531fa2ed..83b8bf3c 100644 --- a/admin/app/utils/rag_prompt.ts +++ b/admin/app/utils/rag_prompt.ts @@ -15,6 +15,7 @@ */ import { parseParameterBillions } from './context_window.js' +import type { ChatSource } from '../../types/chat.js' export type ContextLimits = { maxResults: number; maxTokens: number } export type ContextLimitTier = { maxParams: number; maxResults: number; maxTokens: number } @@ -61,9 +62,50 @@ export function getContextLimitsForModel( export function buildContextBlock(docs: BudgetableChunk[]): string { return docs .map((doc, idx) => { - const title = doc.metadata?.full_title || doc.metadata?.article_title - const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]` + const title = + doc.metadata?.archive_title || doc.metadata?.full_title || doc.metadata?.article_title + const date = doc.metadata?.archive_date + const label = title + ? `[Context ${idx + 1} — ${title}${date ? ` (${date})` : ''}]` + : `[Context ${idx + 1}]` return `${label}\n${doc.text}` }) .join('\n\n') } + +/** + * Build the deduplicated "Sources" list shown under an assistant answer. + * + * Deliberately fed from what was *injected* into the prompt rather than from + * everything retrieval returned: a chunk dropped by the budget planner never + * reached the model, so citing it would credit the answer to a document it was + * not based on -- the exact failure the citation list exists to prevent. + * + * Dedupes on the originating path so a dozen chunks out of one archive collapse + * to one entry, falling back to the title when a point carries no path. Points + * with neither are skipped rather than shown as "Unknown source": an entry the + * user cannot act on is worse than one fewer entry. + */ +export function buildCitations(docs: BudgetableChunk[]): ChatSource[] { + const seen = new Set() + const sources: ChatSource[] = [] + + for (const doc of docs) { + const title = + doc.metadata?.archive_title || doc.metadata?.full_title || doc.metadata?.article_title + const path = doc.metadata?.source as string | undefined + const key = path || title + if (!key || seen.has(key)) continue + seen.add(key) + + sources.push({ + // A user-uploaded PDF carries no embedded title, so fall back to its + // filename -- which is what the user named it and will recognise. + title: title || path!.split('/').pop() || path!, + date: doc.metadata?.archive_date, + source: path, + }) + } + + return sources +} diff --git a/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts b/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts new file mode 100644 index 00000000..1b8f7c93 --- /dev/null +++ b/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'chat_messages' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.text('sources').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('sources') + }) + } +} diff --git a/admin/inertia/components/chat/ChatMessageBubble.tsx b/admin/inertia/components/chat/ChatMessageBubble.tsx index 010b05d0..a514f9bd 100644 --- a/admin/inertia/components/chat/ChatMessageBubble.tsx +++ b/admin/inertia/components/chat/ChatMessageBubble.tsx @@ -102,6 +102,19 @@ export default function ChatMessageBubble({ message }: ChatMessageBubbleProps) { )} + {message.role === 'assistant' && message.sources && message.sources.length > 0 && ( +
+
Sources
+
    + {message.sources.map((src, idx) => ( +
  • + {src.title} + {src.date && ({src.date})} +
    • + ))} +
+
+ )}
{ + setMessages((prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, sources } : m)) + ) + } ) } catch (error: any) { if (error?.name !== 'AbortError') { diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index a68d66cb..2b50194e 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -11,6 +11,7 @@ import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' import BenchmarkResult from '#models/benchmark_result' import { BenchmarkType, RunBenchmarkResponse, SubmitBenchmarkResponse, UpdateBuilderTagResponse } from '../../types/benchmark' +import type { ChatSource } from '../../types/chat' class API { private client: AxiosInstance @@ -320,7 +321,8 @@ class API { async streamChatMessage( chatRequest: OllamaChatRequest, onChunk: (content: string, thinking: string, done: boolean) => void, - signal?: AbortSignal + signal?: AbortSignal, + onSources?: (sources: ChatSource[]) => void ): Promise { // Axios doesn't support ReadableStream in browser, so need to use fetch const response = await fetch('/api/ollama/chat', { @@ -356,6 +358,13 @@ class API { if (data.error) throw new Error('The model encountered an error. Please try again.') + // Citation metadata (#1179) arrives as a distinct trailing event with no + // `message` key -- route it separately rather than through onChunk. + if (data.sources) { + onSources?.(data.sources) + continue + } + onChunk( data.message?.content ?? '', data.message?.thinking ?? '', diff --git a/admin/tests/unit/rag_citations.spec.ts b/admin/tests/unit/rag_citations.spec.ts new file mode 100644 index 00000000..891874e9 --- /dev/null +++ b/admin/tests/unit/rag_citations.spec.ts @@ -0,0 +1,109 @@ +/** + * The "Sources" list shown under an assistant answer (#1179). + * + * The rule these lock in is that a citation may only ever name a document the + * model actually read. Offline there is no second opinion to check an answer + * against, so a citation is the only provenance the user gets — and a wrong one + * is worse than none, because it lends false weight to a fabricated answer. + * + * Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed: + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { buildCitations } from '../../app/utils/rag_prompt.js' + +const chunk = (metadata: Record) => ({ text: 'body', score: 0.5, metadata }) + +test('buildCitations collapses many chunks from one archive into a single entry', () => { + const sources = buildCitations([ + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + ]) + + assert.equal(sources.length, 1) + assert.equal(sources[0].title, 'Survival Library') +}) + +test('buildCitations dedupes on path, not title, so same-named archives stay distinct', () => { + const sources = buildCitations([ + chunk({ source: '/zim/wikipedia_2024.zim', archive_title: 'Wikipedia' }), + chunk({ source: '/zim/wikipedia_2026.zim', archive_title: 'Wikipedia' }), + ]) + + assert.equal(sources.length, 2) + assert.deepEqual( + sources.map((s) => s.source), + ['/zim/wikipedia_2024.zim', '/zim/wikipedia_2026.zim'] + ) +}) + +test('buildCitations prefers the archive title over per-article titles', () => { + const [source] = buildCitations([ + chunk({ + source: '/zim/ifixit.zim', + archive_title: 'iFixit Repair Guides', + full_title: 'Replacing a MacBook battery', + article_title: 'MacBook battery', + }), + ]) + + assert.equal(source.title, 'iFixit Repair Guides') +}) + +test('buildCitations falls back through full_title then article_title', () => { + assert.equal( + buildCitations([chunk({ source: '/a.zim', full_title: 'Full', article_title: 'Article' })])[0] + .title, + 'Full' + ) + assert.equal( + buildCitations([chunk({ source: '/b.zim', article_title: 'Article' })])[0].title, + 'Article' + ) +}) + +test('buildCitations names an untitled upload by its filename', () => { + // A user-uploaded PDF carries no embedded metadata; the filename is what the + // user chose and is the only label they will recognise. + const [source] = buildCitations([chunk({ source: '/kb_uploads/well drilling notes.pdf' })]) + + assert.equal(source.title, 'well drilling notes.pdf') + assert.equal(source.source, '/kb_uploads/well drilling notes.pdf') +}) + +test('buildCitations carries the archive date when one is known', () => { + const [withDate] = buildCitations([ + chunk({ source: '/zim/wiki.zim', archive_title: 'Wikipedia', archive_date: '2026-06' }), + ]) + assert.equal(withDate.date, '2026-06') + + const [withoutDate] = buildCitations([chunk({ source: '/zim/wiki.zim', archive_title: 'Wikipedia' })]) + assert.equal(withoutDate.date, undefined) +}) + +test('buildCitations skips a chunk with neither a path nor a title', () => { + // "Unknown source" is not a citation — it is a row the user cannot act on, + // and it makes the answer look sourced when it is not. + assert.deepEqual(buildCitations([chunk({ chunk_index: 3 })]), []) +}) + +test('buildCitations returns nothing when no context was injected', () => { + // Retrieval skipped, or it declined because nothing cleared the relevance + // floor. No context means no citations, not an empty-looking Sources header. + assert.deepEqual(buildCitations([]), []) +}) + +test('buildCitations preserves injection order', () => { + const sources = buildCitations([ + chunk({ source: '/zim/b.zim', archive_title: 'Second' }), + chunk({ source: '/zim/a.zim', archive_title: 'First' }), + ]) + + assert.deepEqual( + sources.map((s) => s.title), + ['Second', 'First'] + ) +}) diff --git a/admin/types/chat.ts b/admin/types/chat.ts index d57ee489..73d89b0d 100644 --- a/admin/types/chat.ts +++ b/admin/types/chat.ts @@ -1,3 +1,14 @@ +/** + * A single provenance entry under an assistant answer: the document a chunk of + * injected context came from. `source` is the originating file/ZIM path and is + * what dedupes the list; `title` is what the user actually reads. + */ +export interface ChatSource { + title: string + date?: string + source?: string +} + export interface ChatMessage { id: string role: 'system' | 'user' | 'assistant' @@ -7,6 +18,7 @@ export interface ChatMessage { thinking?: string isThinking?: boolean thinkingDuration?: number + sources?: ChatSource[] } export interface ChatSession { diff --git a/admin/types/rag.ts b/admin/types/rag.ts index a75fff56..97a56c86 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -39,6 +39,8 @@ export type RAGResult = { document_id?: string content_type?: string source?: string + archive_title?: string + archive_date?: string } export type RerankedRAGResult = Omit & {