From c4487152c3ef20c4faa4182baa12aaa90ff5fa3c Mon Sep 17 00:00:00 2001 From: John Cortright Date: Sat, 29 Aug 2026 17:49:54 -0700 Subject: [PATCH] feat: show source document and date as citations under RAG chat answers Closes #1179. Surfaces the documents an answer was drawn from as a "Sources" list under the assistant message, with the archive date where one is known, and persists it so reopening a past conversation still shows what backed each answer. Offline there is no second opinion to check an answer against, so provenance is the only signal the user has that a claim came from a real document rather than from the model. This is the other half of retrieval being allowed to decline: one says when nothing relevant was found, this says what was found. Citations are built from the chunks actually injected into the prompt, not from everything retrieval returned. A chunk dropped by the budget planner never reached the model, so citing it would credit the answer to a document it was not based on -- the exact failure the list exists to prevent. The archive title and date have been written into the vector payload by the ZIM ingestion path all along and were simply never read back, so this works on already-embedded content with no re-embed. Rebased onto the RagPipelineService refactor, which landed after this was written: the citation list now comes off the pipeline trace rather than the inline retrieval block that used to live in the controller. Entries with neither a path nor a title are skipped rather than shown as "Unknown source" -- a row the user cannot act on makes an answer look sourced when it is not. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01F8Mgg1vd2eKSs8StZiuyMV --- admin/app/controllers/ollama_controller.ts | 19 ++- admin/app/models/chat_message.ts | 3 + admin/app/services/chat_service.ts | 11 +- admin/app/services/rag_service.ts | 8 ++ admin/app/utils/rag_prompt.ts | 46 +++++++- ...eate_add_sources_to_chat_messages_table.ts | 17 +++ .../components/chat/ChatMessageBubble.tsx | 13 +++ admin/inertia/components/chat/index.tsx | 8 +- admin/inertia/lib/api.ts | 11 +- admin/tests/unit/rag_citations.spec.ts | 109 ++++++++++++++++++ admin/types/chat.ts | 12 ++ admin/types/rag.ts | 2 + 12 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts create mode 100644 admin/tests/unit/rag_citations.spec.ts diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 0930cb56..61c15f59 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -14,6 +14,7 @@ import type { HttpContext } from '@adonisjs/core/http' import { SERVICE_NAMES } from '../../constants/service_names.js' import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js' import type { PipelineTrace } from '../../types/rag.js' +import { buildCitations } from '../utils/rag_prompt.js' import logger from '@adonisjs/core/services/logger' @inject() @@ -79,6 +80,11 @@ export default class OllamaController { reqData.messages = trace.messages const numCtx = trace.numCtx const numPredict = trace.numPredict + // Provenance for the answer about to be generated (#1179). Built from + // trace.injected -- what the model actually read -- and surfaced under the + // answer as "Sources". Empty whenever retrieval was skipped or declined, + // which is the honest result: no context, no citations. + const sources = buildCitations(trace.injected) // Keeping the model resident is what makes the KV cache worth building: // Ollama's default evicts after 5 minutes, which is well inside the time a // user spends reading an answer and typing the next question. @@ -145,11 +151,18 @@ export default class OllamaController { } throw err } + // Trailing citation event, written before end(). It carries no `message` + // key, which is how the client tells it apart from Ollama's own chunks. + if (sources.length > 0) { + response.response.write(`data: ${JSON.stringify({ sources })} + +`) + } response.response.end() // Save assistant message and optionally generate title if (sessionId && fullContent) { - await this.chatService.addMessage(sessionId, 'assistant', fullContent) + await this.chatService.addMessage(sessionId, 'assistant', fullContent, sources) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { this.chatService.generateTitle(sessionId, userContent, fullContent, reqData.model).catch((err) => { @@ -174,7 +187,7 @@ export default class OllamaController { } if (sessionId && result?.message?.content) { - await this.chatService.addMessage(sessionId, 'assistant', result.message.content) + await this.chatService.addMessage(sessionId, 'assistant', result.message.content, sources) const messageCount = await this.chatService.getMessageCount(sessionId) if (messageCount <= 2 && userContent) { this.chatService.generateTitle(sessionId, userContent, result.message.content, reqData.model).catch((err) => { @@ -183,7 +196,7 @@ export default class OllamaController { } } - return result + return { ...result, sources } } catch (error) { if (reqData.stream) { response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`) diff --git a/admin/app/models/chat_message.ts b/admin/app/models/chat_message.ts index f0633ffe..b7602ac7 100644 --- a/admin/app/models/chat_message.ts +++ b/admin/app/models/chat_message.ts @@ -18,6 +18,9 @@ export default class ChatMessage extends BaseModel { @column() declare content: string + @column() + declare sources: string | null + @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' }) declare session: BelongsTo diff --git a/admin/app/services/chat_service.ts b/admin/app/services/chat_service.ts index 7cc46cf2..e85fa344 100644 --- a/admin/app/services/chat_service.ts +++ b/admin/app/services/chat_service.ts @@ -15,6 +15,7 @@ import { pickTitle, resolveStructured, } from '../utils/structured_output.js' +import type { ChatSource } from '../../types/chat.js' /** Sidebar width, near enough. Applied once, to whichever candidate title won. */ const TITLE_MAX_LENGTH = 57 @@ -191,6 +192,7 @@ export class ChatService { role: msg.role, content: msg.content, timestamp: msg.created_at.toJSDate(), + sources: msg.sources ? JSON.parse(msg.sources) : undefined, })), } } catch (error) { @@ -253,12 +255,18 @@ export class ChatService { } } - async addMessage(sessionId: number, role: 'system' | 'user' | 'assistant', content: string) { + async addMessage( + sessionId: number, + role: 'system' | 'user' | 'assistant', + content: string, + sources?: ChatSource[] + ) { try { const message = await ChatMessage.create({ session_id: sessionId, role, content, + sources: sources && sources.length > 0 ? JSON.stringify(sources) : null, }) // Update session's updated_at timestamp @@ -271,6 +279,7 @@ export class ChatService { role: message.role, content: message.content, timestamp: message.created_at.toJSDate(), + sources: sources && sources.length > 0 ? sources : undefined, } } catch (error) { logger.error( diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 2c295219..65190b5a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -979,6 +979,11 @@ export class RagService { document_id: result.payload?.document_id as string | undefined, content_type: result.payload?.content_type as string | undefined, source: result.payload?.source as string | undefined, + // Citation metadata (#1179) -- date/title of the archive a chunk was + // extracted from, when known. Undefined for non-ZIM content, which + // carries no equivalent embedded metadata. + archive_title: result.payload?.archive_title as string | undefined, + archive_date: result.payload?.archive_date as string | undefined, })) const rerankedResults = this.rerankResults(resultsWithMetadata, keywords, query) @@ -1059,6 +1064,9 @@ export class RagService { hierarchy: result.hierarchy, document_id: result.document_id, content_type: result.content_type, + // Citation metadata (#1179) + archive_title: result.archive_title, + archive_date: result.archive_date, }, })) } catch (error) { diff --git a/admin/app/utils/rag_prompt.ts b/admin/app/utils/rag_prompt.ts index 531fa2ed..83b8bf3c 100644 --- a/admin/app/utils/rag_prompt.ts +++ b/admin/app/utils/rag_prompt.ts @@ -15,6 +15,7 @@ */ import { parseParameterBillions } from './context_window.js' +import type { ChatSource } from '../../types/chat.js' export type ContextLimits = { maxResults: number; maxTokens: number } export type ContextLimitTier = { maxParams: number; maxResults: number; maxTokens: number } @@ -61,9 +62,50 @@ export function getContextLimitsForModel( export function buildContextBlock(docs: BudgetableChunk[]): string { return docs .map((doc, idx) => { - const title = doc.metadata?.full_title || doc.metadata?.article_title - const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]` + const title = + doc.metadata?.archive_title || doc.metadata?.full_title || doc.metadata?.article_title + const date = doc.metadata?.archive_date + const label = title + ? `[Context ${idx + 1} — ${title}${date ? ` (${date})` : ''}]` + : `[Context ${idx + 1}]` return `${label}\n${doc.text}` }) .join('\n\n') } + +/** + * Build the deduplicated "Sources" list shown under an assistant answer. + * + * Deliberately fed from what was *injected* into the prompt rather than from + * everything retrieval returned: a chunk dropped by the budget planner never + * reached the model, so citing it would credit the answer to a document it was + * not based on -- the exact failure the citation list exists to prevent. + * + * Dedupes on the originating path so a dozen chunks out of one archive collapse + * to one entry, falling back to the title when a point carries no path. Points + * with neither are skipped rather than shown as "Unknown source": an entry the + * user cannot act on is worse than one fewer entry. + */ +export function buildCitations(docs: BudgetableChunk[]): ChatSource[] { + const seen = new Set() + const sources: ChatSource[] = [] + + for (const doc of docs) { + const title = + doc.metadata?.archive_title || doc.metadata?.full_title || doc.metadata?.article_title + const path = doc.metadata?.source as string | undefined + const key = path || title + if (!key || seen.has(key)) continue + seen.add(key) + + sources.push({ + // A user-uploaded PDF carries no embedded title, so fall back to its + // filename -- which is what the user named it and will recognise. + title: title || path!.split('/').pop() || path!, + date: doc.metadata?.archive_date, + source: path, + }) + } + + return sources +} diff --git a/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts b/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts new file mode 100644 index 00000000..1b8f7c93 --- /dev/null +++ b/admin/database/migrations/1785468975052_create_add_sources_to_chat_messages_table.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'chat_messages' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.text('sources').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('sources') + }) + } +} diff --git a/admin/inertia/components/chat/ChatMessageBubble.tsx b/admin/inertia/components/chat/ChatMessageBubble.tsx index 010b05d0..a514f9bd 100644 --- a/admin/inertia/components/chat/ChatMessageBubble.tsx +++ b/admin/inertia/components/chat/ChatMessageBubble.tsx @@ -102,6 +102,19 @@ export default function ChatMessageBubble({ message }: ChatMessageBubbleProps) { )} + {message.role === 'assistant' && message.sources && message.sources.length > 0 && ( +
+
Sources
+
    + {message.sources.map((src, idx) => ( +
  • + {src.title} + {src.date && ({src.date})} +
    • + ))} +
+
+ )}
{ + setMessages((prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, sources } : m)) + ) + } ) } catch (error: any) { if (error?.name !== 'AbortError') { diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index a68d66cb..2b50194e 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -11,6 +11,7 @@ import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' import BenchmarkResult from '#models/benchmark_result' import { BenchmarkType, RunBenchmarkResponse, SubmitBenchmarkResponse, UpdateBuilderTagResponse } from '../../types/benchmark' +import type { ChatSource } from '../../types/chat' class API { private client: AxiosInstance @@ -320,7 +321,8 @@ class API { async streamChatMessage( chatRequest: OllamaChatRequest, onChunk: (content: string, thinking: string, done: boolean) => void, - signal?: AbortSignal + signal?: AbortSignal, + onSources?: (sources: ChatSource[]) => void ): Promise { // Axios doesn't support ReadableStream in browser, so need to use fetch const response = await fetch('/api/ollama/chat', { @@ -356,6 +358,13 @@ class API { if (data.error) throw new Error('The model encountered an error. Please try again.') + // Citation metadata (#1179) arrives as a distinct trailing event with no + // `message` key -- route it separately rather than through onChunk. + if (data.sources) { + onSources?.(data.sources) + continue + } + onChunk( data.message?.content ?? '', data.message?.thinking ?? '', diff --git a/admin/tests/unit/rag_citations.spec.ts b/admin/tests/unit/rag_citations.spec.ts new file mode 100644 index 00000000..891874e9 --- /dev/null +++ b/admin/tests/unit/rag_citations.spec.ts @@ -0,0 +1,109 @@ +/** + * The "Sources" list shown under an assistant answer (#1179). + * + * The rule these lock in is that a citation may only ever name a document the + * model actually read. Offline there is no second opinion to check an answer + * against, so a citation is the only provenance the user gets — and a wrong one + * is worse than none, because it lends false weight to a fabricated answer. + * + * Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed: + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { buildCitations } from '../../app/utils/rag_prompt.js' + +const chunk = (metadata: Record) => ({ text: 'body', score: 0.5, metadata }) + +test('buildCitations collapses many chunks from one archive into a single entry', () => { + const sources = buildCitations([ + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + chunk({ source: '/zim/survival.zim', archive_title: 'Survival Library' }), + ]) + + assert.equal(sources.length, 1) + assert.equal(sources[0].title, 'Survival Library') +}) + +test('buildCitations dedupes on path, not title, so same-named archives stay distinct', () => { + const sources = buildCitations([ + chunk({ source: '/zim/wikipedia_2024.zim', archive_title: 'Wikipedia' }), + chunk({ source: '/zim/wikipedia_2026.zim', archive_title: 'Wikipedia' }), + ]) + + assert.equal(sources.length, 2) + assert.deepEqual( + sources.map((s) => s.source), + ['/zim/wikipedia_2024.zim', '/zim/wikipedia_2026.zim'] + ) +}) + +test('buildCitations prefers the archive title over per-article titles', () => { + const [source] = buildCitations([ + chunk({ + source: '/zim/ifixit.zim', + archive_title: 'iFixit Repair Guides', + full_title: 'Replacing a MacBook battery', + article_title: 'MacBook battery', + }), + ]) + + assert.equal(source.title, 'iFixit Repair Guides') +}) + +test('buildCitations falls back through full_title then article_title', () => { + assert.equal( + buildCitations([chunk({ source: '/a.zim', full_title: 'Full', article_title: 'Article' })])[0] + .title, + 'Full' + ) + assert.equal( + buildCitations([chunk({ source: '/b.zim', article_title: 'Article' })])[0].title, + 'Article' + ) +}) + +test('buildCitations names an untitled upload by its filename', () => { + // A user-uploaded PDF carries no embedded metadata; the filename is what the + // user chose and is the only label they will recognise. + const [source] = buildCitations([chunk({ source: '/kb_uploads/well drilling notes.pdf' })]) + + assert.equal(source.title, 'well drilling notes.pdf') + assert.equal(source.source, '/kb_uploads/well drilling notes.pdf') +}) + +test('buildCitations carries the archive date when one is known', () => { + const [withDate] = buildCitations([ + chunk({ source: '/zim/wiki.zim', archive_title: 'Wikipedia', archive_date: '2026-06' }), + ]) + assert.equal(withDate.date, '2026-06') + + const [withoutDate] = buildCitations([chunk({ source: '/zim/wiki.zim', archive_title: 'Wikipedia' })]) + assert.equal(withoutDate.date, undefined) +}) + +test('buildCitations skips a chunk with neither a path nor a title', () => { + // "Unknown source" is not a citation — it is a row the user cannot act on, + // and it makes the answer look sourced when it is not. + assert.deepEqual(buildCitations([chunk({ chunk_index: 3 })]), []) +}) + +test('buildCitations returns nothing when no context was injected', () => { + // Retrieval skipped, or it declined because nothing cleared the relevance + // floor. No context means no citations, not an empty-looking Sources header. + assert.deepEqual(buildCitations([]), []) +}) + +test('buildCitations preserves injection order', () => { + const sources = buildCitations([ + chunk({ source: '/zim/b.zim', archive_title: 'Second' }), + chunk({ source: '/zim/a.zim', archive_title: 'First' }), + ]) + + assert.deepEqual( + sources.map((s) => s.title), + ['Second', 'First'] + ) +}) diff --git a/admin/types/chat.ts b/admin/types/chat.ts index d57ee489..73d89b0d 100644 --- a/admin/types/chat.ts +++ b/admin/types/chat.ts @@ -1,3 +1,14 @@ +/** + * A single provenance entry under an assistant answer: the document a chunk of + * injected context came from. `source` is the originating file/ZIM path and is + * what dedupes the list; `title` is what the user actually reads. + */ +export interface ChatSource { + title: string + date?: string + source?: string +} + export interface ChatMessage { id: string role: 'system' | 'user' | 'assistant' @@ -7,6 +18,7 @@ export interface ChatMessage { thinking?: string isThinking?: boolean thinkingDuration?: number + sources?: ChatSource[] } export interface ChatSession { diff --git a/admin/types/rag.ts b/admin/types/rag.ts index a75fff56..97a56c86 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -39,6 +39,8 @@ export type RAGResult = { document_id?: string content_type?: string source?: string + archive_title?: string + archive_date?: string } export type RerankedRAGResult = Omit & {