From 8a920188debbca7baba75a3223303e78c92e185e Mon Sep 17 00:00:00 2001 From: Gabe Graves Date: Sun, 30 Aug 2026 10:47:34 -0700 Subject: [PATCH] feat(ai): support local vision attachments Closes #1176. Salvages @gabegraves's work from #1178, rebased onto the RagPipelineService refactor that landed after it was written. Adds JPEG/PNG/WebP attachments to chat over multipart, leaving text-only JSON requests on their existing path. Images are normalized server-side to bounded JPEG with metadata stripped, capped on count/size/pixels/output, and the temporary uploads are cleaned up on every validation path. They are attached only to the latest user message, are excluded from RAG, titles, logs and persisted history, and are never written to storage. Vision support is resolved per model from what the backend actually reports -- OpenAI-compatible /v1/models metadata, then Ollama /api/show, then llama.cpp /props -- never inferred from a model name. A model that reports no vision support has attachments disabled with a visible explanation; a backend that reports nothing is allowed with a warning and its rejection is translated into plain compatibility guidance rather than a raw error. Changes from the original: - Prompt assembly moved into RagPipelineService in the meantime, so the controller half is rebuilt against the trace and the multipart-aware collection filter is now the only one (dev's request.input() version would have silently dropped the collection on every image request). - Added a native-path converter. dev split chat into native and OpenAI-compat paths after this was written; Ollama's native /api/chat does not take OpenAI content parts, it takes a sibling `images` array of bare base64. Without this every image request on a native Ollama backend would fail. - Dropped the model-recommendation change (selectRecommendedModels and the Qwen3-VL fallback entry). Displacing a recommended model at first-time setup is a product decision and is being taken separately. The "Supports images" label in Easy Setup is kept: it describes what is already on offer rather than changing it. - Dropped the DEFAULT_QUERY_REWRITE_MODEL download prompt his branch carried, which dev deleted in d022c2d. - Removed a duplicate checkModelHasThinking; kept dev's memoized getModelInfo path, since thinking detection is not part of this feature. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01F8Mgg1vd2eKSs8StZiuyMV --- admin/app/controllers/ollama_controller.ts | 165 +++++++-- admin/app/services/ollama_service.ts | 112 +++++- admin/app/utils/chat_images.ts | 125 +++++++ admin/app/utils/model_capabilities.ts | 86 +++++ admin/inertia/components/InfoTooltip.tsx | 23 +- .../inertia/components/chat/ChatInterface.tsx | 163 ++++++++- .../components/chat/ChatMessageBubble.tsx | 12 + admin/inertia/components/chat/index.tsx | 16 +- admin/inertia/lib/api.ts | 50 ++- admin/inertia/lib/chat_stream.ts | 10 + admin/inertia/lib/vision_guidance.ts | 16 + admin/inertia/pages/easy-setup/index.tsx | 12 + admin/tests/unit/chat_images.spec.ts | 274 ++++++++++++++ admin/tests/unit/chat_request_compat.spec.ts | 20 + admin/tests/unit/chat_stream.spec.ts | 19 + .../tests/unit/ollama_controller_chat.spec.ts | 342 ++++++++++++++++++ admin/tests/unit/vision_guidance.spec.ts | 24 ++ admin/types/chat.ts | 8 + admin/types/ollama.ts | 5 + 19 files changed, 1427 insertions(+), 55 deletions(-) create mode 100644 admin/app/utils/chat_images.ts create mode 100644 admin/app/utils/model_capabilities.ts create mode 100644 admin/inertia/lib/chat_stream.ts create mode 100644 admin/inertia/lib/vision_guidance.ts create mode 100644 admin/tests/unit/chat_images.spec.ts create mode 100644 admin/tests/unit/chat_request_compat.spec.ts create mode 100644 admin/tests/unit/chat_stream.spec.ts create mode 100644 admin/tests/unit/ollama_controller_chat.spec.ts create mode 100644 admin/tests/unit/vision_guidance.spec.ts diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 61c15f59..858d2c74 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -16,6 +16,16 @@ import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js' import type { PipelineTrace } from '../../types/rag.js' import { buildCitations } from '../utils/rag_prompt.js' import logger from '@adonisjs/core/services/logger' +import { rm } from 'node:fs/promises' +import { + attachImagesToLatestUserMessage, + ChatImageError, + normalizeChatImages, + type NormalizedChatImage, +} from '../utils/chat_images.js' + +const unknownVisionCompatibilityMessage = (model: string) => + `NOMAD cannot confirm that "${model}" accepts images, and this image request failed. Choose a model marked "Supports images", or remove the image and try again.` @inject() export default class OllamaController { @@ -53,7 +63,78 @@ export default class OllamaController { } async chat({ request, response }: HttpContext) { - const reqData = await request.validateUsing(chatSchema) + const uploadedImages = request.files('images', { + size: '8mb', + extnames: ['jpg', 'jpeg', 'png', 'webp'], + }) + const cleanupUploadedImages = () => + Promise.all( + uploadedImages + .filter((file) => file.tmpPath) + .map((file) => rm(file.tmpPath!, { force: true }).catch(() => undefined)) + ) + let multipartPayload: unknown + if (uploadedImages.length > 0) { + const rawPayload = request.input('payload') + if (typeof rawPayload !== 'string') { + await cleanupUploadedImages() + return response.status(422).send({ message: 'Multipart chat requests require a JSON payload.' }) + } + try { + multipartPayload = JSON.parse(rawPayload) + } catch { + await cleanupUploadedImages() + return response.status(422).send({ message: 'The multipart chat payload is not valid JSON.' }) + } + } + // A multipart chat carries the collection inside the JSON payload rather than + // as a form field, so request.input() alone would silently drop it and every + // image request would search the whole knowledge base. + const rawCollection = + uploadedImages.length > 0 && + multipartPayload && + typeof multipartPayload === 'object' && + 'collection' in multipartPayload + ? (multipartPayload as { collection?: unknown }).collection + : request.input('collection', null) + const collectionFilter: string | null = + typeof rawCollection === 'string' && rawCollection ? rawCollection : null + + let reqData: Awaited> + try { + reqData = + uploadedImages.length > 0 + ? await chatSchema.validate(multipartPayload) + : await request.validateUsing(chatSchema) + } catch (error) { + await cleanupUploadedImages() + throw error + } + + let normalizedImages: NormalizedChatImage[] + try { + normalizedImages = await normalizeChatImages(uploadedImages) + } catch (error) { + if (error instanceof ChatImageError) { + return response.status(error.status).send({ message: error.message }) + } + throw error + } finally { + await cleanupUploadedImages() + } + + const modelCapabilities = await this.ollamaService.getModelCapabilities(reqData.model) + if ( + normalizedImages.length > 0 && + !reqData.messages.some((message) => message.role === 'user') + ) { + return response.status(422).send({ message: 'Images require a user message.' }) + } + if (normalizedImages.length > 0 && modelCapabilities.vision === 'unsupported') { + return response.status(422).send({ + message: `The selected model "${reqData.model}" does not support image input.`, + }) + } // Flush SSE headers immediately so the client connection is open while // pre-processing (query rewriting, RAG lookup) runs in the background. @@ -64,6 +145,7 @@ export default class OllamaController { response.response.flushHeaders() } + let unknownVisionUpstreamRejected = false try { // Everything from system-prompt assembly through query rewriting, // retrieval, context trimming and the num_ctx decision lives in @@ -72,7 +154,6 @@ export default class OllamaController { // settings, both writing rag.enabled). Unset means on, preserving the // behaviour from before the toggle existed. const ragEnabled = (await KVStore.getValue('rag.enabled')) ?? true - const collectionFilter: string | null = request.input('collection', null) const trace = await this.ragPipelineService.buildPrompt(reqData.messages, reqData.model, { collection: collectionFilter ?? undefined, skipRetrieval: !ragEnabled, @@ -94,7 +175,7 @@ export default class OllamaController { // Thinking is only enabled when the model supports it AND the user wants it: the explicit // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF). // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat - const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model) + const thinkingCapability = modelCapabilities.thinking let thinkingEnabled = false if (thinkingCapability) { thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false) @@ -105,6 +186,10 @@ export default class OllamaController { // Separate sessionId and the resolved thinking preference from the Ollama request payload — // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw). const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData + const upstreamMessages = attachImagesToLatestUserMessage( + ollamaRequest.messages, + normalizedImages + ) // Save user message to DB before streaming if sessionId provided let userContent: string | null = null @@ -124,17 +209,18 @@ export default class OllamaController { // blocks every later chat/RAG request until the model is manually stopped (#1065). const abortController = new AbortController() response.response.on('close', () => abortController.abort()) - const stream = await this.ollamaService.chatStream({ - ...ollamaRequest, - think, - thinkingCapable: thinkingCapability, - numCtx, - numPredict, - keepAlive, - signal: abortController.signal, - }) let fullContent = '' try { + const stream = await this.ollamaService.chatStream({ + ...ollamaRequest, + messages: upstreamMessages, + think, + thinkingCapable: thinkingCapability, + numCtx, + numPredict, + keepAlive, + signal: abortController.signal, + }) for await (const chunk of stream) { if (chunk.message?.content) { fullContent += chunk.message.content @@ -149,6 +235,8 @@ export default class OllamaController { logger.debug('[OllamaController] Client disconnected; aborted upstream Ollama generation') return } + unknownVisionUpstreamRejected = + normalizedImages.length > 0 && modelCapabilities.vision === 'unknown' throw err } // Trailing citation event, written before end(). It carries no `message` @@ -174,14 +262,25 @@ export default class OllamaController { } // Non-streaming (legacy) path - const result = await this.ollamaService.chat({ - ...ollamaRequest, - think, - thinkingCapable: thinkingCapability, - numCtx, - numPredict, - keepAlive, - }) + let result + try { + result = await this.ollamaService.chat({ + ...ollamaRequest, + messages: upstreamMessages, + think, + thinkingCapable: thinkingCapability, + numCtx, + numPredict, + keepAlive, + }) + } catch (err) { + // A backend that never advertised vision support rejecting a request that + // carried images is the signal that it cannot see them. Recorded here and + // translated into the compatibility message by the outer catch. + unknownVisionUpstreamRejected = + normalizedImages.length > 0 && modelCapabilities.vision === 'unknown' + throw err + } if (result?.usage) { this._recordUsage(reqData.model, trace, result.usage) } @@ -199,10 +298,22 @@ export default class OllamaController { return { ...result, sources } } catch (error) { if (reqData.stream) { - response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`) + const streamError = + unknownVisionUpstreamRejected + ? { + error: true, + message: unknownVisionCompatibilityMessage(reqData.model), + } + : { error: true } + response.response.write(`data: ${JSON.stringify(streamError)}\n\n`) response.response.end() return } + if (unknownVisionUpstreamRejected) { + return response.status(422).send({ + message: unknownVisionCompatibilityMessage(reqData.model), + }) + } throw error } } @@ -392,15 +503,13 @@ export default class OllamaController { } } - async installedModels({ }: HttpContext) { + async installedModels({}: HttpContext) { const models = await this.ollamaService.getModels() - // Enrich each model with its thinking capability so the chat picker knows which models - // to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show - // results, so this stays cheap on repeat loads. Best-effort per model. - const thinking = await Promise.all( - models.map((m) => this.ollamaService.checkModelHasThinking(m.name)) + // Enrich from backend-reported capabilities; never guess from model names. + const capabilities = await Promise.all( + models.map((m) => this.ollamaService.getModelCapabilities(m.name, m)) ) - return models.map((m, i) => ({ ...m, thinking: thinking[i] })) + return models.map((m, i) => ({ ...m, ...capabilities[i] })) } } diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index b85ed8d0..ca39af93 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -19,6 +19,12 @@ import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' import env from '#start/env' import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js' import KVStore from '#models/kv_store' +import type { ModelCapabilities } from '../utils/model_capabilities.js' +import { + capabilitiesFromOllamaShow, + installedModelsFromOpenAIResponse, + resolveModelCapabilities, +} from '../utils/model_capabilities.js' const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models' const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json') @@ -57,6 +63,7 @@ export type NomadInstalledModel = { size: number digest?: string details?: Record + capabilities?: string[] } /** @@ -103,7 +110,7 @@ export type NomadChatStreamChunk = { type ChatInput = { model: string - messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> + messages: ChatCompletionMessageParam[] think?: boolean | 'medium' // Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but // disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing). @@ -134,6 +141,43 @@ type ChatInput = { signal?: AbortSignal } +/** + * Ollama's native /api/chat does not take OpenAI content parts. It carries images + * as a sibling `images` array of bare base64 (no `data:` prefix) on the message, + * with `content` staying a plain string. The OpenAI-compat path takes the parts + * as-is, so the conversion only belongs on the native side. + */ +function toNativeMessages(messages: ChatInput['messages']): Array<{ + role: string + content: string + images?: string[] +}> { + return messages.map((message) => { + const content = (message as { content?: unknown }).content + if (!Array.isArray(content)) { + return { role: message.role as string, content: (content as string) ?? '' } + } + + const text: string[] = [] + const images: string[] = [] + for (const part of content as Array>) { + if (part?.type === 'text' && typeof part.text === 'string') { + text.push(part.text) + } else if (part?.type === 'image_url' && typeof part.image_url?.url === 'string') { + // Strip the data URL wrapper; Ollama wants the payload only. + images.push(part.image_url.url.replace(/^data:[^;]+;base64,/, '')) + } + } + + const out: { role: string; content: string; images?: string[] } = { + role: message.role as string, + content: text.join('\n'), + } + if (images.length > 0) out.images = images + return out + }) +} + @inject() export class OllamaService { private openai: OpenAI | null = null @@ -147,6 +191,12 @@ export class OllamaService { // trained context length don't change at runtime. Only successful lookups are cached; // transient failures are left uncached so they can be retried. private modelInfoCache: Map = new Map() + // Definitive capabilities are stable for a loaded model. Cache unknown results briefly to + // avoid adding repeated endpoint probes to chat startup while still allowing backend reloads. + private modelCapabilityCache: Map< + string, + { value: ModelCapabilities; expiresAt: number } + > = new Map() constructor() {} @@ -570,7 +620,7 @@ export class OllamaService { const response = await this.ollama.chat({ model: chatRequest.model, - messages: chatRequest.messages, + messages: toNativeMessages(chatRequest.messages) as any, stream: false, // Definedness, not truthiness: `think: false` has to reach Ollama, which // otherwise leaves thinking ON for a capable model. Unset still sends @@ -660,7 +710,7 @@ export class OllamaService { const iterator = await this.ollama.chat({ model: chatRequest.model, - messages: chatRequest.messages, + messages: toNativeMessages(chatRequest.messages) as any, stream: true, // See _chatNative: `think: false` must be sent explicitly, or a thinking-capable // model keeps reasoning regardless of the user's ai.autoThinking preference. @@ -816,6 +866,60 @@ export class OllamaService { } } + public async getModelCapabilities( + modelName: string, + advertisedMetadata?: unknown + ): Promise { + await this._ensureDependencies() + if (!this.baseUrl) return { thinking: false, vision: 'unknown' } + + // Composite OpenAI-compatible routers can advertise different capabilities for each model. + // Prefer that per-model metadata before probing backend-specific endpoints. + const advertised = capabilitiesFromOllamaShow(advertisedMetadata) + if (advertised) { + this.modelCapabilityCache.set(modelName, { + value: advertised, + expiresAt: Number.POSITIVE_INFINITY, + }) + return advertised + } + + const cached = this.modelCapabilityCache.get(modelName) + if (cached && cached.expiresAt > Date.now()) return cached.value + + // Probe per model instead of relying on the backend-wide classification from getModels(). + // Hybrid routers may expose /v1/models plus /api/show without exposing /api/tags. + const detected = await resolveModelCapabilities(null, { + ollamaShow: async () => { + const response = await axios.post( + `${this.baseUrl}/api/show`, + { model: modelName }, + { timeout: 3000 } + ) + return response.data + }, + // A direct llama.cpp server reports the loaded model's input modalities at /props. + llamaProps: async () => { + const response = await axios.get(`${this.baseUrl}/props`, { timeout: 3000 }) + return response.data + }, + }) + if (detected) { + this.modelCapabilityCache.set(modelName, { + value: detected, + expiresAt: Number.POSITIVE_INFINITY, + }) + return detected + } + + const unknown: ModelCapabilities = { thinking: false, vision: 'unknown' } + this.modelCapabilityCache.set(modelName, { + value: unknown, + expiresAt: Date.now() + 30_000, + }) + return unknown + } + public async checkModelHasThinking(modelName: string): Promise { return (await this.getModelInfo(modelName)).hasThinking } @@ -1134,7 +1238,7 @@ export class OllamaService { logger.info('[OllamaService] /api/tags unavailable, falling back to /v1/models') try { const modelList = await this.openai!.models.list() - const models: NomadInstalledModel[] = modelList.data.map((m) => ({ name: m.id, size: 0 })) + const models: NomadInstalledModel[] = installedModelsFromOpenAIResponse(modelList) if (includeEmbeddings) return models return models.filter((m) => !m.name.includes('embed')) } catch (err) { diff --git a/admin/app/utils/chat_images.ts b/admin/app/utils/chat_images.ts new file mode 100644 index 00000000..d26e665d --- /dev/null +++ b/admin/app/utils/chat_images.ts @@ -0,0 +1,125 @@ +import type { MultipartFile } from '@adonisjs/bodyparser/types' +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js' +import sharp from 'sharp' + +export const MAX_CHAT_IMAGES = 4 +export const MAX_CHAT_IMAGE_BYTES = 8 * 1024 * 1024 +export const MAX_CHAT_IMAGE_PIXELS = 40_000_000 +export const MAX_NORMALIZED_IMAGE_BYTES = 4 * 1024 * 1024 +export const MAX_NORMALIZED_IMAGE_DIMENSION = 2048 + +const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp']) + +export class ChatImageError extends Error { + constructor( + message: string, + public readonly status: 413 | 415 | 422 + ) { + super(message) + this.name = 'ChatImageError' + } +} + +export type NormalizedChatImage = { + name: string + dataUrl: string +} + +export async function normalizeChatImages(files: MultipartFile[]): Promise { + if (files.length > MAX_CHAT_IMAGES) { + throw new ChatImageError(`Attach no more than ${MAX_CHAT_IMAGES} images.`, 422) + } + + const normalized: NormalizedChatImage[] = [] + for (const file of files) { + normalized.push(await normalizeChatImage(file)) + } + return normalized +} + +async function normalizeChatImage(file: MultipartFile): Promise { + if (!file.tmpPath) { + throw new ChatImageError(`Could not process "${file.clientName}".`, 422) + } + if (file.size > MAX_CHAT_IMAGE_BYTES) { + throw new ChatImageError(`"${file.clientName}" exceeds the 8 MB per-image limit.`, 413) + } + if (!file.isValid) { + throw new ChatImageError(`"${file.clientName}" is not a valid image upload.`, 422) + } + + try { + const pipeline = sharp(file.tmpPath, { + animated: false, + failOn: 'warning', + limitInputPixels: MAX_CHAT_IMAGE_PIXELS, + }) + const metadata = await pipeline.metadata() + + if (!metadata.format || !SUPPORTED_IMAGE_FORMATS.has(metadata.format)) { + throw new ChatImageError( + `"${file.clientName}" is not supported. Use JPEG, PNG, or WebP.`, + 415 + ) + } + if ((metadata.pages ?? 1) > 1) { + throw new ChatImageError(`Animated images are not supported.`, 415) + } + + const normalized = await pipeline + .rotate() + .resize({ + width: MAX_NORMALIZED_IMAGE_DIMENSION, + height: MAX_NORMALIZED_IMAGE_DIMENSION, + fit: 'inside', + withoutEnlargement: true, + }) + .flatten({ background: '#ffffff' }) + .jpeg({ quality: 85 }) + .toBuffer() + + if (normalized.byteLength > MAX_NORMALIZED_IMAGE_BYTES) { + throw new ChatImageError( + `"${file.clientName}" remains too large after image processing.`, + 413 + ) + } + + return { + name: file.clientName, + dataUrl: `data:image/jpeg;base64,${normalized.toString('base64')}`, + } + } catch (error) { + if (error instanceof ChatImageError) throw error + throw new ChatImageError(`"${file.clientName}" could not be decoded as an image.`, 422) + } +} + +export function attachImagesToLatestUserMessage( + messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, + images: NormalizedChatImage[] +): ChatCompletionMessageParam[] { + if (images.length === 0) return messages + + let latestUserIndex = -1 + messages.forEach((message, index) => { + if (message.role === 'user') latestUserIndex = index + }) + if (latestUserIndex < 0) { + throw new ChatImageError('Images require a user message.', 422) + } + + return messages.map((message, index) => { + if (index !== latestUserIndex) return message + return { + role: 'user', + content: [ + { type: 'text', text: message.content }, + ...images.map((image) => ({ + type: 'image_url' as const, + image_url: { url: image.dataUrl }, + })), + ], + } + }) +} diff --git a/admin/app/utils/model_capabilities.ts b/admin/app/utils/model_capabilities.ts new file mode 100644 index 00000000..c709b997 --- /dev/null +++ b/admin/app/utils/model_capabilities.ts @@ -0,0 +1,86 @@ +import type { ModelVisionCapability } from '../../types/ollama.js' + +export type ModelCapabilities = { + thinking: boolean + vision: ModelVisionCapability +} + +export type ModelCapabilityProbes = { + ollamaShow: () => Promise + llamaProps: () => Promise +} + +export type InstalledModelMetadata = { + name: string + size: number + capabilities?: string[] +} + +export function capabilitiesFromOllamaShow(value: unknown): ModelCapabilities | null { + if (!value || typeof value !== 'object') return null + const capabilities = (value as { capabilities?: unknown }).capabilities + if (!Array.isArray(capabilities) || !capabilities.every((item) => typeof item === 'string')) { + return null + } + + return { + thinking: capabilities.includes('thinking'), + vision: capabilities.includes('vision') ? 'supported' : 'unsupported', + } +} + +export function visionCapabilityFromLlamaProps(value: unknown): ModelVisionCapability | null { + if (!value || typeof value !== 'object') return null + const modalities = (value as { modalities?: unknown }).modalities + if (modalities && typeof modalities === 'object' && !Array.isArray(modalities)) { + const vision = (modalities as { vision?: unknown }).vision + return typeof vision === 'boolean' ? (vision ? 'supported' : 'unsupported') : null + } + if (Array.isArray(modalities) && modalities.every((item) => typeof item === 'string')) { + return modalities.includes('image') || modalities.includes('vision') + ? 'supported' + : 'unsupported' + } + return null +} + +export async function resolveModelCapabilities( + advertisedMetadata: unknown, + probes: ModelCapabilityProbes +): Promise { + const advertised = capabilitiesFromOllamaShow(advertisedMetadata) + if (advertised) return advertised + + try { + const ollama = capabilitiesFromOllamaShow(await probes.ollamaShow()) + if (ollama) return ollama + } catch {} + + try { + const vision = visionCapabilityFromLlamaProps(await probes.llamaProps()) + if (vision) return { thinking: false, vision } + } catch {} + + return null +} + +export function installedModelsFromOpenAIResponse(value: unknown): InstalledModelMetadata[] { + if (!value || typeof value !== 'object') return [] + const data = (value as { data?: unknown }).data + if (!Array.isArray(data)) return [] + + return data.flatMap((entry) => { + if (!entry || typeof entry !== 'object') return [] + const { id, capabilities } = entry as { id?: unknown; capabilities?: unknown } + if (typeof id !== 'string') return [] + + const model: InstalledModelMetadata = { name: id, size: 0 } + if ( + Array.isArray(capabilities) && + capabilities.every((capability) => typeof capability === 'string') + ) { + model.capabilities = capabilities + } + return [model] + }) +} diff --git a/admin/inertia/components/InfoTooltip.tsx b/admin/inertia/components/InfoTooltip.tsx index 172ee5ab..6b384d06 100644 --- a/admin/inertia/components/InfoTooltip.tsx +++ b/admin/inertia/components/InfoTooltip.tsx @@ -7,10 +7,9 @@ interface InfoTooltipProps { // Which side of the icon the tooltip pops toward. Defaults to 'top' (existing behavior); // use 'bottom' when the icon sits near the top of the viewport so it isn't clipped. position?: 'top' | 'bottom' - // Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'right' when - // the icon sits near the right edge so the bubble expands leftward into open space instead of - // being squeezed against the viewport edge (which forces one-word-per-line wrapping). - align?: 'center' | 'right' + // Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'left' near the + // left edge so the bubble expands rightward, or 'right' near the right edge so it expands leftward. + align?: 'center' | 'left' | 'right' } export default function InfoTooltip({ @@ -37,12 +36,16 @@ export default function InfoTooltip({ {isVisible && (
{text} @@ -51,7 +54,13 @@ export default function InfoTooltip({ position === 'bottom' ? 'bottom-full border-b-desert-stone-dark' : 'top-full border-t-desert-stone-dark' - } ${align === 'right' ? 'right-3' : 'left-1/2 -translate-x-1/2'}`} + } ${ + align === 'left' + ? 'left-3' + : align === 'right' + ? 'right-3' + : 'left-1/2 -translate-x-1/2' + }`} />
diff --git a/admin/inertia/components/chat/ChatInterface.tsx b/admin/inertia/components/chat/ChatInterface.tsx index 4e983f3c..b9c665a1 100644 --- a/admin/inertia/components/chat/ChatInterface.tsx +++ b/admin/inertia/components/chat/ChatInterface.tsx @@ -1,24 +1,33 @@ -import { IconSend, IconWand } from '@tabler/icons-react' +import { IconPhoto, IconSend, IconWand, IconX } from '@tabler/icons-react' import { useState, useRef, useEffect } from 'react' import classNames from '~/lib/classNames' -import { ChatMessage } from '../../../types/chat' +import { ChatImageAttachment, ChatMessage } from '../../../types/chat' +import type { ModelVisionCapability } from '../../../types/ollama' import ChatMessageBubble from './ChatMessageBubble' import ChatAssistantAvatar from './ChatAssistantAvatar' import BouncingDots from '../BouncingDots' import { usePage } from '@inertiajs/react' +import InfoTooltip from '../InfoTooltip' +import { visionAttachmentGuidance } from '../../lib/vision_guidance' interface ChatInterfaceProps { messages: ChatMessage[] - onSendMessage: (message: string) => void + onSendMessage: (message: string, images: ChatImageAttachment[]) => void + visionCapability: ModelVisionCapability isLoading?: boolean chatSuggestions?: string[] chatSuggestionsEnabled?: boolean chatSuggestionsLoading?: boolean } +const MAX_VISION_IMAGES = 4 +const MAX_VISION_IMAGE_BYTES = 8 * 1024 * 1024 +const SUPPORTED_VISION_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']) + export default function ChatInterface({ messages, onSendMessage, + visionCapability, isLoading = false, chatSuggestions = [], chatSuggestionsEnabled = false, @@ -26,8 +35,30 @@ export default function ChatInterface({ }: ChatInterfaceProps) { const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props const [input, setInput] = useState('') + const [images, setImages] = useState([]) const messagesEndRef = useRef(null) const textareaRef = useRef(null) + const imageInputRef = useRef(null) + const previewUrlsRef = useRef>(new Set()) + + useEffect( + () => () => { + previewUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)) + previewUrlsRef.current.clear() + }, + [] + ) + + useEffect(() => { + if (visionCapability !== 'unsupported') return + setImages((current) => { + current.forEach((image) => { + URL.revokeObjectURL(image.previewUrl) + previewUrlsRef.current.delete(image.previewUrl) + }) + return [] + }) + }, [visionCapability]) const scrollToBottom = () => { messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) @@ -39,15 +70,70 @@ export default function ChatInterface({ const handleSubmit = (e: React.FormEvent) => { e.preventDefault() - if (input.trim() && !isLoading) { - onSendMessage(input.trim()) + if ((input.trim() || images.length > 0) && !isLoading) { + onSendMessage(input.trim() || 'Describe the attached image.', images) setInput('') + setImages([]) + if (imageInputRef.current) imageInputRef.current.value = '' if (textareaRef.current) { textareaRef.current.style.height = 'auto' } } } + const handleImageSelection = (event: React.ChangeEvent) => { + const selected = Array.from(event.target.files ?? []) + event.target.value = '' + if (selected.length === 0) return + + const availableSlots = MAX_VISION_IMAGES - images.length + if (availableSlots <= 0) { + addNotification({ type: 'error', message: `You can attach up to ${MAX_VISION_IMAGES} images.` }) + return + } + + const attachments: ChatImageAttachment[] = [] + for (const file of selected.slice(0, availableSlots)) { + if (!SUPPORTED_VISION_TYPES.has(file.type)) { + addNotification({ + type: 'error', + message: `${file.name} is not supported. Use JPEG, PNG, or WebP.`, + }) + continue + } + if (file.size > MAX_VISION_IMAGE_BYTES) { + addNotification({ + type: 'error', + message: `${file.name} exceeds the 8 MB per-image limit.`, + }) + continue + } + + const previewUrl = URL.createObjectURL(file) + previewUrlsRef.current.add(previewUrl) + attachments.push({ + id: crypto.randomUUID(), + name: file.name, + file, + previewUrl, + }) + } + + if (selected.length > availableSlots) { + addNotification({ + type: 'error', + message: `Only the first ${availableSlots} selected image(s) were attached.`, + }) + } + setImages((current) => [...current, ...attachments]) + } + + const removeImage = (image: ChatImageAttachment) => { + URL.revokeObjectURL(image.previewUrl) + previewUrlsRef.current.delete(image.previewUrl) + setImages((current) => current.filter((item) => item.id !== image.id)) + } + const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault() @@ -139,7 +225,67 @@ export default function ChatInterface({ )}
+ {images.length > 0 && ( +
+ {images.map((image) => ( +
+ {image.name} + +
+ ))} +
+ )}
+ +
+ + {visionCapability !== 'supported' && ( + + )} +