diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 61c15f59..858d2c74 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -16,6 +16,16 @@ import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js' import type { PipelineTrace } from '../../types/rag.js' import { buildCitations } from '../utils/rag_prompt.js' import logger from '@adonisjs/core/services/logger' +import { rm } from 'node:fs/promises' +import { + attachImagesToLatestUserMessage, + ChatImageError, + normalizeChatImages, + type NormalizedChatImage, +} from '../utils/chat_images.js' + +const unknownVisionCompatibilityMessage = (model: string) => + `NOMAD cannot confirm that "${model}" accepts images, and this image request failed. Choose a model marked "Supports images", or remove the image and try again.` @inject() export default class OllamaController { @@ -53,7 +63,78 @@ export default class OllamaController { } async chat({ request, response }: HttpContext) { - const reqData = await request.validateUsing(chatSchema) + const uploadedImages = request.files('images', { + size: '8mb', + extnames: ['jpg', 'jpeg', 'png', 'webp'], + }) + const cleanupUploadedImages = () => + Promise.all( + uploadedImages + .filter((file) => file.tmpPath) + .map((file) => rm(file.tmpPath!, { force: true }).catch(() => undefined)) + ) + let multipartPayload: unknown + if (uploadedImages.length > 0) { + const rawPayload = request.input('payload') + if (typeof rawPayload !== 'string') { + await cleanupUploadedImages() + return response.status(422).send({ message: 'Multipart chat requests require a JSON payload.' }) + } + try { + multipartPayload = JSON.parse(rawPayload) + } catch { + await cleanupUploadedImages() + return response.status(422).send({ message: 'The multipart chat payload is not valid JSON.' }) + } + } + // A multipart chat carries the collection inside the JSON payload rather than + // as a form field, so request.input() alone would silently drop it and every + // image request would search the whole knowledge base. + const rawCollection = + uploadedImages.length > 0 && + multipartPayload && + typeof multipartPayload === 'object' && + 'collection' in multipartPayload + ? (multipartPayload as { collection?: unknown }).collection + : request.input('collection', null) + const collectionFilter: string | null = + typeof rawCollection === 'string' && rawCollection ? rawCollection : null + + let reqData: Awaited> + try { + reqData = + uploadedImages.length > 0 + ? await chatSchema.validate(multipartPayload) + : await request.validateUsing(chatSchema) + } catch (error) { + await cleanupUploadedImages() + throw error + } + + let normalizedImages: NormalizedChatImage[] + try { + normalizedImages = await normalizeChatImages(uploadedImages) + } catch (error) { + if (error instanceof ChatImageError) { + return response.status(error.status).send({ message: error.message }) + } + throw error + } finally { + await cleanupUploadedImages() + } + + const modelCapabilities = await this.ollamaService.getModelCapabilities(reqData.model) + if ( + normalizedImages.length > 0 && + !reqData.messages.some((message) => message.role === 'user') + ) { + return response.status(422).send({ message: 'Images require a user message.' }) + } + if (normalizedImages.length > 0 && modelCapabilities.vision === 'unsupported') { + return response.status(422).send({ + message: `The selected model "${reqData.model}" does not support image input.`, + }) + } // Flush SSE headers immediately so the client connection is open while // pre-processing (query rewriting, RAG lookup) runs in the background. @@ -64,6 +145,7 @@ export default class OllamaController { response.response.flushHeaders() } + let unknownVisionUpstreamRejected = false try { // Everything from system-prompt assembly through query rewriting, // retrieval, context trimming and the num_ctx decision lives in @@ -72,7 +154,6 @@ export default class OllamaController { // settings, both writing rag.enabled). Unset means on, preserving the // behaviour from before the toggle existed. const ragEnabled = (await KVStore.getValue('rag.enabled')) ?? true - const collectionFilter: string | null = request.input('collection', null) const trace = await this.ragPipelineService.buildPrompt(reqData.messages, reqData.model, { collection: collectionFilter ?? undefined, skipRetrieval: !ragEnabled, @@ -94,7 +175,7 @@ export default class OllamaController { // Thinking is only enabled when the model supports it AND the user wants it: the explicit // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF). // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat - const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model) + const thinkingCapability = modelCapabilities.thinking let thinkingEnabled = false if (thinkingCapability) { thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false) @@ -105,6 +186,10 @@ export default class OllamaController { // Separate sessionId and the resolved thinking preference from the Ollama request payload — // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw). const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData + const upstreamMessages = attachImagesToLatestUserMessage( + ollamaRequest.messages, + normalizedImages + ) // Save user message to DB before streaming if sessionId provided let userContent: string | null = null @@ -124,17 +209,18 @@ export default class OllamaController { // blocks every later chat/RAG request until the model is manually stopped (#1065). const abortController = new AbortController() response.response.on('close', () => abortController.abort()) - const stream = await this.ollamaService.chatStream({ - ...ollamaRequest, - think, - thinkingCapable: thinkingCapability, - numCtx, - numPredict, - keepAlive, - signal: abortController.signal, - }) let fullContent = '' try { + const stream = await this.ollamaService.chatStream({ + ...ollamaRequest, + messages: upstreamMessages, + think, + thinkingCapable: thinkingCapability, + numCtx, + numPredict, + keepAlive, + signal: abortController.signal, + }) for await (const chunk of stream) { if (chunk.message?.content) { fullContent += chunk.message.content @@ -149,6 +235,8 @@ export default class OllamaController { logger.debug('[OllamaController] Client disconnected; aborted upstream Ollama generation') return } + unknownVisionUpstreamRejected = + normalizedImages.length > 0 && modelCapabilities.vision === 'unknown' throw err } // Trailing citation event, written before end(). It carries no `message` @@ -174,14 +262,25 @@ export default class OllamaController { } // Non-streaming (legacy) path - const result = await this.ollamaService.chat({ - ...ollamaRequest, - think, - thinkingCapable: thinkingCapability, - numCtx, - numPredict, - keepAlive, - }) + let result + try { + result = await this.ollamaService.chat({ + ...ollamaRequest, + messages: upstreamMessages, + think, + thinkingCapable: thinkingCapability, + numCtx, + numPredict, + keepAlive, + }) + } catch (err) { + // A backend that never advertised vision support rejecting a request that + // carried images is the signal that it cannot see them. Recorded here and + // translated into the compatibility message by the outer catch. + unknownVisionUpstreamRejected = + normalizedImages.length > 0 && modelCapabilities.vision === 'unknown' + throw err + } if (result?.usage) { this._recordUsage(reqData.model, trace, result.usage) } @@ -199,10 +298,22 @@ export default class OllamaController { return { ...result, sources } } catch (error) { if (reqData.stream) { - response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`) + const streamError = + unknownVisionUpstreamRejected + ? { + error: true, + message: unknownVisionCompatibilityMessage(reqData.model), + } + : { error: true } + response.response.write(`data: ${JSON.stringify(streamError)}\n\n`) response.response.end() return } + if (unknownVisionUpstreamRejected) { + return response.status(422).send({ + message: unknownVisionCompatibilityMessage(reqData.model), + }) + } throw error } } @@ -392,15 +503,13 @@ export default class OllamaController { } } - async installedModels({ }: HttpContext) { + async installedModels({}: HttpContext) { const models = await this.ollamaService.getModels() - // Enrich each model with its thinking capability so the chat picker knows which models - // to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show - // results, so this stays cheap on repeat loads. Best-effort per model. - const thinking = await Promise.all( - models.map((m) => this.ollamaService.checkModelHasThinking(m.name)) + // Enrich from backend-reported capabilities; never guess from model names. + const capabilities = await Promise.all( + models.map((m) => this.ollamaService.getModelCapabilities(m.name, m)) ) - return models.map((m, i) => ({ ...m, thinking: thinking[i] })) + return models.map((m, i) => ({ ...m, ...capabilities[i] })) } } diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index b85ed8d0..ca39af93 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -19,6 +19,12 @@ import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' import env from '#start/env' import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js' import KVStore from '#models/kv_store' +import type { ModelCapabilities } from '../utils/model_capabilities.js' +import { + capabilitiesFromOllamaShow, + installedModelsFromOpenAIResponse, + resolveModelCapabilities, +} from '../utils/model_capabilities.js' const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models' const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json') @@ -57,6 +63,7 @@ export type NomadInstalledModel = { size: number digest?: string details?: Record + capabilities?: string[] } /** @@ -103,7 +110,7 @@ export type NomadChatStreamChunk = { type ChatInput = { model: string - messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> + messages: ChatCompletionMessageParam[] think?: boolean | 'medium' // Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but // disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing). @@ -134,6 +141,43 @@ type ChatInput = { signal?: AbortSignal } +/** + * Ollama's native /api/chat does not take OpenAI content parts. It carries images + * as a sibling `images` array of bare base64 (no `data:` prefix) on the message, + * with `content` staying a plain string. The OpenAI-compat path takes the parts + * as-is, so the conversion only belongs on the native side. + */ +function toNativeMessages(messages: ChatInput['messages']): Array<{ + role: string + content: string + images?: string[] +}> { + return messages.map((message) => { + const content = (message as { content?: unknown }).content + if (!Array.isArray(content)) { + return { role: message.role as string, content: (content as string) ?? '' } + } + + const text: string[] = [] + const images: string[] = [] + for (const part of content as Array>) { + if (part?.type === 'text' && typeof part.text === 'string') { + text.push(part.text) + } else if (part?.type === 'image_url' && typeof part.image_url?.url === 'string') { + // Strip the data URL wrapper; Ollama wants the payload only. + images.push(part.image_url.url.replace(/^data:[^;]+;base64,/, '')) + } + } + + const out: { role: string; content: string; images?: string[] } = { + role: message.role as string, + content: text.join('\n'), + } + if (images.length > 0) out.images = images + return out + }) +} + @inject() export class OllamaService { private openai: OpenAI | null = null @@ -147,6 +191,12 @@ export class OllamaService { // trained context length don't change at runtime. Only successful lookups are cached; // transient failures are left uncached so they can be retried. private modelInfoCache: Map = new Map() + // Definitive capabilities are stable for a loaded model. Cache unknown results briefly to + // avoid adding repeated endpoint probes to chat startup while still allowing backend reloads. + private modelCapabilityCache: Map< + string, + { value: ModelCapabilities; expiresAt: number } + > = new Map() constructor() {} @@ -570,7 +620,7 @@ export class OllamaService { const response = await this.ollama.chat({ model: chatRequest.model, - messages: chatRequest.messages, + messages: toNativeMessages(chatRequest.messages) as any, stream: false, // Definedness, not truthiness: `think: false` has to reach Ollama, which // otherwise leaves thinking ON for a capable model. Unset still sends @@ -660,7 +710,7 @@ export class OllamaService { const iterator = await this.ollama.chat({ model: chatRequest.model, - messages: chatRequest.messages, + messages: toNativeMessages(chatRequest.messages) as any, stream: true, // See _chatNative: `think: false` must be sent explicitly, or a thinking-capable // model keeps reasoning regardless of the user's ai.autoThinking preference. @@ -816,6 +866,60 @@ export class OllamaService { } } + public async getModelCapabilities( + modelName: string, + advertisedMetadata?: unknown + ): Promise { + await this._ensureDependencies() + if (!this.baseUrl) return { thinking: false, vision: 'unknown' } + + // Composite OpenAI-compatible routers can advertise different capabilities for each model. + // Prefer that per-model metadata before probing backend-specific endpoints. + const advertised = capabilitiesFromOllamaShow(advertisedMetadata) + if (advertised) { + this.modelCapabilityCache.set(modelName, { + value: advertised, + expiresAt: Number.POSITIVE_INFINITY, + }) + return advertised + } + + const cached = this.modelCapabilityCache.get(modelName) + if (cached && cached.expiresAt > Date.now()) return cached.value + + // Probe per model instead of relying on the backend-wide classification from getModels(). + // Hybrid routers may expose /v1/models plus /api/show without exposing /api/tags. + const detected = await resolveModelCapabilities(null, { + ollamaShow: async () => { + const response = await axios.post( + `${this.baseUrl}/api/show`, + { model: modelName }, + { timeout: 3000 } + ) + return response.data + }, + // A direct llama.cpp server reports the loaded model's input modalities at /props. + llamaProps: async () => { + const response = await axios.get(`${this.baseUrl}/props`, { timeout: 3000 }) + return response.data + }, + }) + if (detected) { + this.modelCapabilityCache.set(modelName, { + value: detected, + expiresAt: Number.POSITIVE_INFINITY, + }) + return detected + } + + const unknown: ModelCapabilities = { thinking: false, vision: 'unknown' } + this.modelCapabilityCache.set(modelName, { + value: unknown, + expiresAt: Date.now() + 30_000, + }) + return unknown + } + public async checkModelHasThinking(modelName: string): Promise { return (await this.getModelInfo(modelName)).hasThinking } @@ -1134,7 +1238,7 @@ export class OllamaService { logger.info('[OllamaService] /api/tags unavailable, falling back to /v1/models') try { const modelList = await this.openai!.models.list() - const models: NomadInstalledModel[] = modelList.data.map((m) => ({ name: m.id, size: 0 })) + const models: NomadInstalledModel[] = installedModelsFromOpenAIResponse(modelList) if (includeEmbeddings) return models return models.filter((m) => !m.name.includes('embed')) } catch (err) { diff --git a/admin/app/utils/chat_images.ts b/admin/app/utils/chat_images.ts new file mode 100644 index 00000000..d26e665d --- /dev/null +++ b/admin/app/utils/chat_images.ts @@ -0,0 +1,125 @@ +import type { MultipartFile } from '@adonisjs/bodyparser/types' +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js' +import sharp from 'sharp' + +export const MAX_CHAT_IMAGES = 4 +export const MAX_CHAT_IMAGE_BYTES = 8 * 1024 * 1024 +export const MAX_CHAT_IMAGE_PIXELS = 40_000_000 +export const MAX_NORMALIZED_IMAGE_BYTES = 4 * 1024 * 1024 +export const MAX_NORMALIZED_IMAGE_DIMENSION = 2048 + +const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp']) + +export class ChatImageError extends Error { + constructor( + message: string, + public readonly status: 413 | 415 | 422 + ) { + super(message) + this.name = 'ChatImageError' + } +} + +export type NormalizedChatImage = { + name: string + dataUrl: string +} + +export async function normalizeChatImages(files: MultipartFile[]): Promise { + if (files.length > MAX_CHAT_IMAGES) { + throw new ChatImageError(`Attach no more than ${MAX_CHAT_IMAGES} images.`, 422) + } + + const normalized: NormalizedChatImage[] = [] + for (const file of files) { + normalized.push(await normalizeChatImage(file)) + } + return normalized +} + +async function normalizeChatImage(file: MultipartFile): Promise { + if (!file.tmpPath) { + throw new ChatImageError(`Could not process "${file.clientName}".`, 422) + } + if (file.size > MAX_CHAT_IMAGE_BYTES) { + throw new ChatImageError(`"${file.clientName}" exceeds the 8 MB per-image limit.`, 413) + } + if (!file.isValid) { + throw new ChatImageError(`"${file.clientName}" is not a valid image upload.`, 422) + } + + try { + const pipeline = sharp(file.tmpPath, { + animated: false, + failOn: 'warning', + limitInputPixels: MAX_CHAT_IMAGE_PIXELS, + }) + const metadata = await pipeline.metadata() + + if (!metadata.format || !SUPPORTED_IMAGE_FORMATS.has(metadata.format)) { + throw new ChatImageError( + `"${file.clientName}" is not supported. Use JPEG, PNG, or WebP.`, + 415 + ) + } + if ((metadata.pages ?? 1) > 1) { + throw new ChatImageError(`Animated images are not supported.`, 415) + } + + const normalized = await pipeline + .rotate() + .resize({ + width: MAX_NORMALIZED_IMAGE_DIMENSION, + height: MAX_NORMALIZED_IMAGE_DIMENSION, + fit: 'inside', + withoutEnlargement: true, + }) + .flatten({ background: '#ffffff' }) + .jpeg({ quality: 85 }) + .toBuffer() + + if (normalized.byteLength > MAX_NORMALIZED_IMAGE_BYTES) { + throw new ChatImageError( + `"${file.clientName}" remains too large after image processing.`, + 413 + ) + } + + return { + name: file.clientName, + dataUrl: `data:image/jpeg;base64,${normalized.toString('base64')}`, + } + } catch (error) { + if (error instanceof ChatImageError) throw error + throw new ChatImageError(`"${file.clientName}" could not be decoded as an image.`, 422) + } +} + +export function attachImagesToLatestUserMessage( + messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>, + images: NormalizedChatImage[] +): ChatCompletionMessageParam[] { + if (images.length === 0) return messages + + let latestUserIndex = -1 + messages.forEach((message, index) => { + if (message.role === 'user') latestUserIndex = index + }) + if (latestUserIndex < 0) { + throw new ChatImageError('Images require a user message.', 422) + } + + return messages.map((message, index) => { + if (index !== latestUserIndex) return message + return { + role: 'user', + content: [ + { type: 'text', text: message.content }, + ...images.map((image) => ({ + type: 'image_url' as const, + image_url: { url: image.dataUrl }, + })), + ], + } + }) +} diff --git a/admin/app/utils/model_capabilities.ts b/admin/app/utils/model_capabilities.ts new file mode 100644 index 00000000..c709b997 --- /dev/null +++ b/admin/app/utils/model_capabilities.ts @@ -0,0 +1,86 @@ +import type { ModelVisionCapability } from '../../types/ollama.js' + +export type ModelCapabilities = { + thinking: boolean + vision: ModelVisionCapability +} + +export type ModelCapabilityProbes = { + ollamaShow: () => Promise + llamaProps: () => Promise +} + +export type InstalledModelMetadata = { + name: string + size: number + capabilities?: string[] +} + +export function capabilitiesFromOllamaShow(value: unknown): ModelCapabilities | null { + if (!value || typeof value !== 'object') return null + const capabilities = (value as { capabilities?: unknown }).capabilities + if (!Array.isArray(capabilities) || !capabilities.every((item) => typeof item === 'string')) { + return null + } + + return { + thinking: capabilities.includes('thinking'), + vision: capabilities.includes('vision') ? 'supported' : 'unsupported', + } +} + +export function visionCapabilityFromLlamaProps(value: unknown): ModelVisionCapability | null { + if (!value || typeof value !== 'object') return null + const modalities = (value as { modalities?: unknown }).modalities + if (modalities && typeof modalities === 'object' && !Array.isArray(modalities)) { + const vision = (modalities as { vision?: unknown }).vision + return typeof vision === 'boolean' ? (vision ? 'supported' : 'unsupported') : null + } + if (Array.isArray(modalities) && modalities.every((item) => typeof item === 'string')) { + return modalities.includes('image') || modalities.includes('vision') + ? 'supported' + : 'unsupported' + } + return null +} + +export async function resolveModelCapabilities( + advertisedMetadata: unknown, + probes: ModelCapabilityProbes +): Promise { + const advertised = capabilitiesFromOllamaShow(advertisedMetadata) + if (advertised) return advertised + + try { + const ollama = capabilitiesFromOllamaShow(await probes.ollamaShow()) + if (ollama) return ollama + } catch {} + + try { + const vision = visionCapabilityFromLlamaProps(await probes.llamaProps()) + if (vision) return { thinking: false, vision } + } catch {} + + return null +} + +export function installedModelsFromOpenAIResponse(value: unknown): InstalledModelMetadata[] { + if (!value || typeof value !== 'object') return [] + const data = (value as { data?: unknown }).data + if (!Array.isArray(data)) return [] + + return data.flatMap((entry) => { + if (!entry || typeof entry !== 'object') return [] + const { id, capabilities } = entry as { id?: unknown; capabilities?: unknown } + if (typeof id !== 'string') return [] + + const model: InstalledModelMetadata = { name: id, size: 0 } + if ( + Array.isArray(capabilities) && + capabilities.every((capability) => typeof capability === 'string') + ) { + model.capabilities = capabilities + } + return [model] + }) +} diff --git a/admin/inertia/components/InfoTooltip.tsx b/admin/inertia/components/InfoTooltip.tsx index 172ee5ab..6b384d06 100644 --- a/admin/inertia/components/InfoTooltip.tsx +++ b/admin/inertia/components/InfoTooltip.tsx @@ -7,10 +7,9 @@ interface InfoTooltipProps { // Which side of the icon the tooltip pops toward. Defaults to 'top' (existing behavior); // use 'bottom' when the icon sits near the top of the viewport so it isn't clipped. position?: 'top' | 'bottom' - // Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'right' when - // the icon sits near the right edge so the bubble expands leftward into open space instead of - // being squeezed against the viewport edge (which forces one-word-per-line wrapping). - align?: 'center' | 'right' + // Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'left' near the + // left edge so the bubble expands rightward, or 'right' near the right edge so it expands leftward. + align?: 'center' | 'left' | 'right' } export default function InfoTooltip({ @@ -37,12 +36,16 @@ export default function InfoTooltip({ {isVisible && (
{text} @@ -51,7 +54,13 @@ export default function InfoTooltip({ position === 'bottom' ? 'bottom-full border-b-desert-stone-dark' : 'top-full border-t-desert-stone-dark' - } ${align === 'right' ? 'right-3' : 'left-1/2 -translate-x-1/2'}`} + } ${ + align === 'left' + ? 'left-3' + : align === 'right' + ? 'right-3' + : 'left-1/2 -translate-x-1/2' + }`} />
diff --git a/admin/inertia/components/chat/ChatInterface.tsx b/admin/inertia/components/chat/ChatInterface.tsx index 4e983f3c..b9c665a1 100644 --- a/admin/inertia/components/chat/ChatInterface.tsx +++ b/admin/inertia/components/chat/ChatInterface.tsx @@ -1,24 +1,33 @@ -import { IconSend, IconWand } from '@tabler/icons-react' +import { IconPhoto, IconSend, IconWand, IconX } from '@tabler/icons-react' import { useState, useRef, useEffect } from 'react' import classNames from '~/lib/classNames' -import { ChatMessage } from '../../../types/chat' +import { ChatImageAttachment, ChatMessage } from '../../../types/chat' +import type { ModelVisionCapability } from '../../../types/ollama' import ChatMessageBubble from './ChatMessageBubble' import ChatAssistantAvatar from './ChatAssistantAvatar' import BouncingDots from '../BouncingDots' import { usePage } from '@inertiajs/react' +import InfoTooltip from '../InfoTooltip' +import { visionAttachmentGuidance } from '../../lib/vision_guidance' interface ChatInterfaceProps { messages: ChatMessage[] - onSendMessage: (message: string) => void + onSendMessage: (message: string, images: ChatImageAttachment[]) => void + visionCapability: ModelVisionCapability isLoading?: boolean chatSuggestions?: string[] chatSuggestionsEnabled?: boolean chatSuggestionsLoading?: boolean } +const MAX_VISION_IMAGES = 4 +const MAX_VISION_IMAGE_BYTES = 8 * 1024 * 1024 +const SUPPORTED_VISION_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']) + export default function ChatInterface({ messages, onSendMessage, + visionCapability, isLoading = false, chatSuggestions = [], chatSuggestionsEnabled = false, @@ -26,8 +35,30 @@ export default function ChatInterface({ }: ChatInterfaceProps) { const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props const [input, setInput] = useState('') + const [images, setImages] = useState([]) const messagesEndRef = useRef(null) const textareaRef = useRef(null) + const imageInputRef = useRef(null) + const previewUrlsRef = useRef>(new Set()) + + useEffect( + () => () => { + previewUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)) + previewUrlsRef.current.clear() + }, + [] + ) + + useEffect(() => { + if (visionCapability !== 'unsupported') return + setImages((current) => { + current.forEach((image) => { + URL.revokeObjectURL(image.previewUrl) + previewUrlsRef.current.delete(image.previewUrl) + }) + return [] + }) + }, [visionCapability]) const scrollToBottom = () => { messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) @@ -39,15 +70,70 @@ export default function ChatInterface({ const handleSubmit = (e: React.FormEvent) => { e.preventDefault() - if (input.trim() && !isLoading) { - onSendMessage(input.trim()) + if ((input.trim() || images.length > 0) && !isLoading) { + onSendMessage(input.trim() || 'Describe the attached image.', images) setInput('') + setImages([]) + if (imageInputRef.current) imageInputRef.current.value = '' if (textareaRef.current) { textareaRef.current.style.height = 'auto' } } } + const handleImageSelection = (event: React.ChangeEvent) => { + const selected = Array.from(event.target.files ?? []) + event.target.value = '' + if (selected.length === 0) return + + const availableSlots = MAX_VISION_IMAGES - images.length + if (availableSlots <= 0) { + addNotification({ type: 'error', message: `You can attach up to ${MAX_VISION_IMAGES} images.` }) + return + } + + const attachments: ChatImageAttachment[] = [] + for (const file of selected.slice(0, availableSlots)) { + if (!SUPPORTED_VISION_TYPES.has(file.type)) { + addNotification({ + type: 'error', + message: `${file.name} is not supported. Use JPEG, PNG, or WebP.`, + }) + continue + } + if (file.size > MAX_VISION_IMAGE_BYTES) { + addNotification({ + type: 'error', + message: `${file.name} exceeds the 8 MB per-image limit.`, + }) + continue + } + + const previewUrl = URL.createObjectURL(file) + previewUrlsRef.current.add(previewUrl) + attachments.push({ + id: crypto.randomUUID(), + name: file.name, + file, + previewUrl, + }) + } + + if (selected.length > availableSlots) { + addNotification({ + type: 'error', + message: `Only the first ${availableSlots} selected image(s) were attached.`, + }) + } + setImages((current) => [...current, ...attachments]) + } + + const removeImage = (image: ChatImageAttachment) => { + URL.revokeObjectURL(image.previewUrl) + previewUrlsRef.current.delete(image.previewUrl) + setImages((current) => current.filter((item) => item.id !== image.id)) + } + const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault() @@ -139,7 +225,67 @@ export default function ChatInterface({ )}
+ {images.length > 0 && ( +
+ {images.map((image) => ( +
+ {image.name} + +
+ ))} +
+ )}
+ +
+ + {visionCapability !== 'supported' && ( + + )} +