mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 05:14:19 +02:00
feat(ai): support local vision attachments
Closes #1176. Salvages @gabegraves's work from #1178, rebased onto the
RagPipelineService refactor that landed after it was written.
Adds JPEG/PNG/WebP attachments to chat over multipart, leaving text-only JSON
requests on their existing path. Images are normalized server-side to bounded
JPEG with metadata stripped, capped on count/size/pixels/output, and the
temporary uploads are cleaned up on every validation path. They are attached
only to the latest user message, are excluded from RAG, titles, logs and
persisted history, and are never written to storage.
Vision support is resolved per model from what the backend actually reports --
OpenAI-compatible /v1/models metadata, then Ollama /api/show, then llama.cpp
/props -- never inferred from a model name. A model that reports no vision
support has attachments disabled with a visible explanation; a backend that
reports nothing is allowed with a warning and its rejection is translated into
plain compatibility guidance rather than a raw error.
Changes from the original:
- Prompt assembly moved into RagPipelineService in the meantime, so the
controller half is rebuilt against the trace and the multipart-aware
collection filter is now the only one (dev's request.input() version would
have silently dropped the collection on every image request).
- Added a native-path converter. dev split chat into native and OpenAI-compat
paths after this was written; Ollama's native /api/chat does not take OpenAI
content parts, it takes a sibling `images` array of bare base64. Without this
every image request on a native Ollama backend would fail.
- Dropped the model-recommendation change (selectRecommendedModels and the
Qwen3-VL fallback entry). Displacing a recommended model at first-time setup
is a product decision and is being taken separately. The "Supports images"
label in Easy Setup is kept: it describes what is already on offer rather
than changing it.
- Dropped the DEFAULT_QUERY_REWRITE_MODEL download prompt his branch carried,
which dev deleted in d022c2d.
- Removed a duplicate checkModelHasThinking; kept dev's memoized getModelInfo
path, since thinking detection is not part of this feature.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01F8Mgg1vd2eKSs8StZiuyMV
This commit is contained in:
Gabe Graves
Jake Turner
parent
d1b96962ac
commit
edbfe1ad78
|
|
@ -16,6 +16,16 @@ import { DEFAULT_KEEP_ALIVE } from '../../constants/ollama.js'
|
|||
import type { PipelineTrace } from '../../types/rag.js'
|
||||
import { buildCitations } from '../utils/rag_prompt.js'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import {
|
||||
attachImagesToLatestUserMessage,
|
||||
ChatImageError,
|
||||
normalizeChatImages,
|
||||
type NormalizedChatImage,
|
||||
} from '../utils/chat_images.js'
|
||||
|
||||
const unknownVisionCompatibilityMessage = (model: string) =>
|
||||
`NOMAD cannot confirm that "${model}" accepts images, and this image request failed. Choose a model marked "Supports images", or remove the image and try again.`
|
||||
|
||||
@inject()
|
||||
export default class OllamaController {
|
||||
|
|
@ -53,7 +63,78 @@ export default class OllamaController {
|
|||
}
|
||||
|
||||
async chat({ request, response }: HttpContext) {
|
||||
const reqData = await request.validateUsing(chatSchema)
|
||||
const uploadedImages = request.files('images', {
|
||||
size: '8mb',
|
||||
extnames: ['jpg', 'jpeg', 'png', 'webp'],
|
||||
})
|
||||
const cleanupUploadedImages = () =>
|
||||
Promise.all(
|
||||
uploadedImages
|
||||
.filter((file) => file.tmpPath)
|
||||
.map((file) => rm(file.tmpPath!, { force: true }).catch(() => undefined))
|
||||
)
|
||||
let multipartPayload: unknown
|
||||
if (uploadedImages.length > 0) {
|
||||
const rawPayload = request.input('payload')
|
||||
if (typeof rawPayload !== 'string') {
|
||||
await cleanupUploadedImages()
|
||||
return response.status(422).send({ message: 'Multipart chat requests require a JSON payload.' })
|
||||
}
|
||||
try {
|
||||
multipartPayload = JSON.parse(rawPayload)
|
||||
} catch {
|
||||
await cleanupUploadedImages()
|
||||
return response.status(422).send({ message: 'The multipart chat payload is not valid JSON.' })
|
||||
}
|
||||
}
|
||||
// A multipart chat carries the collection inside the JSON payload rather than
|
||||
// as a form field, so request.input() alone would silently drop it and every
|
||||
// image request would search the whole knowledge base.
|
||||
const rawCollection =
|
||||
uploadedImages.length > 0 &&
|
||||
multipartPayload &&
|
||||
typeof multipartPayload === 'object' &&
|
||||
'collection' in multipartPayload
|
||||
? (multipartPayload as { collection?: unknown }).collection
|
||||
: request.input('collection', null)
|
||||
const collectionFilter: string | null =
|
||||
typeof rawCollection === 'string' && rawCollection ? rawCollection : null
|
||||
|
||||
let reqData: Awaited<ReturnType<typeof chatSchema.validate>>
|
||||
try {
|
||||
reqData =
|
||||
uploadedImages.length > 0
|
||||
? await chatSchema.validate(multipartPayload)
|
||||
: await request.validateUsing(chatSchema)
|
||||
} catch (error) {
|
||||
await cleanupUploadedImages()
|
||||
throw error
|
||||
}
|
||||
|
||||
let normalizedImages: NormalizedChatImage[]
|
||||
try {
|
||||
normalizedImages = await normalizeChatImages(uploadedImages)
|
||||
} catch (error) {
|
||||
if (error instanceof ChatImageError) {
|
||||
return response.status(error.status).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
await cleanupUploadedImages()
|
||||
}
|
||||
|
||||
const modelCapabilities = await this.ollamaService.getModelCapabilities(reqData.model)
|
||||
if (
|
||||
normalizedImages.length > 0 &&
|
||||
!reqData.messages.some((message) => message.role === 'user')
|
||||
) {
|
||||
return response.status(422).send({ message: 'Images require a user message.' })
|
||||
}
|
||||
if (normalizedImages.length > 0 && modelCapabilities.vision === 'unsupported') {
|
||||
return response.status(422).send({
|
||||
message: `The selected model "${reqData.model}" does not support image input.`,
|
||||
})
|
||||
}
|
||||
|
||||
// Flush SSE headers immediately so the client connection is open while
|
||||
// pre-processing (query rewriting, RAG lookup) runs in the background.
|
||||
|
|
@ -64,6 +145,7 @@ export default class OllamaController {
|
|||
response.response.flushHeaders()
|
||||
}
|
||||
|
||||
let unknownVisionUpstreamRejected = false
|
||||
try {
|
||||
// Everything from system-prompt assembly through query rewriting,
|
||||
// retrieval, context trimming and the num_ctx decision lives in
|
||||
|
|
@ -72,7 +154,6 @@ export default class OllamaController {
|
|||
// settings, both writing rag.enabled). Unset means on, preserving the
|
||||
// behaviour from before the toggle existed.
|
||||
const ragEnabled = (await KVStore.getValue('rag.enabled')) ?? true
|
||||
const collectionFilter: string | null = request.input('collection', null)
|
||||
const trace = await this.ragPipelineService.buildPrompt(reqData.messages, reqData.model, {
|
||||
collection: collectionFilter ?? undefined,
|
||||
skipRetrieval: !ragEnabled,
|
||||
|
|
@ -94,7 +175,7 @@ export default class OllamaController {
|
|||
// Thinking is only enabled when the model supports it AND the user wants it: the explicit
|
||||
// per-request preference wins, otherwise the global default (ai.autoThinking, default OFF).
|
||||
// If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat
|
||||
const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model)
|
||||
const thinkingCapability = modelCapabilities.thinking
|
||||
let thinkingEnabled = false
|
||||
if (thinkingCapability) {
|
||||
thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false)
|
||||
|
|
@ -105,6 +186,10 @@ export default class OllamaController {
|
|||
// Separate sessionId and the resolved thinking preference from the Ollama request payload —
|
||||
// Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw).
|
||||
const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData
|
||||
const upstreamMessages = attachImagesToLatestUserMessage(
|
||||
ollamaRequest.messages,
|
||||
normalizedImages
|
||||
)
|
||||
|
||||
// Save user message to DB before streaming if sessionId provided
|
||||
let userContent: string | null = null
|
||||
|
|
@ -124,17 +209,18 @@ export default class OllamaController {
|
|||
// blocks every later chat/RAG request until the model is manually stopped (#1065).
|
||||
const abortController = new AbortController()
|
||||
response.response.on('close', () => abortController.abort())
|
||||
const stream = await this.ollamaService.chatStream({
|
||||
...ollamaRequest,
|
||||
think,
|
||||
thinkingCapable: thinkingCapability,
|
||||
numCtx,
|
||||
numPredict,
|
||||
keepAlive,
|
||||
signal: abortController.signal,
|
||||
})
|
||||
let fullContent = ''
|
||||
try {
|
||||
const stream = await this.ollamaService.chatStream({
|
||||
...ollamaRequest,
|
||||
messages: upstreamMessages,
|
||||
think,
|
||||
thinkingCapable: thinkingCapability,
|
||||
numCtx,
|
||||
numPredict,
|
||||
keepAlive,
|
||||
signal: abortController.signal,
|
||||
})
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.message?.content) {
|
||||
fullContent += chunk.message.content
|
||||
|
|
@ -149,6 +235,8 @@ export default class OllamaController {
|
|||
logger.debug('[OllamaController] Client disconnected; aborted upstream Ollama generation')
|
||||
return
|
||||
}
|
||||
unknownVisionUpstreamRejected =
|
||||
normalizedImages.length > 0 && modelCapabilities.vision === 'unknown'
|
||||
throw err
|
||||
}
|
||||
// Trailing citation event, written before end(). It carries no `message`
|
||||
|
|
@ -174,14 +262,25 @@ export default class OllamaController {
|
|||
}
|
||||
|
||||
// Non-streaming (legacy) path
|
||||
const result = await this.ollamaService.chat({
|
||||
...ollamaRequest,
|
||||
think,
|
||||
thinkingCapable: thinkingCapability,
|
||||
numCtx,
|
||||
numPredict,
|
||||
keepAlive,
|
||||
})
|
||||
let result
|
||||
try {
|
||||
result = await this.ollamaService.chat({
|
||||
...ollamaRequest,
|
||||
messages: upstreamMessages,
|
||||
think,
|
||||
thinkingCapable: thinkingCapability,
|
||||
numCtx,
|
||||
numPredict,
|
||||
keepAlive,
|
||||
})
|
||||
} catch (err) {
|
||||
// A backend that never advertised vision support rejecting a request that
|
||||
// carried images is the signal that it cannot see them. Recorded here and
|
||||
// translated into the compatibility message by the outer catch.
|
||||
unknownVisionUpstreamRejected =
|
||||
normalizedImages.length > 0 && modelCapabilities.vision === 'unknown'
|
||||
throw err
|
||||
}
|
||||
if (result?.usage) {
|
||||
this._recordUsage(reqData.model, trace, result.usage)
|
||||
}
|
||||
|
|
@ -199,10 +298,22 @@ export default class OllamaController {
|
|||
return { ...result, sources }
|
||||
} catch (error) {
|
||||
if (reqData.stream) {
|
||||
response.response.write(`data: ${JSON.stringify({ error: true })}\n\n`)
|
||||
const streamError =
|
||||
unknownVisionUpstreamRejected
|
||||
? {
|
||||
error: true,
|
||||
message: unknownVisionCompatibilityMessage(reqData.model),
|
||||
}
|
||||
: { error: true }
|
||||
response.response.write(`data: ${JSON.stringify(streamError)}\n\n`)
|
||||
response.response.end()
|
||||
return
|
||||
}
|
||||
if (unknownVisionUpstreamRejected) {
|
||||
return response.status(422).send({
|
||||
message: unknownVisionCompatibilityMessage(reqData.model),
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
|
@ -392,15 +503,13 @@ export default class OllamaController {
|
|||
}
|
||||
}
|
||||
|
||||
async installedModels({ }: HttpContext) {
|
||||
async installedModels({}: HttpContext) {
|
||||
const models = await this.ollamaService.getModels()
|
||||
// Enrich each model with its thinking capability so the chat picker knows which models
|
||||
// to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show
|
||||
// results, so this stays cheap on repeat loads. Best-effort per model.
|
||||
const thinking = await Promise.all(
|
||||
models.map((m) => this.ollamaService.checkModelHasThinking(m.name))
|
||||
// Enrich from backend-reported capabilities; never guess from model names.
|
||||
const capabilities = await Promise.all(
|
||||
models.map((m) => this.ollamaService.getModelCapabilities(m.name, m))
|
||||
)
|
||||
return models.map((m, i) => ({ ...m, thinking: thinking[i] }))
|
||||
return models.map((m, i) => ({ ...m, ...capabilities[i] }))
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,12 @@ import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
|||
import env from '#start/env'
|
||||
import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js'
|
||||
import KVStore from '#models/kv_store'
|
||||
import type { ModelCapabilities } from '../utils/model_capabilities.js'
|
||||
import {
|
||||
capabilitiesFromOllamaShow,
|
||||
installedModelsFromOpenAIResponse,
|
||||
resolveModelCapabilities,
|
||||
} from '../utils/model_capabilities.js'
|
||||
|
||||
const NOMAD_MODELS_API_PATH = '/api/v1/ollama/models'
|
||||
const MODELS_CACHE_FILE = path.join(process.cwd(), 'storage', 'ollama-models-cache.json')
|
||||
|
|
@ -57,6 +63,7 @@ export type NomadInstalledModel = {
|
|||
size: number
|
||||
digest?: string
|
||||
details?: Record<string, any>
|
||||
capabilities?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -103,7 +110,7 @@ export type NomadChatStreamChunk = {
|
|||
|
||||
type ChatInput = {
|
||||
model: string
|
||||
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
|
||||
messages: ChatCompletionMessageParam[]
|
||||
think?: boolean | 'medium'
|
||||
// Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but
|
||||
// disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing).
|
||||
|
|
@ -134,6 +141,43 @@ type ChatInput = {
|
|||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
/**
|
||||
* Ollama's native /api/chat does not take OpenAI content parts. It carries images
|
||||
* as a sibling `images` array of bare base64 (no `data:` prefix) on the message,
|
||||
* with `content` staying a plain string. The OpenAI-compat path takes the parts
|
||||
* as-is, so the conversion only belongs on the native side.
|
||||
*/
|
||||
function toNativeMessages(messages: ChatInput['messages']): Array<{
|
||||
role: string
|
||||
content: string
|
||||
images?: string[]
|
||||
}> {
|
||||
return messages.map((message) => {
|
||||
const content = (message as { content?: unknown }).content
|
||||
if (!Array.isArray(content)) {
|
||||
return { role: message.role as string, content: (content as string) ?? '' }
|
||||
}
|
||||
|
||||
const text: string[] = []
|
||||
const images: string[] = []
|
||||
for (const part of content as Array<Record<string, any>>) {
|
||||
if (part?.type === 'text' && typeof part.text === 'string') {
|
||||
text.push(part.text)
|
||||
} else if (part?.type === 'image_url' && typeof part.image_url?.url === 'string') {
|
||||
// Strip the data URL wrapper; Ollama wants the payload only.
|
||||
images.push(part.image_url.url.replace(/^data:[^;]+;base64,/, ''))
|
||||
}
|
||||
}
|
||||
|
||||
const out: { role: string; content: string; images?: string[] } = {
|
||||
role: message.role as string,
|
||||
content: text.join('\n'),
|
||||
}
|
||||
if (images.length > 0) out.images = images
|
||||
return out
|
||||
})
|
||||
}
|
||||
|
||||
@inject()
|
||||
export class OllamaService {
|
||||
private openai: OpenAI | null = null
|
||||
|
|
@ -147,6 +191,12 @@ export class OllamaService {
|
|||
// trained context length don't change at runtime. Only successful lookups are cached;
|
||||
// transient failures are left uncached so they can be retried.
|
||||
private modelInfoCache: Map<string, NomadModelInfo> = new Map()
|
||||
// Definitive capabilities are stable for a loaded model. Cache unknown results briefly to
|
||||
// avoid adding repeated endpoint probes to chat startup while still allowing backend reloads.
|
||||
private modelCapabilityCache: Map<
|
||||
string,
|
||||
{ value: ModelCapabilities; expiresAt: number }
|
||||
> = new Map()
|
||||
|
||||
constructor() {}
|
||||
|
||||
|
|
@ -570,7 +620,7 @@ export class OllamaService {
|
|||
|
||||
const response = await this.ollama.chat({
|
||||
model: chatRequest.model,
|
||||
messages: chatRequest.messages,
|
||||
messages: toNativeMessages(chatRequest.messages) as any,
|
||||
stream: false,
|
||||
// Definedness, not truthiness: `think: false` has to reach Ollama, which
|
||||
// otherwise leaves thinking ON for a capable model. Unset still sends
|
||||
|
|
@ -660,7 +710,7 @@ export class OllamaService {
|
|||
|
||||
const iterator = await this.ollama.chat({
|
||||
model: chatRequest.model,
|
||||
messages: chatRequest.messages,
|
||||
messages: toNativeMessages(chatRequest.messages) as any,
|
||||
stream: true,
|
||||
// See _chatNative: `think: false` must be sent explicitly, or a thinking-capable
|
||||
// model keeps reasoning regardless of the user's ai.autoThinking preference.
|
||||
|
|
@ -816,6 +866,60 @@ export class OllamaService {
|
|||
}
|
||||
}
|
||||
|
||||
public async getModelCapabilities(
|
||||
modelName: string,
|
||||
advertisedMetadata?: unknown
|
||||
): Promise<ModelCapabilities> {
|
||||
await this._ensureDependencies()
|
||||
if (!this.baseUrl) return { thinking: false, vision: 'unknown' }
|
||||
|
||||
// Composite OpenAI-compatible routers can advertise different capabilities for each model.
|
||||
// Prefer that per-model metadata before probing backend-specific endpoints.
|
||||
const advertised = capabilitiesFromOllamaShow(advertisedMetadata)
|
||||
if (advertised) {
|
||||
this.modelCapabilityCache.set(modelName, {
|
||||
value: advertised,
|
||||
expiresAt: Number.POSITIVE_INFINITY,
|
||||
})
|
||||
return advertised
|
||||
}
|
||||
|
||||
const cached = this.modelCapabilityCache.get(modelName)
|
||||
if (cached && cached.expiresAt > Date.now()) return cached.value
|
||||
|
||||
// Probe per model instead of relying on the backend-wide classification from getModels().
|
||||
// Hybrid routers may expose /v1/models plus /api/show without exposing /api/tags.
|
||||
const detected = await resolveModelCapabilities(null, {
|
||||
ollamaShow: async () => {
|
||||
const response = await axios.post(
|
||||
`${this.baseUrl}/api/show`,
|
||||
{ model: modelName },
|
||||
{ timeout: 3000 }
|
||||
)
|
||||
return response.data
|
||||
},
|
||||
// A direct llama.cpp server reports the loaded model's input modalities at /props.
|
||||
llamaProps: async () => {
|
||||
const response = await axios.get(`${this.baseUrl}/props`, { timeout: 3000 })
|
||||
return response.data
|
||||
},
|
||||
})
|
||||
if (detected) {
|
||||
this.modelCapabilityCache.set(modelName, {
|
||||
value: detected,
|
||||
expiresAt: Number.POSITIVE_INFINITY,
|
||||
})
|
||||
return detected
|
||||
}
|
||||
|
||||
const unknown: ModelCapabilities = { thinking: false, vision: 'unknown' }
|
||||
this.modelCapabilityCache.set(modelName, {
|
||||
value: unknown,
|
||||
expiresAt: Date.now() + 30_000,
|
||||
})
|
||||
return unknown
|
||||
}
|
||||
|
||||
public async checkModelHasThinking(modelName: string): Promise<boolean> {
|
||||
return (await this.getModelInfo(modelName)).hasThinking
|
||||
}
|
||||
|
|
@ -1134,7 +1238,7 @@ export class OllamaService {
|
|||
logger.info('[OllamaService] /api/tags unavailable, falling back to /v1/models')
|
||||
try {
|
||||
const modelList = await this.openai!.models.list()
|
||||
const models: NomadInstalledModel[] = modelList.data.map((m) => ({ name: m.id, size: 0 }))
|
||||
const models: NomadInstalledModel[] = installedModelsFromOpenAIResponse(modelList)
|
||||
if (includeEmbeddings) return models
|
||||
return models.filter((m) => !m.name.includes('embed'))
|
||||
} catch (err) {
|
||||
|
|
|
|||
125
admin/app/utils/chat_images.ts
Normal file
125
admin/app/utils/chat_images.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
import type { MultipartFile } from '@adonisjs/bodyparser/types'
|
||||
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
|
||||
import sharp from 'sharp'
|
||||
|
||||
export const MAX_CHAT_IMAGES = 4
|
||||
export const MAX_CHAT_IMAGE_BYTES = 8 * 1024 * 1024
|
||||
export const MAX_CHAT_IMAGE_PIXELS = 40_000_000
|
||||
export const MAX_NORMALIZED_IMAGE_BYTES = 4 * 1024 * 1024
|
||||
export const MAX_NORMALIZED_IMAGE_DIMENSION = 2048
|
||||
|
||||
const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp'])
|
||||
|
||||
export class ChatImageError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly status: 413 | 415 | 422
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'ChatImageError'
|
||||
}
|
||||
}
|
||||
|
||||
export type NormalizedChatImage = {
|
||||
name: string
|
||||
dataUrl: string
|
||||
}
|
||||
|
||||
export async function normalizeChatImages(files: MultipartFile[]): Promise<NormalizedChatImage[]> {
|
||||
if (files.length > MAX_CHAT_IMAGES) {
|
||||
throw new ChatImageError(`Attach no more than ${MAX_CHAT_IMAGES} images.`, 422)
|
||||
}
|
||||
|
||||
const normalized: NormalizedChatImage[] = []
|
||||
for (const file of files) {
|
||||
normalized.push(await normalizeChatImage(file))
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
async function normalizeChatImage(file: MultipartFile): Promise<NormalizedChatImage> {
|
||||
if (!file.tmpPath) {
|
||||
throw new ChatImageError(`Could not process "${file.clientName}".`, 422)
|
||||
}
|
||||
if (file.size > MAX_CHAT_IMAGE_BYTES) {
|
||||
throw new ChatImageError(`"${file.clientName}" exceeds the 8 MB per-image limit.`, 413)
|
||||
}
|
||||
if (!file.isValid) {
|
||||
throw new ChatImageError(`"${file.clientName}" is not a valid image upload.`, 422)
|
||||
}
|
||||
|
||||
try {
|
||||
const pipeline = sharp(file.tmpPath, {
|
||||
animated: false,
|
||||
failOn: 'warning',
|
||||
limitInputPixels: MAX_CHAT_IMAGE_PIXELS,
|
||||
})
|
||||
const metadata = await pipeline.metadata()
|
||||
|
||||
if (!metadata.format || !SUPPORTED_IMAGE_FORMATS.has(metadata.format)) {
|
||||
throw new ChatImageError(
|
||||
`"${file.clientName}" is not supported. Use JPEG, PNG, or WebP.`,
|
||||
415
|
||||
)
|
||||
}
|
||||
if ((metadata.pages ?? 1) > 1) {
|
||||
throw new ChatImageError(`Animated images are not supported.`, 415)
|
||||
}
|
||||
|
||||
const normalized = await pipeline
|
||||
.rotate()
|
||||
.resize({
|
||||
width: MAX_NORMALIZED_IMAGE_DIMENSION,
|
||||
height: MAX_NORMALIZED_IMAGE_DIMENSION,
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.flatten({ background: '#ffffff' })
|
||||
.jpeg({ quality: 85 })
|
||||
.toBuffer()
|
||||
|
||||
if (normalized.byteLength > MAX_NORMALIZED_IMAGE_BYTES) {
|
||||
throw new ChatImageError(
|
||||
`"${file.clientName}" remains too large after image processing.`,
|
||||
413
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
name: file.clientName,
|
||||
dataUrl: `data:image/jpeg;base64,${normalized.toString('base64')}`,
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ChatImageError) throw error
|
||||
throw new ChatImageError(`"${file.clientName}" could not be decoded as an image.`, 422)
|
||||
}
|
||||
}
|
||||
|
||||
export function attachImagesToLatestUserMessage(
|
||||
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
|
||||
images: NormalizedChatImage[]
|
||||
): ChatCompletionMessageParam[] {
|
||||
if (images.length === 0) return messages
|
||||
|
||||
let latestUserIndex = -1
|
||||
messages.forEach((message, index) => {
|
||||
if (message.role === 'user') latestUserIndex = index
|
||||
})
|
||||
if (latestUserIndex < 0) {
|
||||
throw new ChatImageError('Images require a user message.', 422)
|
||||
}
|
||||
|
||||
return messages.map((message, index) => {
|
||||
if (index !== latestUserIndex) return message
|
||||
return {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: message.content },
|
||||
...images.map((image) => ({
|
||||
type: 'image_url' as const,
|
||||
image_url: { url: image.dataUrl },
|
||||
})),
|
||||
],
|
||||
}
|
||||
})
|
||||
}
|
||||
86
admin/app/utils/model_capabilities.ts
Normal file
86
admin/app/utils/model_capabilities.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import type { ModelVisionCapability } from '../../types/ollama.js'
|
||||
|
||||
export type ModelCapabilities = {
|
||||
thinking: boolean
|
||||
vision: ModelVisionCapability
|
||||
}
|
||||
|
||||
export type ModelCapabilityProbes = {
|
||||
ollamaShow: () => Promise<unknown>
|
||||
llamaProps: () => Promise<unknown>
|
||||
}
|
||||
|
||||
export type InstalledModelMetadata = {
|
||||
name: string
|
||||
size: number
|
||||
capabilities?: string[]
|
||||
}
|
||||
|
||||
export function capabilitiesFromOllamaShow(value: unknown): ModelCapabilities | null {
|
||||
if (!value || typeof value !== 'object') return null
|
||||
const capabilities = (value as { capabilities?: unknown }).capabilities
|
||||
if (!Array.isArray(capabilities) || !capabilities.every((item) => typeof item === 'string')) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
thinking: capabilities.includes('thinking'),
|
||||
vision: capabilities.includes('vision') ? 'supported' : 'unsupported',
|
||||
}
|
||||
}
|
||||
|
||||
export function visionCapabilityFromLlamaProps(value: unknown): ModelVisionCapability | null {
|
||||
if (!value || typeof value !== 'object') return null
|
||||
const modalities = (value as { modalities?: unknown }).modalities
|
||||
if (modalities && typeof modalities === 'object' && !Array.isArray(modalities)) {
|
||||
const vision = (modalities as { vision?: unknown }).vision
|
||||
return typeof vision === 'boolean' ? (vision ? 'supported' : 'unsupported') : null
|
||||
}
|
||||
if (Array.isArray(modalities) && modalities.every((item) => typeof item === 'string')) {
|
||||
return modalities.includes('image') || modalities.includes('vision')
|
||||
? 'supported'
|
||||
: 'unsupported'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export async function resolveModelCapabilities(
|
||||
advertisedMetadata: unknown,
|
||||
probes: ModelCapabilityProbes
|
||||
): Promise<ModelCapabilities | null> {
|
||||
const advertised = capabilitiesFromOllamaShow(advertisedMetadata)
|
||||
if (advertised) return advertised
|
||||
|
||||
try {
|
||||
const ollama = capabilitiesFromOllamaShow(await probes.ollamaShow())
|
||||
if (ollama) return ollama
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
const vision = visionCapabilityFromLlamaProps(await probes.llamaProps())
|
||||
if (vision) return { thinking: false, vision }
|
||||
} catch {}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export function installedModelsFromOpenAIResponse(value: unknown): InstalledModelMetadata[] {
|
||||
if (!value || typeof value !== 'object') return []
|
||||
const data = (value as { data?: unknown }).data
|
||||
if (!Array.isArray(data)) return []
|
||||
|
||||
return data.flatMap((entry) => {
|
||||
if (!entry || typeof entry !== 'object') return []
|
||||
const { id, capabilities } = entry as { id?: unknown; capabilities?: unknown }
|
||||
if (typeof id !== 'string') return []
|
||||
|
||||
const model: InstalledModelMetadata = { name: id, size: 0 }
|
||||
if (
|
||||
Array.isArray(capabilities) &&
|
||||
capabilities.every((capability) => typeof capability === 'string')
|
||||
) {
|
||||
model.capabilities = capabilities
|
||||
}
|
||||
return [model]
|
||||
})
|
||||
}
|
||||
|
|
@ -7,10 +7,9 @@ interface InfoTooltipProps {
|
|||
// Which side of the icon the tooltip pops toward. Defaults to 'top' (existing behavior);
|
||||
// use 'bottom' when the icon sits near the top of the viewport so it isn't clipped.
|
||||
position?: 'top' | 'bottom'
|
||||
// Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'right' when
|
||||
// the icon sits near the right edge so the bubble expands leftward into open space instead of
|
||||
// being squeezed against the viewport edge (which forces one-word-per-line wrapping).
|
||||
align?: 'center' | 'right'
|
||||
// Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'left' near the
|
||||
// left edge so the bubble expands rightward, or 'right' near the right edge so it expands leftward.
|
||||
align?: 'center' | 'left' | 'right'
|
||||
}
|
||||
|
||||
export default function InfoTooltip({
|
||||
|
|
@ -37,12 +36,16 @@ export default function InfoTooltip({
|
|||
{isVisible && (
|
||||
<div
|
||||
className={`absolute z-50 ${position === 'bottom' ? 'top-full mt-2' : 'bottom-full mb-2'} ${
|
||||
align === 'right' ? 'right-0' : 'left-1/2 -translate-x-1/2'
|
||||
align === 'left'
|
||||
? 'left-0'
|
||||
: align === 'right'
|
||||
? 'right-0'
|
||||
: 'left-1/2 -translate-x-1/2'
|
||||
}`}
|
||||
>
|
||||
<div
|
||||
className={`bg-desert-stone-dark text-white text-xs rounded-lg px-3 py-2 whitespace-normal shadow-lg ${
|
||||
align === 'right' ? 'w-64' : 'max-w-xs'
|
||||
align === 'center' ? 'max-w-xs' : 'w-64'
|
||||
}`}
|
||||
>
|
||||
{text}
|
||||
|
|
@ -51,7 +54,13 @@ export default function InfoTooltip({
|
|||
position === 'bottom'
|
||||
? 'bottom-full border-b-desert-stone-dark'
|
||||
: 'top-full border-t-desert-stone-dark'
|
||||
} ${align === 'right' ? 'right-3' : 'left-1/2 -translate-x-1/2'}`}
|
||||
} ${
|
||||
align === 'left'
|
||||
? 'left-3'
|
||||
: align === 'right'
|
||||
? 'right-3'
|
||||
: 'left-1/2 -translate-x-1/2'
|
||||
}`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,24 +1,33 @@
|
|||
import { IconSend, IconWand } from '@tabler/icons-react'
|
||||
import { IconPhoto, IconSend, IconWand, IconX } from '@tabler/icons-react'
|
||||
import { useState, useRef, useEffect } from 'react'
|
||||
import classNames from '~/lib/classNames'
|
||||
import { ChatMessage } from '../../../types/chat'
|
||||
import { ChatImageAttachment, ChatMessage } from '../../../types/chat'
|
||||
import type { ModelVisionCapability } from '../../../types/ollama'
|
||||
import ChatMessageBubble from './ChatMessageBubble'
|
||||
import ChatAssistantAvatar from './ChatAssistantAvatar'
|
||||
import BouncingDots from '../BouncingDots'
|
||||
import { usePage } from '@inertiajs/react'
|
||||
import InfoTooltip from '../InfoTooltip'
|
||||
import { visionAttachmentGuidance } from '../../lib/vision_guidance'
|
||||
|
||||
interface ChatInterfaceProps {
|
||||
messages: ChatMessage[]
|
||||
onSendMessage: (message: string) => void
|
||||
onSendMessage: (message: string, images: ChatImageAttachment[]) => void
|
||||
visionCapability: ModelVisionCapability
|
||||
isLoading?: boolean
|
||||
chatSuggestions?: string[]
|
||||
chatSuggestionsEnabled?: boolean
|
||||
chatSuggestionsLoading?: boolean
|
||||
}
|
||||
|
||||
const MAX_VISION_IMAGES = 4
|
||||
const MAX_VISION_IMAGE_BYTES = 8 * 1024 * 1024
|
||||
const SUPPORTED_VISION_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp'])
|
||||
|
||||
export default function ChatInterface({
|
||||
messages,
|
||||
onSendMessage,
|
||||
visionCapability,
|
||||
isLoading = false,
|
||||
chatSuggestions = [],
|
||||
chatSuggestionsEnabled = false,
|
||||
|
|
@ -26,8 +35,30 @@ export default function ChatInterface({
|
|||
}: ChatInterfaceProps) {
|
||||
const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props
|
||||
const [input, setInput] = useState('')
|
||||
const [images, setImages] = useState<ChatImageAttachment[]>([])
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null)
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null)
|
||||
const imageInputRef = useRef<HTMLInputElement>(null)
|
||||
const previewUrlsRef = useRef<Set<string>>(new Set())
|
||||
|
||||
useEffect(
|
||||
() => () => {
|
||||
previewUrlsRef.current.forEach((url) => URL.revokeObjectURL(url))
|
||||
previewUrlsRef.current.clear()
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (visionCapability !== 'unsupported') return
|
||||
setImages((current) => {
|
||||
current.forEach((image) => {
|
||||
URL.revokeObjectURL(image.previewUrl)
|
||||
previewUrlsRef.current.delete(image.previewUrl)
|
||||
})
|
||||
return []
|
||||
})
|
||||
}, [visionCapability])
|
||||
|
||||
const scrollToBottom = () => {
|
||||
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' })
|
||||
|
|
@ -39,15 +70,70 @@ export default function ChatInterface({
|
|||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault()
|
||||
if (input.trim() && !isLoading) {
|
||||
onSendMessage(input.trim())
|
||||
if ((input.trim() || images.length > 0) && !isLoading) {
|
||||
onSendMessage(input.trim() || 'Describe the attached image.', images)
|
||||
setInput('')
|
||||
setImages([])
|
||||
if (imageInputRef.current) imageInputRef.current.value = ''
|
||||
if (textareaRef.current) {
|
||||
textareaRef.current.style.height = 'auto'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleImageSelection = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const selected = Array.from(event.target.files ?? [])
|
||||
event.target.value = ''
|
||||
if (selected.length === 0) return
|
||||
|
||||
const availableSlots = MAX_VISION_IMAGES - images.length
|
||||
if (availableSlots <= 0) {
|
||||
addNotification({ type: 'error', message: `You can attach up to ${MAX_VISION_IMAGES} images.` })
|
||||
return
|
||||
}
|
||||
|
||||
const attachments: ChatImageAttachment[] = []
|
||||
for (const file of selected.slice(0, availableSlots)) {
|
||||
if (!SUPPORTED_VISION_TYPES.has(file.type)) {
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: `${file.name} is not supported. Use JPEG, PNG, or WebP.`,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (file.size > MAX_VISION_IMAGE_BYTES) {
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: `${file.name} exceeds the 8 MB per-image limit.`,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const previewUrl = URL.createObjectURL(file)
|
||||
previewUrlsRef.current.add(previewUrl)
|
||||
attachments.push({
|
||||
id: crypto.randomUUID(),
|
||||
name: file.name,
|
||||
file,
|
||||
previewUrl,
|
||||
})
|
||||
}
|
||||
|
||||
if (selected.length > availableSlots) {
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: `Only the first ${availableSlots} selected image(s) were attached.`,
|
||||
})
|
||||
}
|
||||
setImages((current) => [...current, ...attachments])
|
||||
}
|
||||
|
||||
const removeImage = (image: ChatImageAttachment) => {
|
||||
URL.revokeObjectURL(image.previewUrl)
|
||||
previewUrlsRef.current.delete(image.previewUrl)
|
||||
setImages((current) => current.filter((item) => item.id !== image.id))
|
||||
}
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) {
|
||||
e.preventDefault()
|
||||
|
|
@ -139,7 +225,67 @@ export default function ChatInterface({
|
|||
)}
|
||||
</div>
|
||||
<div className="border-t border-border-subtle bg-surface-primary px-6 py-4 flex-shrink-0 min-h-[90px]">
|
||||
{images.length > 0 && (
|
||||
<div className="mb-3 flex flex-wrap gap-3" aria-label="Attached images">
|
||||
{images.map((image) => (
|
||||
<div
|
||||
key={image.id}
|
||||
className="relative h-20 w-20 overflow-hidden rounded-lg border border-border-default bg-surface-secondary"
|
||||
>
|
||||
<img src={image.previewUrl} alt={image.name} className="h-full w-full object-cover" />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeImage(image)}
|
||||
className="absolute right-1 top-1 rounded-full bg-surface-primary/90 p-1 text-text-primary hover:bg-surface-primary"
|
||||
aria-label={`Remove ${image.name}`}
|
||||
disabled={isLoading}
|
||||
>
|
||||
<IconX className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<form onSubmit={handleSubmit} className="flex gap-3 items-end">
|
||||
<input
|
||||
ref={imageInputRef}
|
||||
type="file"
|
||||
accept="image/jpeg,image/png,image/webp"
|
||||
multiple
|
||||
className="hidden"
|
||||
onChange={handleImageSelection}
|
||||
/>
|
||||
<div className="mb-2 flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => imageInputRef.current?.click()}
|
||||
disabled={
|
||||
isLoading ||
|
||||
images.length >= MAX_VISION_IMAGES ||
|
||||
visionCapability === 'unsupported'
|
||||
}
|
||||
className={classNames(
|
||||
'p-3 rounded-lg transition-colors flex-shrink-0',
|
||||
isLoading ||
|
||||
images.length >= MAX_VISION_IMAGES ||
|
||||
visionCapability === 'unsupported'
|
||||
? 'bg-border-default text-text-muted cursor-not-allowed'
|
||||
: 'border border-border-default text-text-secondary hover:bg-surface-secondary'
|
||||
)}
|
||||
aria-label="Attach images"
|
||||
>
|
||||
<IconPhoto className="h-6 w-6" aria-hidden="true" />
|
||||
</button>
|
||||
{visionCapability !== 'supported' && (
|
||||
<InfoTooltip
|
||||
position="top"
|
||||
align="left"
|
||||
text={
|
||||
visionAttachmentGuidance(visionCapability)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1 relative">
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
|
|
@ -155,10 +301,10 @@ export default function ChatInterface({
|
|||
</div>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={!input.trim() || isLoading}
|
||||
disabled={(!input.trim() && images.length === 0) || isLoading}
|
||||
className={classNames(
|
||||
'p-3 rounded-lg transition-all duration-200 flex-shrink-0 mb-2',
|
||||
!input.trim() || isLoading
|
||||
(!input.trim() && images.length === 0) || isLoading
|
||||
? 'bg-border-default text-text-muted cursor-not-allowed'
|
||||
: 'bg-desert-green text-white hover:bg-desert-green/90 hover:scale-105'
|
||||
)}
|
||||
|
|
@ -170,6 +316,9 @@ export default function ChatInterface({
|
|||
)}
|
||||
</button>
|
||||
</form>
|
||||
<p className="mt-2 text-xs text-text-muted" aria-live="polite">
|
||||
{visionAttachmentGuidance(visionCapability)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,18 @@ export default function ChatMessageBubble({ message }: ChatMessageBubbleProps) {
|
|||
message.role === 'user' ? 'bg-desert-green text-white' : 'bg-surface-secondary text-text-primary'
|
||||
)}
|
||||
>
|
||||
{message.images && message.images.length > 0 && (
|
||||
<div className="mb-3 grid grid-cols-2 gap-2">
|
||||
{message.images.map((image) => (
|
||||
<img
|
||||
key={image.id}
|
||||
src={image.previewUrl}
|
||||
alt={image.name}
|
||||
className="max-h-64 w-full rounded-md bg-surface-primary/20 object-contain"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{message.isThinking && message.thinking && (
|
||||
<div className="mb-3 rounded border border-amber-200 bg-amber-50 px-3 py-2 text-xs">
|
||||
<div className="mb-1 flex items-center gap-1.5 font-medium text-amber-700">
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import StyledModal from '../StyledModal'
|
|||
import api from '~/lib/api'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import { useModals } from '~/context/ModalContext'
|
||||
import { ChatMessage } from '../../../types/chat'
|
||||
import { ChatImageAttachment, ChatMessage } from '../../../types/chat'
|
||||
import classNames from '~/lib/classNames'
|
||||
import { IconMenu2, IconX } from '@tabler/icons-react'
|
||||
import { useSystemSetting } from '~/hooks/useSystemSetting'
|
||||
|
|
@ -139,6 +139,8 @@ export default function Chat({
|
|||
|
||||
const selectedModelSupportsThinking =
|
||||
installedModels.find((m) => m.name === selectedModel)?.thinking === true
|
||||
const selectedModelVisionCapability =
|
||||
installedModels.find((m) => m.name === selectedModel)?.vision ?? 'unknown'
|
||||
|
||||
// Effective thinking preference for a model: explicit override wins, else the global default.
|
||||
const effectiveThinking = useCallback(
|
||||
|
|
@ -179,6 +181,7 @@ export default function Chat({
|
|||
mutationFn: (request: {
|
||||
model: string
|
||||
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
|
||||
images?: File[]
|
||||
sessionId?: number
|
||||
think?: boolean
|
||||
collection?: string
|
||||
|
|
@ -357,7 +360,7 @@ export default function Chat({
|
|||
)
|
||||
|
||||
const handleSendMessage = useCallback(
|
||||
async (content: string) => {
|
||||
async (content: string, images: ChatImageAttachment[] = []) => {
|
||||
let sessionId = activeSessionId
|
||||
|
||||
// Create a new session if none exists
|
||||
|
|
@ -377,6 +380,7 @@ export default function Chat({
|
|||
id: `msg-${Date.now()}`,
|
||||
role: 'user',
|
||||
content,
|
||||
images,
|
||||
timestamp: new Date(),
|
||||
}
|
||||
|
||||
|
|
@ -410,6 +414,7 @@ export default function Chat({
|
|||
stream: true,
|
||||
sessionId: sessionId ? Number(sessionId) : undefined, think: effectiveThinking(selectedModel),
|
||||
collection: collectionFilter || undefined,
|
||||
images: images.map((image) => image.file),
|
||||
},
|
||||
(chunkContent, chunkThinking, done) => {
|
||||
if (chunkThinking.length > 0 && thinkingStartTime === null) {
|
||||
|
|
@ -478,7 +483,10 @@ export default function Chat({
|
|||
{
|
||||
id: assistantMsgId,
|
||||
role: 'assistant',
|
||||
content: 'Sorry, there was an error processing your request. Please try again.',
|
||||
content:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'Sorry, there was an error processing your request. Please try again.',
|
||||
timestamp: new Date(),
|
||||
},
|
||||
]
|
||||
|
|
@ -507,6 +515,7 @@ export default function Chat({
|
|||
sessionId: sessionId ? Number(sessionId) : undefined,
|
||||
think: effectiveThinking(selectedModel),
|
||||
collection: collectionFilter || undefined,
|
||||
images: images.map((image) => image.file),
|
||||
})
|
||||
}
|
||||
},
|
||||
|
|
@ -675,6 +684,7 @@ export default function Chat({
|
|||
<ChatInterface
|
||||
messages={messages}
|
||||
onSendMessage={handleSendMessage}
|
||||
visionCapability={selectedModelVisionCapability}
|
||||
isLoading={isStreamingResponse || chatMutation.isPending}
|
||||
chatSuggestions={chatSuggestions}
|
||||
chatSuggestionsEnabled={suggestionsEnabled}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,27 @@ import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatReq
|
|||
import BenchmarkResult from '#models/benchmark_result'
|
||||
import { BenchmarkType, RunBenchmarkResponse, SubmitBenchmarkResponse, UpdateBuilderTagResponse } from '../../types/benchmark'
|
||||
import type { ChatSource } from '../../types/chat'
|
||||
import { chatStreamErrorMessage } from './chat_stream.js'
|
||||
|
||||
type OllamaChatRequestWithImages = OllamaChatRequest & { images?: File[] }
|
||||
|
||||
function serializeChatRequest(chatRequest: OllamaChatRequestWithImages): {
|
||||
body: BodyInit
|
||||
headers?: Record<string, string>
|
||||
} {
|
||||
const { images = [], ...payload } = chatRequest
|
||||
if (images.length === 0) {
|
||||
return {
|
||||
body: JSON.stringify(payload),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}
|
||||
}
|
||||
|
||||
const formData = new FormData()
|
||||
formData.append('payload', JSON.stringify(payload))
|
||||
images.forEach((image) => formData.append('images', image, image.name))
|
||||
return { body: formData }
|
||||
}
|
||||
|
||||
class API {
|
||||
private client: AxiosInstance
|
||||
|
|
@ -311,29 +332,45 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async sendChatMessage(chatRequest: OllamaChatRequest) {
|
||||
async sendChatMessage(chatRequest: OllamaChatRequestWithImages) {
|
||||
return catchInternal(async () => {
|
||||
if (chatRequest.images?.length) {
|
||||
const serialized = serializeChatRequest({ ...chatRequest, stream: false })
|
||||
const response = await fetch('/api/ollama/chat', {
|
||||
method: 'POST',
|
||||
headers: serialized.headers,
|
||||
body: serialized.body,
|
||||
})
|
||||
const responseBody = await response.json().catch(() => null)
|
||||
if (!response.ok) {
|
||||
throw new Error(responseBody?.message ?? `HTTP error: ${response.status}`)
|
||||
}
|
||||
return responseBody as NomadChatResponse
|
||||
}
|
||||
|
||||
const response = await this.client.post<NomadChatResponse>('/ollama/chat', chatRequest)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async streamChatMessage(
|
||||
chatRequest: OllamaChatRequest,
|
||||
chatRequest: OllamaChatRequestWithImages,
|
||||
onChunk: (content: string, thinking: string, done: boolean) => void,
|
||||
signal?: AbortSignal,
|
||||
onSources?: (sources: ChatSource[]) => void
|
||||
): Promise<void> {
|
||||
// Axios doesn't support ReadableStream in browser, so need to use fetch
|
||||
const serialized = serializeChatRequest({ ...chatRequest, stream: true })
|
||||
const response = await fetch('/api/ollama/chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ...chatRequest, stream: true }),
|
||||
headers: serialized.headers,
|
||||
body: serialized.body,
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`HTTP error: ${response.status}`)
|
||||
const errorBody = await response.json().catch(() => null)
|
||||
throw new Error(errorBody?.message ?? `HTTP error: ${response.status}`)
|
||||
}
|
||||
|
||||
const reader = response.body.getReader()
|
||||
|
|
@ -356,7 +393,8 @@ class API {
|
|||
data = JSON.parse(line.slice(6))
|
||||
} catch { continue /* skip malformed chunks */ }
|
||||
|
||||
if (data.error) throw new Error('The model encountered an error. Please try again.')
|
||||
const streamError = chatStreamErrorMessage(data)
|
||||
if (streamError) throw new Error(streamError)
|
||||
|
||||
// Citation metadata (#1179) arrives as a distinct trailing event with no
|
||||
// `message` key -- route it separately rather than through onChunk.
|
||||
|
|
|
|||
10
admin/inertia/lib/chat_stream.ts
Normal file
10
admin/inertia/lib/chat_stream.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
const GENERIC_CHAT_STREAM_ERROR = 'The model encountered an error. Please try again.'
|
||||
|
||||
export function chatStreamErrorMessage(event: unknown): string | null {
|
||||
if (!event || typeof event !== 'object' || !('error' in event) || !event.error) {
|
||||
return null
|
||||
}
|
||||
|
||||
const message = 'message' in event ? event.message : undefined
|
||||
return typeof message === 'string' && message.trim() ? message : GENERIC_CHAT_STREAM_ERROR
|
||||
}
|
||||
16
admin/inertia/lib/vision_guidance.ts
Normal file
16
admin/inertia/lib/vision_guidance.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import type { ModelVisionCapability } from '../../types/ollama.js'
|
||||
|
||||
const TEMPORARY_IMAGE_NOTICE =
|
||||
'Images are sent only with this request. They are not saved and will disappear after you send them or reload this page.'
|
||||
|
||||
export function visionAttachmentGuidance(capability: ModelVisionCapability): string {
|
||||
if (capability === 'unsupported') {
|
||||
return 'This model cannot use images. Choose a model marked “Supports images” in Models & Settings.'
|
||||
}
|
||||
|
||||
if (capability === 'unknown') {
|
||||
return `NOMAD cannot confirm whether this model accepts images. You can try an image, but the request will fail if the model is text-only. ${TEMPORARY_IMAGE_NOTICE}`
|
||||
}
|
||||
|
||||
return TEMPORARY_IMAGE_NOTICE
|
||||
}
|
||||
|
|
@ -1128,6 +1128,18 @@ export default function EasySetupWizard(props: {
|
|||
Size: {model.tags[0].size}
|
||||
</div>
|
||||
)}
|
||||
{model.tags?.[0]?.input.toLowerCase().includes('image') && (
|
||||
<div
|
||||
className={classNames(
|
||||
'mt-1 text-xs font-medium',
|
||||
selectedAiModels.includes(model.name)
|
||||
? 'text-green-100'
|
||||
: 'text-desert-green'
|
||||
)}
|
||||
>
|
||||
Supports images
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div
|
||||
className={classNames(
|
||||
|
|
|
|||
274
admin/tests/unit/chat_images.spec.ts
Normal file
274
admin/tests/unit/chat_images.spec.ts
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import type { MultipartFile } from '@adonisjs/bodyparser/types'
|
||||
import sharp from 'sharp'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
attachImagesToLatestUserMessage,
|
||||
ChatImageError,
|
||||
normalizeChatImages,
|
||||
} from '../../app/utils/chat_images.js'
|
||||
import {
|
||||
capabilitiesFromOllamaShow,
|
||||
installedModelsFromOpenAIResponse,
|
||||
resolveModelCapabilities,
|
||||
visionCapabilityFromLlamaProps,
|
||||
} from '../../app/utils/model_capabilities.js'
|
||||
|
||||
function uploadedFile(path: string, size: number, name = 'sample.png'): MultipartFile {
|
||||
return {
|
||||
tmpPath: path,
|
||||
size,
|
||||
clientName: name,
|
||||
isValid: true,
|
||||
} as MultipartFile
|
||||
}
|
||||
|
||||
test('normalizes a supported image to a bounded JPEG data URL', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'nomad-chat-image-'))
|
||||
const path = join(directory, 'sample.png')
|
||||
const input = await sharp({
|
||||
create: {
|
||||
width: 32,
|
||||
height: 24,
|
||||
channels: 3,
|
||||
background: '#556b2f',
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
await writeFile(path, input)
|
||||
|
||||
try {
|
||||
const [image] = await normalizeChatImages([uploadedFile(path, input.byteLength)])
|
||||
assert.equal(image.name, 'sample.png')
|
||||
assert.match(image.dataUrl, /^data:image\/jpeg;base64,/)
|
||||
const decoded = Buffer.from(image.dataUrl.split(',')[1], 'base64')
|
||||
const metadata = await sharp(decoded).metadata()
|
||||
assert.equal(metadata.format, 'jpeg')
|
||||
assert.equal(metadata.width, 32)
|
||||
assert.equal(metadata.height, 24)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('normalizes image bytes when the filename extension does not match the encoding', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'nomad-chat-image-'))
|
||||
const path = join(directory, 'mislabeled.png')
|
||||
const input = await sharp({
|
||||
create: {
|
||||
width: 32,
|
||||
height: 24,
|
||||
channels: 3,
|
||||
background: '#556b2f',
|
||||
},
|
||||
})
|
||||
.webp()
|
||||
.toBuffer()
|
||||
await writeFile(path, input)
|
||||
|
||||
try {
|
||||
const [image] = await normalizeChatImages([
|
||||
uploadedFile(path, input.byteLength, 'mislabeled.png'),
|
||||
])
|
||||
assert.equal(image.name, 'mislabeled.png')
|
||||
assert.match(image.dataUrl, /^data:image\/jpeg;base64,/)
|
||||
const decoded = Buffer.from(image.dataUrl.split(',')[1], 'base64')
|
||||
const metadata = await sharp(decoded).metadata()
|
||||
assert.equal(metadata.format, 'jpeg')
|
||||
assert.equal(metadata.width, 32)
|
||||
assert.equal(metadata.height, 24)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects payloads with more than four images', async () => {
|
||||
const files = Array.from({ length: 5 }, (_, index) =>
|
||||
uploadedFile(`/tmp/image-${index}.png`, 10, `image-${index}.png`)
|
||||
)
|
||||
await assert.rejects(
|
||||
() => normalizeChatImages(files),
|
||||
(error: unknown) => {
|
||||
assert.ok(error instanceof ChatImageError)
|
||||
assert.equal(error.status, 422)
|
||||
return true
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
test('attaches images only to the latest user message', () => {
|
||||
const messages = [
|
||||
{ role: 'user' as const, content: 'Earlier question' },
|
||||
{ role: 'assistant' as const, content: 'Earlier answer' },
|
||||
{ role: 'user' as const, content: 'What is shown?' },
|
||||
]
|
||||
const result = attachImagesToLatestUserMessage(messages, [
|
||||
{ name: 'sample.webp', dataUrl: 'data:image/webp;base64,AAAA' },
|
||||
])
|
||||
|
||||
assert.equal(result[0].content, 'Earlier question')
|
||||
assert.equal(result[1].content, 'Earlier answer')
|
||||
assert.deepEqual(result[2], {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'What is shown?' },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/webp;base64,AAAA' },
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test('detects vision capability from Ollama and llama.cpp metadata', () => {
|
||||
assert.deepEqual(capabilitiesFromOllamaShow({ capabilities: ['completion', 'vision'] }), {
|
||||
thinking: false,
|
||||
vision: 'supported',
|
||||
})
|
||||
assert.deepEqual(capabilitiesFromOllamaShow({ capabilities: ['completion', 'thinking'] }), {
|
||||
thinking: true,
|
||||
vision: 'unsupported',
|
||||
})
|
||||
assert.equal(visionCapabilityFromLlamaProps({ modalities: { vision: true } }), 'supported')
|
||||
assert.equal(visionCapabilityFromLlamaProps({ modalities: { vision: false } }), 'unsupported')
|
||||
assert.equal(visionCapabilityFromLlamaProps({ modalities: ['text', 'image'] }), 'supported')
|
||||
assert.equal(visionCapabilityFromLlamaProps({ model: 'unknown' }), null)
|
||||
})
|
||||
|
||||
test('resolves per-model capabilities advertised by a hybrid router', async () => {
|
||||
let showProbeCalls = 0
|
||||
let propsProbeCalls = 0
|
||||
const probes = {
|
||||
ollamaShow: async () => {
|
||||
showProbeCalls += 1
|
||||
return null
|
||||
},
|
||||
llamaProps: async () => {
|
||||
propsProbeCalls += 1
|
||||
return null
|
||||
},
|
||||
}
|
||||
|
||||
const visionModel = await resolveModelCapabilities(
|
||||
{ capabilities: ['completion', 'vision', 'tools', 'thinking'] },
|
||||
probes
|
||||
)
|
||||
const textModel = await resolveModelCapabilities(
|
||||
{ capabilities: ['completion', 'tools', 'thinking'] },
|
||||
probes
|
||||
)
|
||||
|
||||
assert.deepEqual(visionModel, { thinking: true, vision: 'supported' })
|
||||
assert.deepEqual(textModel, { thinking: true, vision: 'unsupported' })
|
||||
assert.equal(showProbeCalls, 0)
|
||||
assert.equal(propsProbeCalls, 0)
|
||||
})
|
||||
|
||||
test('preserves per-model capabilities from an OpenAI-compatible model list', () => {
|
||||
const models = installedModelsFromOpenAIResponse({
|
||||
data: [
|
||||
{
|
||||
id: 'qwen3.6:35b-a3b-hauhau-aggressive',
|
||||
capabilities: ['completion', 'vision', 'tools', 'thinking'],
|
||||
},
|
||||
{
|
||||
id: 'qwopus3.6-coder-compat-mtp:27b',
|
||||
capabilities: ['completion', 'tools', 'thinking'],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
assert.deepEqual(models, [
|
||||
{
|
||||
name: 'qwen3.6:35b-a3b-hauhau-aggressive',
|
||||
size: 0,
|
||||
capabilities: ['completion', 'vision', 'tools', 'thinking'],
|
||||
},
|
||||
{
|
||||
name: 'qwopus3.6-coder-compat-mtp:27b',
|
||||
size: 0,
|
||||
capabilities: ['completion', 'tools', 'thinking'],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test('falls back per model from Ollama metadata to llama.cpp properties', async () => {
|
||||
let propsProbeCalls = 0
|
||||
const ollamaModel = await resolveModelCapabilities(null, {
|
||||
ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
|
||||
llamaProps: async () => {
|
||||
propsProbeCalls += 1
|
||||
return { modalities: { vision: false } }
|
||||
},
|
||||
})
|
||||
const llamaModel = await resolveModelCapabilities(null, {
|
||||
ollamaShow: async () => {
|
||||
throw new Error('404')
|
||||
},
|
||||
llamaProps: async () => {
|
||||
propsProbeCalls += 1
|
||||
return { modalities: { vision: false } }
|
||||
},
|
||||
})
|
||||
|
||||
assert.deepEqual(ollamaModel, { thinking: false, vision: 'supported' })
|
||||
assert.deepEqual(llamaModel, { thinking: false, vision: 'unsupported' })
|
||||
assert.equal(propsProbeCalls, 1)
|
||||
})
|
||||
|
||||
test('classifies the four supported backend capability cases', async () => {
|
||||
const backendCases = [
|
||||
{
|
||||
name: 'Ollama vision',
|
||||
advertised: null,
|
||||
ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
|
||||
llamaProps: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
expected: { thinking: false, vision: 'supported' },
|
||||
},
|
||||
{
|
||||
name: 'llama.cpp vision',
|
||||
advertised: null,
|
||||
ollamaShow: async () => {
|
||||
throw new Error('404')
|
||||
},
|
||||
llamaProps: async () => ({ modalities: { vision: true } }),
|
||||
expected: { thinking: false, vision: 'supported' },
|
||||
},
|
||||
{
|
||||
name: 'known text-only',
|
||||
advertised: { capabilities: ['completion', 'thinking'] },
|
||||
ollamaShow: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
llamaProps: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
expected: { thinking: true, vision: 'unsupported' },
|
||||
},
|
||||
{
|
||||
name: 'unknown OpenAI-compatible',
|
||||
advertised: null,
|
||||
ollamaShow: async () => {
|
||||
throw new Error('404')
|
||||
},
|
||||
llamaProps: async () => {
|
||||
throw new Error('404')
|
||||
},
|
||||
expected: null,
|
||||
},
|
||||
] as const
|
||||
|
||||
for (const backend of backendCases) {
|
||||
const actual = await resolveModelCapabilities(backend.advertised, {
|
||||
ollamaShow: backend.ollamaShow,
|
||||
llamaProps: backend.llamaProps,
|
||||
})
|
||||
assert.deepEqual(actual, backend.expected, backend.name)
|
||||
}
|
||||
})
|
||||
20
admin/tests/unit/chat_request_compat.spec.ts
Normal file
20
admin/tests/unit/chat_request_compat.spec.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { chatSchema } from '../../app/validators/ollama.js'
|
||||
|
||||
const message = (content: string) => ({ role: 'user' as const, content })
|
||||
|
||||
test('text-only chat validation preserves empty and long conversation histories', async () => {
|
||||
const empty = await chatSchema.validate({
|
||||
model: 'text-model',
|
||||
messages: [],
|
||||
stream: true,
|
||||
})
|
||||
const long = await chatSchema.validate({
|
||||
model: 'text-model',
|
||||
messages: Array.from({ length: 201 }, (_, index) => message(`Message ${index}`)),
|
||||
})
|
||||
|
||||
assert.equal(empty.messages.length, 0)
|
||||
assert.equal(long.messages.length, 201)
|
||||
})
|
||||
19
admin/tests/unit/chat_stream.spec.ts
Normal file
19
admin/tests/unit/chat_stream.spec.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { chatStreamErrorMessage } from '../../inertia/lib/chat_stream.js'
|
||||
|
||||
test('chat stream preserves an actionable server error message', () => {
|
||||
assert.equal(
|
||||
chatStreamErrorMessage({
|
||||
error: true,
|
||||
message:
|
||||
'NOMAD cannot confirm that this model accepts images. Choose a model marked “Supports images”.',
|
||||
}),
|
||||
'NOMAD cannot confirm that this model accepts images. Choose a model marked “Supports images”.'
|
||||
)
|
||||
assert.equal(
|
||||
chatStreamErrorMessage({ error: true }),
|
||||
'The model encountered an error. Please try again.'
|
||||
)
|
||||
assert.equal(chatStreamErrorMessage({ done: true }), null)
|
||||
})
|
||||
342
admin/tests/unit/ollama_controller_chat.spec.ts
Normal file
342
admin/tests/unit/ollama_controller_chat.spec.ts
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import sharp from 'sharp'
|
||||
import OllamaController from '../../app/controllers/ollama_controller.js'
|
||||
|
||||
class FakeSseResponse extends EventEmitter {
|
||||
chunks: string[] = []
|
||||
ended = false
|
||||
|
||||
setHeader() {}
|
||||
flushHeaders() {}
|
||||
write(chunk: string) {
|
||||
this.chunks.push(chunk)
|
||||
}
|
||||
end() {
|
||||
this.ended = true
|
||||
}
|
||||
}
|
||||
|
||||
test('unknown backend image rejection returns actionable compatibility details over SSE', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'nomad-controller-image-'))
|
||||
const imagePath = join(directory, 'sample.png')
|
||||
const image = await sharp({
|
||||
create: {
|
||||
width: 4,
|
||||
height: 4,
|
||||
channels: 3,
|
||||
background: '#556b2f',
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
await writeFile(imagePath, image)
|
||||
|
||||
const rawResponse = new FakeSseResponse()
|
||||
const payload = {
|
||||
model: 'openai-compatible-model',
|
||||
messages: [{ role: 'user', content: 'What is shown?' }],
|
||||
stream: true,
|
||||
}
|
||||
const request = {
|
||||
files: () => [
|
||||
{
|
||||
tmpPath: imagePath,
|
||||
size: image.byteLength,
|
||||
clientName: 'sample.png',
|
||||
isValid: true,
|
||||
},
|
||||
],
|
||||
input: (name: string, fallback?: unknown) =>
|
||||
name === 'payload' ? JSON.stringify(payload) : fallback,
|
||||
}
|
||||
const response = {
|
||||
response: rawResponse,
|
||||
status: () => response,
|
||||
send: () => undefined,
|
||||
}
|
||||
const ollamaService = {
|
||||
getModelCapabilities: async () => ({ thinking: false, vision: 'unknown' as const }),
|
||||
chatStream: async () => {
|
||||
throw new Error('The upstream backend rejected image_url content.')
|
||||
},
|
||||
}
|
||||
const controller = new OllamaController(
|
||||
{} as any,
|
||||
{} as any,
|
||||
ollamaService as any,
|
||||
// dev moved prompt assembly, NOMAD.md injection and retrieval into
|
||||
// RagPipelineService after this test was written. The stub returns the
|
||||
// messages untouched so these cases still exercise only the vision paths.
|
||||
{
|
||||
buildPrompt: async (messages: unknown) => ({
|
||||
messages,
|
||||
numCtx: undefined,
|
||||
numPredict: undefined,
|
||||
retrieved: [],
|
||||
injected: [],
|
||||
}),
|
||||
} as any,
|
||||
{
|
||||
hasDocuments: async () => false,
|
||||
searchSimilarDocuments: async () => [],
|
||||
} as any,
|
||||
{ record: async () => undefined } as any
|
||||
)
|
||||
|
||||
try {
|
||||
await controller.chat({ request, response } as any)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
assert.equal(rawResponse.ended, true)
|
||||
assert.equal(rawResponse.chunks.length, 1)
|
||||
const event = JSON.parse(rawResponse.chunks[0].replace(/^data: /, '').trim())
|
||||
assert.equal(event.error, true)
|
||||
assert.match(event.message, /cannot confirm/i)
|
||||
assert.match(event.message, /supports images/i)
|
||||
assert.doesNotMatch(event.message, /backend|metadata|projector/i)
|
||||
})
|
||||
|
||||
test('unknown backend image rejection returns actionable compatibility details without streaming', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'nomad-controller-image-'))
|
||||
const imagePath = join(directory, 'sample.png')
|
||||
const image = await sharp({
|
||||
create: {
|
||||
width: 4,
|
||||
height: 4,
|
||||
channels: 3,
|
||||
background: '#556b2f',
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
await writeFile(imagePath, image)
|
||||
|
||||
const payload = {
|
||||
model: 'openai-compatible-model',
|
||||
messages: [{ role: 'user', content: 'What is shown?' }],
|
||||
stream: false,
|
||||
}
|
||||
const request = {
|
||||
files: () => [
|
||||
{
|
||||
tmpPath: imagePath,
|
||||
size: image.byteLength,
|
||||
clientName: 'sample.png',
|
||||
isValid: true,
|
||||
},
|
||||
],
|
||||
input: (name: string, fallback?: unknown) =>
|
||||
name === 'payload' ? JSON.stringify(payload) : fallback,
|
||||
}
|
||||
let statusCode: number | undefined
|
||||
let responseBody: unknown
|
||||
const response = {
|
||||
response: new FakeSseResponse(),
|
||||
status: (code: number) => {
|
||||
statusCode = code
|
||||
return response
|
||||
},
|
||||
send: (body: unknown) => {
|
||||
responseBody = body
|
||||
return body
|
||||
},
|
||||
}
|
||||
const ollamaService = {
|
||||
getModelCapabilities: async () => ({ thinking: false, vision: 'unknown' as const }),
|
||||
chat: async () => {
|
||||
throw new Error('The upstream backend rejected image_url content.')
|
||||
},
|
||||
}
|
||||
const controller = new OllamaController(
|
||||
{} as any,
|
||||
{} as any,
|
||||
ollamaService as any,
|
||||
// dev moved prompt assembly, NOMAD.md injection and retrieval into
|
||||
// RagPipelineService after this test was written. The stub returns the
|
||||
// messages untouched so these cases still exercise only the vision paths.
|
||||
{
|
||||
buildPrompt: async (messages: unknown) => ({
|
||||
messages,
|
||||
numCtx: undefined,
|
||||
numPredict: undefined,
|
||||
retrieved: [],
|
||||
injected: [],
|
||||
}),
|
||||
} as any,
|
||||
{
|
||||
hasDocuments: async () => false,
|
||||
searchSimilarDocuments: async () => [],
|
||||
} as any,
|
||||
{ record: async () => undefined } as any
|
||||
)
|
||||
|
||||
try {
|
||||
await controller.chat({ request, response } as any)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
assert.equal(statusCode, 422)
|
||||
assert.match((responseBody as { message: string }).message, /cannot confirm/i)
|
||||
assert.match((responseBody as { message: string }).message, /supports images/i)
|
||||
assert.doesNotMatch(
|
||||
(responseBody as { message: string }).message,
|
||||
/backend|metadata|projector/i
|
||||
)
|
||||
})
|
||||
|
||||
test('unknown backend image requests do not mislabel preprocessing failures as incompatibility', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'nomad-controller-image-'))
|
||||
const imagePath = join(directory, 'sample.png')
|
||||
const image = await sharp({
|
||||
create: {
|
||||
width: 4,
|
||||
height: 4,
|
||||
channels: 3,
|
||||
background: '#556b2f',
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
await writeFile(imagePath, image)
|
||||
|
||||
const rawResponse = new FakeSseResponse()
|
||||
const payload = {
|
||||
model: 'openai-compatible-model',
|
||||
messages: [{ role: 'user', content: 'What is shown?' }],
|
||||
stream: true,
|
||||
}
|
||||
const request = {
|
||||
files: () => [
|
||||
{
|
||||
tmpPath: imagePath,
|
||||
size: image.byteLength,
|
||||
clientName: 'sample.png',
|
||||
isValid: true,
|
||||
},
|
||||
],
|
||||
input: (name: string, fallback?: unknown) =>
|
||||
name === 'payload' ? JSON.stringify(payload) : fallback,
|
||||
}
|
||||
const response = {
|
||||
response: rawResponse,
|
||||
status: () => response,
|
||||
send: () => undefined,
|
||||
}
|
||||
let inferenceCalled = false
|
||||
const ollamaService = {
|
||||
getModelCapabilities: async () => ({ thinking: false, vision: 'unknown' as const }),
|
||||
chatStream: async () => {
|
||||
inferenceCalled = true
|
||||
throw new Error('Inference should not be reached')
|
||||
},
|
||||
}
|
||||
const controller = new OllamaController(
|
||||
{} as any,
|
||||
{} as any,
|
||||
ollamaService as any,
|
||||
// Prompt assembly (NOMAD.md included) now lives in RagPipelineService, so
|
||||
// that is where this failure originates. The assertion is unchanged and is
|
||||
// the one that matters: a failure before inference must not reach the model,
|
||||
// and must still close the stream with a single error event.
|
||||
{
|
||||
buildPrompt: async () => {
|
||||
throw new Error('Prompt assembly unavailable')
|
||||
},
|
||||
} as any,
|
||||
{
|
||||
hasDocuments: async () => false,
|
||||
searchSimilarDocuments: async () => [],
|
||||
} as any,
|
||||
{ record: async () => undefined } as any
|
||||
)
|
||||
|
||||
try {
|
||||
await controller.chat({ request, response } as any)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
assert.equal(inferenceCalled, false)
|
||||
assert.equal(rawResponse.ended, true)
|
||||
assert.equal(rawResponse.chunks.length, 1)
|
||||
const event = JSON.parse(rawResponse.chunks[0].replace(/^data: /, '').trim())
|
||||
assert.deepEqual(event, { error: true })
|
||||
})
|
||||
|
||||
test('text-only JSON chat preserves long histories through the controller boundary', async () => {
|
||||
const messages = Array.from({ length: 201 }, (_, index) => ({
|
||||
role: 'user' as const,
|
||||
content: `Message ${index}`,
|
||||
}))
|
||||
const payload = {
|
||||
model: 'known-text-model',
|
||||
messages,
|
||||
stream: false,
|
||||
}
|
||||
const request = {
|
||||
files: () => [],
|
||||
input: (name: string, fallback?: unknown) =>
|
||||
name === 'collection' ? 'A'.repeat(500) : fallback,
|
||||
validateUsing: (schema: { validate(value: unknown): Promise<unknown> }) =>
|
||||
schema.validate(payload),
|
||||
}
|
||||
const response = {
|
||||
response: new FakeSseResponse(),
|
||||
status: () => response,
|
||||
send: () => undefined,
|
||||
}
|
||||
let upstreamMessages: unknown[] = []
|
||||
const ollamaService = {
|
||||
getModelCapabilities: async () => ({ thinking: false, vision: 'unsupported' as const }),
|
||||
chat: async (chatRequest: { messages: unknown[] }) => {
|
||||
upstreamMessages = chatRequest.messages
|
||||
return {
|
||||
message: { content: 'Text response' },
|
||||
done: true,
|
||||
model: 'known-text-model',
|
||||
}
|
||||
},
|
||||
}
|
||||
const controller = new OllamaController(
|
||||
{} as any,
|
||||
{} as any,
|
||||
ollamaService as any,
|
||||
// dev moved prompt assembly, NOMAD.md injection and retrieval into
|
||||
// RagPipelineService after this test was written. The stub returns the
|
||||
// messages untouched so these cases still exercise only the vision paths.
|
||||
{
|
||||
buildPrompt: async (messages: unknown) => ({
|
||||
messages,
|
||||
numCtx: undefined,
|
||||
numPredict: undefined,
|
||||
retrieved: [],
|
||||
injected: [],
|
||||
}),
|
||||
} as any,
|
||||
{
|
||||
hasDocuments: async () => false,
|
||||
searchSimilarDocuments: async () => [],
|
||||
} as any,
|
||||
{ record: async () => undefined } as any
|
||||
)
|
||||
|
||||
const result = await controller.chat({ request, response } as any)
|
||||
|
||||
assert.ok(result)
|
||||
assert.equal(result.message.content, 'Text response')
|
||||
assert.equal(upstreamMessages.length, 202)
|
||||
assert.deepEqual(upstreamMessages.slice(1), messages)
|
||||
assert.ok(
|
||||
upstreamMessages.every(
|
||||
(message) => typeof (message as { content?: unknown }).content === 'string'
|
||||
)
|
||||
)
|
||||
})
|
||||
24
admin/tests/unit/vision_guidance.spec.ts
Normal file
24
admin/tests/unit/vision_guidance.spec.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { visionAttachmentGuidance } from '../../inertia/lib/vision_guidance.js'
|
||||
|
||||
test('vision guidance makes temporary image handling explicit', () => {
|
||||
const guidance = visionAttachmentGuidance('supported')
|
||||
|
||||
assert.match(guidance, /only with this request/i)
|
||||
assert.match(guidance, /not saved/i)
|
||||
assert.match(guidance, /reload/i)
|
||||
})
|
||||
|
||||
test('unknown vision guidance explains the user choice without backend jargon', () => {
|
||||
const guidance = visionAttachmentGuidance('unknown')
|
||||
|
||||
assert.match(guidance, /cannot confirm/i)
|
||||
assert.match(guidance, /try/i)
|
||||
assert.match(guidance, /request will fail/i)
|
||||
assert.doesNotMatch(guidance, /backend|metadata|projector/i)
|
||||
})
|
||||
|
||||
test('unsupported vision guidance points users to a compatible model', () => {
|
||||
assert.match(visionAttachmentGuidance('unsupported'), /supports images/i)
|
||||
})
|
||||
|
|
@ -13,6 +13,7 @@ export interface ChatMessage {
|
|||
id: string
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
content: string
|
||||
images?: ChatImageAttachment[]
|
||||
timestamp: Date
|
||||
isStreaming?: boolean
|
||||
thinking?: string
|
||||
|
|
@ -21,6 +22,13 @@ export interface ChatMessage {
|
|||
sources?: ChatSource[]
|
||||
}
|
||||
|
||||
export interface ChatImageAttachment {
|
||||
id: string
|
||||
name: string
|
||||
file: File
|
||||
previewUrl: string
|
||||
}
|
||||
|
||||
export interface ChatSession {
|
||||
id: string
|
||||
title: string
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ export type OllamaChatRequest = {
|
|||
collection?: string
|
||||
}
|
||||
|
||||
export type ModelVisionCapability = 'supported' | 'unsupported' | 'unknown'
|
||||
|
||||
export type OllamaChatResponse = {
|
||||
model: string
|
||||
created_at: string
|
||||
|
|
@ -55,6 +57,9 @@ export type NomadInstalledModel = {
|
|||
details?: Record<string, any>
|
||||
// Whether the model supports "thinking" (set by the installed-models endpoint enrichment).
|
||||
thinking?: boolean
|
||||
// Capability detection is authoritative for Ollama and llama.cpp. Other OpenAI-compatible
|
||||
// backends report "unknown" so the user can still try an image-capable model.
|
||||
vision?: ModelVisionCapability
|
||||
}
|
||||
|
||||
export type NomadChatResponse = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user