fix(ai): surface model download errors and prevent silent retry loops

Model downloads that fail (e.g., when Ollama is too old for a model)
were silently retrying 40 times with no UI feedback. Now errors are
broadcast via SSE and shown in the Active Model Downloads section.
Version mismatch errors use UnrecoverableError to fail immediately
instead of retrying. Stale failed jobs are cleared on retry so users
aren't permanently blocked.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood 2026-03-20 17:55:41 -07:00 committed by Jake Turner
parent 808660b4bf
commit 8c04fbaee3
4 changed files with 78 additions and 20 deletions

View File

@ -1,4 +1,4 @@
import { Job } from 'bullmq' import { Job, UnrecoverableError } from 'bullmq'
import { QueueService } from '#services/queue_service' import { QueueService } from '#services/queue_service'
import { createHash } from 'crypto' import { createHash } from 'crypto'
import logger from '@adonisjs/core/services/logger' import logger from '@adonisjs/core/services/logger'
@ -63,6 +63,10 @@ export class DownloadModelJob {
logger.error( logger.error(
`[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}` `[DownloadModelJob] Failed to initiate download for model ${modelName}: ${result.message}`
) )
// Don't retry errors that will never succeed (e.g., Ollama version too old)
if (result.retryable === false) {
throw new UnrecoverableError(result.message)
}
throw new Error(`Failed to initiate download for model: ${result.message}`) throw new Error(`Failed to initiate download for model: ${result.message}`)
} }
@ -85,6 +89,15 @@ export class DownloadModelJob {
const queue = queueService.getQueue(this.queue) const queue = queueService.getQueue(this.queue)
const jobId = this.getJobId(params.modelName) const jobId = this.getJobId(params.modelName)
// Clear any previous failed job so a fresh attempt can be dispatched
const existing = await queue.getJob(jobId)
if (existing) {
const state = await existing.getState()
if (state === 'failed') {
await existing.remove()
}
}
try { try {
const job = await queue.add(this.key, params, { const job = await queue.add(this.key, params, {
jobId, jobId,
@ -104,9 +117,9 @@ export class DownloadModelJob {
} }
} catch (error) { } catch (error) {
if (error.message.includes('job already exists')) { if (error.message.includes('job already exists')) {
const existing = await queue.getJob(jobId) const active = await queue.getJob(jobId)
return { return {
job: existing, job: active,
created: false, created: false,
message: `Job already exists for model ${params.modelName}`, message: `Job already exists for model ${params.modelName}`,
} }

View File

@ -51,7 +51,7 @@ export class OllamaService {
* @param model Model name to download * @param model Model name to download
* @returns Success status and message * @returns Success status and message
*/ */
async downloadModel(model: string, progressCallback?: (percent: number) => void): Promise<{ success: boolean; message: string }> { async downloadModel(model: string, progressCallback?: (percent: number) => void): Promise<{ success: boolean; message: string; retryable?: boolean }> {
try { try {
await this._ensureDependencies() await this._ensureDependencies()
if (!this.ollama) { if (!this.ollama) {
@ -86,11 +86,21 @@ export class OllamaService {
logger.info(`[OllamaService] Model "${model}" downloaded successfully.`) logger.info(`[OllamaService] Model "${model}" downloaded successfully.`)
return { success: true, message: 'Model downloaded successfully.' } return { success: true, message: 'Model downloaded successfully.' }
} catch (error) { } catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error)
logger.error( logger.error(
`[OllamaService] Failed to download model "${model}": ${error instanceof Error ? error.message : error `[OllamaService] Failed to download model "${model}": ${errorMessage}`
}`
) )
return { success: false, message: 'Failed to download model.' }
// Check for version mismatch (Ollama 412 response)
const isVersionMismatch = errorMessage.includes('newer version of Ollama')
const userMessage = isVersionMismatch
? 'This model requires a newer version of Ollama. Please update AI Assistant from the Apps page.'
: `Failed to download model: ${errorMessage}`
// Broadcast failure to connected clients so UI can show the error
this.broadcastDownloadError(model, userMessage)
return { success: false, message: userMessage, retryable: !isVersionMismatch }
} }
} }
@ -379,6 +389,15 @@ export class OllamaService {
return models return models
} }
private broadcastDownloadError(model: string, error: string) {
transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
model,
percent: -1,
error,
timestamp: new Date().toISOString(),
})
}
private broadcastDownloadProgress(model: string, percent: number) { private broadcastDownloadProgress(model: string, percent: number) {
transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, { transmit.broadcast(BROADCAST_CHANNELS.OLLAMA_MODEL_DOWNLOAD, {
model, model,

View File

@ -1,6 +1,7 @@
import useOllamaModelDownloads from '~/hooks/useOllamaModelDownloads' import useOllamaModelDownloads from '~/hooks/useOllamaModelDownloads'
import HorizontalBarChart from './HorizontalBarChart' import HorizontalBarChart from './HorizontalBarChart'
import StyledSectionHeader from './StyledSectionHeader' import StyledSectionHeader from './StyledSectionHeader'
import { IconAlertTriangle } from '@tabler/icons-react'
interface ActiveModelDownloadsProps { interface ActiveModelDownloadsProps {
withHeader?: boolean withHeader?: boolean
@ -17,19 +18,31 @@ const ActiveModelDownloads = ({ withHeader = false }: ActiveModelDownloadsProps)
downloads.map((download) => ( downloads.map((download) => (
<div <div
key={download.model} key={download.model}
className="bg-desert-white rounded-lg p-4 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow" className={`bg-desert-white rounded-lg p-4 border shadow-sm hover:shadow-lg transition-shadow ${
download.error ? 'border-red-400' : 'border-desert-stone-light'
}`}
> >
<HorizontalBarChart {download.error ? (
items={[ <div className="flex items-start gap-3">
{ <IconAlertTriangle className="text-red-500 flex-shrink-0 mt-0.5" size={20} />
label: download.model, <div>
value: download.percent, <p className="font-medium text-text-primary">{download.model}</p>
total: '100%', <p className="text-sm text-red-600 mt-1">{download.error}</p>
used: `${download.percent.toFixed(1)}%`, </div>
type: 'ollama-model', </div>
}, ) : (
]} <HorizontalBarChart
/> items={[
{
label: download.model,
value: download.percent,
total: '100%',
used: `${download.percent.toFixed(1)}%`,
type: 'ollama-model',
},
]}
/>
)}
</div> </div>
)) ))
) : ( ) : (

View File

@ -5,6 +5,7 @@ export type OllamaModelDownload = {
model: string model: string
percent: number percent: number
timestamp: string timestamp: string
error?: string
} }
export default function useOllamaModelDownloads() { export default function useOllamaModelDownloads() {
@ -17,7 +18,19 @@ export default function useOllamaModelDownloads() {
setDownloads((prev) => { setDownloads((prev) => {
const updated = new Map(prev) const updated = new Map(prev)
if (data.percent >= 100) { if (data.percent === -1) {
// Download failed — show error state, auto-remove after 15 seconds
updated.set(data.model, data)
const errorTimeout = setTimeout(() => {
timeoutsRef.current.delete(errorTimeout)
setDownloads((current) => {
const next = new Map(current)
next.delete(data.model)
return next
})
}, 15000)
timeoutsRef.current.add(errorTimeout)
} else if (data.percent >= 100) {
// If download is complete, keep it for a short time before removing to allow UI to show 100% progress // If download is complete, keep it for a short time before removing to allow UI to show 100% progress
updated.set(data.model, data) updated.set(data.model, data)
const timeout = setTimeout(() => { const timeout = setTimeout(() => {