From ac437f8429badee9aac448c33c79a958b61059b5 Mon Sep 17 00:00:00 2001 From: jakeaturner Date: Thu, 24 Sep 2026 23:09:47 +0000 Subject: [PATCH] fix(gpu): detect Ollama CPU fallback on AMD and offer the right fix GpuPassthroughRemediationProvider returned early unless the NVIDIA runtime was registered with Docker. AMD doesn't register one, so the log-based CPU-fallback detection added in v1.35.0-rc.1 never ran on AMD boxes. On a Phoenix1 iGPU running ollama:rocm with /dev/kfd and /dev/dri passed through, Ollama logged library=cpu while gpuHealth reported status "ok". The gpuHealth side had its own bug. hasNvidiaRuntime and hasRocmRuntime were set only inside the lspci probe branch. When lspci returned a plausible controller, the else branch reported "ok" without probing and left both flags false. A blind reinstall would not have helped either: gfx1103 needs HSA_OVERRIDE_GFX_VERSION=11.0.0, and without the gfx marker or ai.amdHsaOverride a reinstall rebuilds the same CPU-bound container. - Add resolveExpectedGpuVendor(). It reads the nomad_ollama container config first (nvidia DeviceRequest, ROCm image tag, /dev/kfd), then the nvidia runtime, then the AMD GPU type when no container exists. /dev/dri alone is not treated as AMD. A CPU container on an AMD box is trusted as deliberate (#1232 fallback, acceleration disabled). - Gate the provider on the expected vendor. NVIDIA auto-reinstall is unchanged but now skips when the container requests a GPU with no runtime registered. AMD is detect-only: the provider logs whether a reinstall would change the container and never reinstalls itself. - In gpuHealth, set the runtime flags up front and let Ollama's startup log decide health whenever a GPU is expected, whatever lspci returned. A CPU-only log now reports passthrough_failed and skips the nvidia-smi exec. lspci stays the display source when clean. - Parse the first GPU "inference compute" line instead of the last line, which could be the CPU fallback device, and recognize Vulkan. Move the startup-log reader into ollama_compute so both callers share it. - Report amdReinstallWouldChange, amdGfxTarget, currentHsaOverride and suggestedHsaOverride on AMD passthrough_failed. The gfx target is parsed from Ollama's "amdgpu is not supported" line (best-effort). - Add GpuPassthroughAlert, shared by Settings > System and Settings > Models. When a reinstall would rebuild the same AMD container, it offers "Fix: Set GFX Override" instead of a reinstall. The dialog saves ai.amdHsaOverride and then reinstalls. - Expose ai.amdHsaOverride through the settings API. The value lands in the container env, so only x.y.z, "none" or empty is accepted. - Silence the HSA override resolver's logging on the gpuHealth poll. The #1325 case where no GPU runtime is expected and lspci reports a controller still returns "ok" unprobed. Fixes #1344 Refs #1325, #1232 --- admin/app/services/docker_service.ts | 21 +- admin/app/services/ollama_compute.ts | 202 +++++++++++++ admin/app/services/system_service.ts | 267 ++++++++---------- admin/app/validators/settings.ts | 13 + admin/constants/kv_store.ts | 1 + .../components/GpuPassthroughAlert.tsx | 217 ++++++++++++++ admin/inertia/pages/settings/models.tsx | 78 +---- admin/inertia/pages/settings/system.tsx | 85 +----- .../gpu_passthrough_remediation_provider.ts | 131 +++++---- admin/tests/unit/ollama_compute.spec.ts | 189 ++++++++++++- .../unit/settings_amd_hsa_override.spec.ts | 16 ++ admin/types/system.ts | 12 + 12 files changed, 871 insertions(+), 361 deletions(-) create mode 100644 admin/inertia/components/GpuPassthroughAlert.tsx create mode 100644 admin/tests/unit/settings_amd_hsa_override.spec.ts diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index e5ebdba8..e0ef20c7 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -1550,6 +1550,15 @@ export class DockerService { } } + /** + * The HSA_OVERRIDE_GFX_VERSION an AMD install or reinstall would apply right now. + * Exposed so GpuPassthroughRemediationProvider can tell whether a reinstall would + * change the running container. + */ + async getAmdHsaOverride(options: { quiet?: boolean } = {}): Promise { + return this._resolveAmdHsaOverride(options) + } + /** * Resolve the HSA_OVERRIDE_GFX_VERSION value for the host's AMD GPU. * @@ -1589,32 +1598,32 @@ export class DockerService { return pepper } - private async _resolveAmdHsaOverride(): Promise { + private async _resolveAmdHsaOverride({ quiet = false }: { quiet?: boolean } = {}): Promise { const manualRaw = await KVStore.getValue('ai.amdHsaOverride') if (manualRaw !== null && manualRaw !== undefined && String(manualRaw).trim() !== '') { const manual = String(manualRaw).trim().toLowerCase() if (manual === 'none' || manual === 'off' || manual === 'false') { - logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride') + if (!quiet) logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride') return null } if (/^\d+\.\d+\.\d+$/.test(manual)) { - logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`) + if (!quiet) logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`) return manual } - logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`) + if (!quiet) logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`) } try { const gfx = (await readFile('/app/storage/.nomad-amd-gfx', 'utf8')).trim() const mapped = this._mapGfxToHsaOverride(gfx) - logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`) + if (!quiet) logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`) return mapped } catch { // Marker absent — most likely an existing install upgraded without re-running // install_nomad.sh. Fall through to the default. } - logger.warn( + if (!quiet) logger.warn( '[DockerService] AMD GPU configured but no gfx marker (/app/storage/.nomad-amd-gfx) and no ' + 'ai.amdHsaOverride KV; relying on native ROCm discovery. iGPUs not on the bundled rocblas ' + 'allowlist (e.g. 780M/gfx1103, 680M/gfx1035) will silently fall back to CPU. Set the ' + diff --git a/admin/app/services/ollama_compute.ts b/admin/app/services/ollama_compute.ts index a947e246..81b61bfc 100644 --- a/admin/app/services/ollama_compute.ts +++ b/admin/app/services/ollama_compute.ts @@ -1,8 +1,13 @@ +import type Docker from 'dockerode' +import { mapGfxToHsaOverride } from '../utils/amd_hsa_override.js' + /** * Pure helpers for classifying which compute backend Ollama actually loaded, * parsed from its container logs. Extracted so both SystemService (GPU health * display) and GpuPassthroughRemediationProvider (auto-reinstall decision) share * one source of truth, and so the logic is unit-testable without Docker. + * readOllamaStartupLogs is the one Docker-facing helper, kept here so both + * callers read the same log window. * * Ollama writes one `inference compute` log line per detected device a few seconds * after startup, e.g.: @@ -39,3 +44,200 @@ export function classifyOllamaComputeBackend(logText: string): OllamaComputeBack return 'unknown' } + +export type OllamaGpuLibrary = 'CUDA' | 'ROCm' | 'Vulkan' + +export type OllamaGpuInfo = { + library: OllamaGpuLibrary + name: string + vramMiB: number +} + +/** + * Parse the GPU device Ollama reported at startup, or null when no GPU line is + * present. Takes the first GPU line rather than the last `inference compute` + * line: Ollama also lists a CPU fallback device, and when that line lands last + * a working GPU would otherwise read as no GPU at all. + */ +export function parseOllamaGpuFromLogs(logText: string): OllamaGpuInfo | null { + const line = logText + .split('\n') + .find((l) => l.includes(INFERENCE_LINE_MARKER) && GPU_LIBRARY_RE.test(l)) + if (!line) return null + + const library = line.match(GPU_LIBRARY_RE)![1] as OllamaGpuLibrary + const descMatch = line.match(/description="([^"]+)"/) + const totalMatch = line.match(/total="([0-9.]+)\s*GiB"/) + const fallbackName = library === 'CUDA' ? 'NVIDIA GPU' : library === 'ROCm' ? 'AMD GPU' : 'GPU' + + return { + library, + name: descMatch?.[1] || fallbackName, + vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0, + } +} + +export type GpuVendor = 'nvidia' | 'amd' + +/** + * The subset of `docker inspect nomad_ollama` the GPU checks read. Structural so + * dockerode's ContainerInspectInfo satisfies it and tests can pass plain objects. + */ +export type OllamaContainerSnapshot = { + Config?: { Image?: string; Env?: string[] | null } + HostConfig?: { + Devices?: Array<{ PathOnHost?: string }> | null + DeviceRequests?: Array<{ Driver?: string }> | null + } +} + +function hasDevice(container: OllamaContainerSnapshot, path: string): boolean { + return (container.HostConfig?.Devices ?? []).some((d) => d.PathOnHost === path) +} + +// Matches rocm in the tag only (ollama/ollama:rocm, :0.12.0-rocm), not the repo path. +function isRocmImage(image: string | undefined): boolean { + return /:[^/:]*rocm[^/:]*$/i.test(image ?? '') +} + +/** + * Which GPU vendor, if any, Ollama is expected to run on. + * + * The container's own config comes first because it records what the install + * actually set up, independent of how the host is detected today: + * - an nvidia DeviceRequest means NVIDIA + * - a ROCm image tag or a /dev/kfd device means AMD (/dev/dri alone is not + * AMD-specific, Intel and Vulkan setups pass it too) + * A registered nvidia runtime comes next, which covers a container created + * CPU-only before the runtime was added. + * + * The persisted AMD GPU type counts only when no container exists. An AMD + * container without ROCm config is deliberate: either acceleration was turned + * off, or the daemon rejected /dev/kfd and the install fell back to CPU (#1232). + * Flagging it would offer a reinstall that rebuilds the same CPU container. AMD + * never registers a Docker runtime, so gating on the runtime alone skips every + * AMD box (#1344). + */ +export function resolveExpectedGpuVendor(input: { + hasNvidiaRuntime: boolean + amdConfigured: boolean + container: OllamaContainerSnapshot | null +}): GpuVendor | null { + const { container } = input + if (container) { + if ((container.HostConfig?.DeviceRequests ?? []).some((r) => r.Driver === 'nvidia')) { + return 'nvidia' + } + if (isRocmImage(container.Config?.Image) || hasDevice(container, '/dev/kfd')) { + return 'amd' + } + } + if (input.hasNvidiaRuntime) return 'nvidia' + if (!container && input.amdConfigured) return 'amd' + return null +} + +/** + * Where a running AMD nomad_ollama container differs from what a reinstall would + * create. An empty list means a reinstall would rebuild the same container, so it + * cannot move Ollama off the CPU and the fix lies elsewhere (usually the HSA + * override for an iGPU missing from ROCm's allowlist). + */ +export function diffAmdOllamaConfig( + container: OllamaContainerSnapshot, + desiredHsaOverride: string | null +): string[] { + const drift: string[] = [] + const env = container.Config?.Env ?? [] + + if (!isRocmImage(container.Config?.Image)) { + drift.push(`image is ${container.Config?.Image ?? 'unknown'}, not the ROCm build`) + } + if (!hasDevice(container, '/dev/kfd')) drift.push('/dev/kfd is not passed through') + if (!hasDevice(container, '/dev/dri')) drift.push('/dev/dri is not passed through') + + const currentOverride = + env.find((e) => e.startsWith('HSA_OVERRIDE_GFX_VERSION='))?.split('=')[1] ?? null + if (currentOverride !== desiredHsaOverride) { + drift.push( + `HSA_OVERRIDE_GFX_VERSION is ${currentOverride ?? 'unset'}, expected ${desiredHsaOverride ?? 'unset'}` + ) + } + if (!env.includes('OLLAMA_IGPU_ENABLE=1')) drift.push('OLLAMA_IGPU_ENABLE=1 is not set') + + return drift +} + +/** + * The gfx target Ollama named when it rejected an AMD GPU, e.g. 'gfx1103' from + * msg="amdgpu is not supported (supported types:[gfx1030 ...])" gpu_type=gfx1103 ... + * Best-effort: the exact wording varies across Ollama releases, so callers treat + * null as "unknown" and fall back to asking the user. + */ +export function parseAmdGfxTargetFromLogs(logText: string): string | null { + return logText.match(/gpu_type=(gfx[0-9a-f]+)\b/i)?.[1]?.toLowerCase() ?? null +} + +/** + * Why an AMD container expected on the GPU is running on CPU, in the shape + * GpuHealthStatus reports to the UI. + */ +export function diagnoseAmdCpuFallback( + container: OllamaContainerSnapshot, + logText: string, + desiredHsaOverride: string | null +) { + const gfx = parseAmdGfxTargetFromLogs(logText) + const current = (container.Config?.Env ?? []) + .find((e) => e.startsWith('HSA_OVERRIDE_GFX_VERSION=')) + ?.split('=')[1] + const suggested = gfx ? mapGfxToHsaOverride(gfx) : null + return { + amdReinstallWouldChange: diffAmdOllamaConfig(container, desiredHsaOverride).length > 0, + ...(gfx && { amdGfxTarget: gfx }), + ...(current && { currentHsaOverride: current }), + ...(suggested && { suggestedHsaOverride: suggested }), + } +} + +/** + * Read nomad_ollama's logs from its startup window (the five minutes after it + * booted), where the "inference compute" line is emitted. A tail:N read is + * fragile here: under embedding load the container writes >1000 lines/min, + * which pushes the line out of any reasonable tail within minutes, while the + * startup window is bounded and never ages out. Falls back to tail:500 when + * StartedAt is missing. Returns '' on any error so callers classify the backend + * as 'unknown' rather than throwing. + */ +export async function readOllamaStartupLogs( + container: Docker.Container, + inspect?: { State?: { StartedAt?: string } } +): Promise { + try { + const state = inspect ?? (await container.inspect()) + const startedAtRaw = state?.State?.StartedAt + const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN + + const logsOpts: { + stdout: true + stderr: true + follow: false + since?: number + until?: number + tail?: number + } = { stdout: true, stderr: true, follow: false } + + if (Number.isFinite(startedAtMs) && startedAtMs > 0) { + const startedAtSec = Math.floor(startedAtMs / 1000) + logsOpts.since = startedAtSec + logsOpts.until = startedAtSec + 300 + } else { + logsOpts.tail = 500 + } + + const buf = (await container.logs(logsOpts)) as unknown as Buffer + return buf.toString('utf8') + } catch { + return '' + } +} diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 694fe8bb..73d627e2 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -22,6 +22,14 @@ import KVStore from '#models/kv_store' import { KV_STORE_SCHEMA, KVStoreKey } from '../../types/kv_store.js' import { isNewerVersion } from '../utils/version.js' import { isUnresolvedGpuModel } from '../utils/gpu_model.js' +import { + classifyOllamaComputeBackend, + diagnoseAmdCpuFallback, + parseOllamaGpuFromLogs, + readOllamaStartupLogs, + resolveExpectedGpuVendor, + type OllamaGpuInfo, +} from '#services/ollama_compute' import { invalidateAssistantNameCache } from '../../config/inertia.js' import { invalidateMinRelevanceCache } from '../utils/rag_relevance.js' import { invalidateResponseStyleCache } from '../utils/response_style.js' @@ -103,6 +111,20 @@ export class SystemService { return false } + /** + * The running nomad_ollama container and its inspect data, or null when it is + * not running. The inspect data records the GPU config the install actually + * applied (DeviceRequests, /dev/kfd, the ROCm image), which is what the GPU + * health check keys on. + */ + private async _findRunningOllamaContainer() { + const containers = await this.dockerService.docker.listContainers({ all: false }) + const summary = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) + if (!summary) return null + const container = this.dockerService.docker.getContainer(summary.Id) + return { container, inspect: await container.inspect() } + } + /** * Probe Ollama startup logs for the canonical "inference compute" line that records * which compute backend was selected. This catches silent CPU fallback (e.g. when @@ -114,70 +136,11 @@ export class SystemService { * - the line has not been emitted (Ollama still starting up) * - logs show CPU-only operation (no GPU detected) */ - async getOllamaInferenceComputeFromLogs(): Promise<{ - library: 'CUDA' | 'ROCm' - name: string - vramMiB: number - } | null> { + async getOllamaInferenceComputeFromLogs(): Promise { try { - const containers = await this.dockerService.docker.listContainers({ all: false }) - const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) - if (!ollamaContainer) return null - - const container = this.dockerService.docker.getContainer(ollamaContainer.Id) - - // Read logs only from the first 5 minutes after container start. The - // "inference compute" line is written once during Ollama's GPU discovery - // phase, within seconds of startup. Using tail:N here is fragile: under - // active embedding workloads we've seen >1000 lines/min, which pushes the - // line past any reasonable tail in minutes. Pinning to the startup window - // is bounded (~5 min of logs regardless of container uptime) and never - // ages out. - // - // Fall back to the previous tail:500 strategy if StartedAt is missing or - // unparseable — we can't construct a since/until window without it, but - // tail:500 is still useful when the container just started and the line - // is still recent. - const inspect = await container.inspect() - const startedAtRaw = inspect?.State?.StartedAt - const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN - const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0 - - const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = { - stdout: true, - stderr: true, - follow: false, - } - if (hasValidStartedAt) { - const startedAtSec = Math.floor(startedAtMs / 1000) - logsOpts.since = startedAtSec - logsOpts.until = startedAtSec + 300 // 5-minute window - } else { - logger.warn( - `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe` - ) - logsOpts.tail = 500 - } - const buf = (await container.logs(logsOpts)) as unknown as Buffer - const logs = buf.toString('utf8') - - const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"')) - if (lines.length === 0) return null - - const lastLine = lines[lines.length - 1] - const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/) - if (!libraryMatch) return null - - const descMatch = lastLine.match(/description="([^"]+)"/) - const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/) - - return { - library: libraryMatch[1] as 'CUDA' | 'ROCm', - name: - descMatch?.[1] || - (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'), - vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0, - } + const ollama = await this._findRunningOllamaContainer() + if (!ollama) return null + return parseOllamaGpuFromLogs(await readOllamaStartupLogs(ollama.container, ollama.inspect)) } catch (error) { logger.warn( `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}` @@ -526,91 +489,106 @@ export class SystemService { // Run the probes when controllers are empty (common inside Docker), // when lspci gave us bogus discrete-GPU BAR0 values that need replacing, - // or when it named a card it couldn't resolve. - if ( + // or when it named a card it couldn't resolve. Only in those cases do + // probe results replace the lspci controllers; a clean lspci reading + // stays as the display source. + const lspciNeedsProbe = !graphics.controllers || graphics.controllers.length === 0 || hasLspciBogusDgpuVram || hasUnresolvedGpuName - ) { - const runtimes = dockerInfo.Runtimes || {} - gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes + const useProbeControllers = (controllers: typeof graphics.controllers) => { + if (lspciNeedsProbe) graphics.controllers = controllers + } + const toControllers = (gpus: Array<{ vendor: string; model: string; vram: number }>) => + gpus.map((gpu) => ({ + model: gpu.model, + vendor: gpu.vendor, + bus: '', + vram: gpu.vram, + vramDynamic: false, + })) - // AMD doesn't register a Docker runtime. Detection sources, in priority order: - // 1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install) - // 2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh) - // The marker file matters because the System page should reflect AMD presence - // even before AI Assistant has been installed for the first time. - let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined - if (!savedGpuType) { - try { - savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() - } catch {} - } - const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') - const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' - gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled + // Runtime flags are reported whichever path runs below. They used to be set + // only inside the probe branch, so a box whose lspci reading looked clean + // reported hasRocmRuntime: false even with an AMD GPU configured (#1344). + const runtimes = dockerInfo.Runtimes || {} + gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes - if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) { - gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd' + // AMD doesn't register a Docker runtime. Detection sources, in priority order: + // 1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install) + // 2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh) + // The marker file matters because the System page should reflect AMD presence + // even before AI Assistant has been installed for the first time. + let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined + if (!savedGpuType) { + try { + savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim() + } catch {} + } + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false' + gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled + + // When Ollama is expected on a GPU, its startup log decides health whatever + // lspci returned. A clean lspci reading says a GPU exists, not that Ollama + // uses it: an AMD iGPU silently on CPU used to land in the else branch below + // and report 'ok' (#1344). + const ollama = await this._findRunningOllamaContainer().catch(() => null) + const expectedVendor = resolveExpectedGpuVendor({ + hasNvidiaRuntime: gpuHealth.hasNvidiaRuntime, + amdConfigured: gpuHealth.hasRocmRuntime, + container: ollama?.inspect ?? null, + }) + + if (expectedVendor || lspciNeedsProbe) { + if (expectedVendor) { + gpuHealth.gpuVendor = expectedVendor // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback - const logInfo = await this.getOllamaInferenceComputeFromLogs() + const ollamaLogs = ollama + ? await readOllamaStartupLogs(ollama.container, ollama.inspect) + : '' + const logInfo = parseOllamaGpuFromLogs(ollamaLogs) if (logInfo) { - graphics.controllers = [ + useProbeControllers([ { model: logInfo.name, - vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD', + vendor: + logInfo.library === 'CUDA' ? 'NVIDIA' : logInfo.library === 'ROCm' ? 'AMD' : '', bus: '', vram: logInfo.vramMiB, vramDynamic: false, }, - ] + ]) gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true - } else if (gpuHealth.hasNvidiaRuntime) { + } else if (classifyOllamaComputeBackend(ollamaLogs) === 'cpu') { + // Ollama reported its devices and none was a GPU. Definitive for both + // vendors, so skip the nvidia-smi exec, which cannot run in a CPU-only + // container anyway. + gpuHealth.status = 'passthrough_failed' + logger.warn( + `${expectedVendor === 'amd' ? 'AMD' : 'NVIDIA'} GPU expected but Ollama reported a CPU-only backend at startup` + ) + } else if (expectedVendor === 'nvidia') { // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when // the log parser hasn't seen a startup line yet (e.g. log rotation, // very fresh container). Distinguishes "no Ollama container" from // "container exists but GPU broken". const nvidiaInfo = await this.getNvidiaSmiInfo() if (Array.isArray(nvidiaInfo)) { - graphics.controllers = nvidiaInfo.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) + useProbeControllers(toControllers(nvidiaInfo)) gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true - } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) - gpuHealth.status = 'ok' - gpuHealth.ollamaGpuAccessible = true - } else { - gpuHealth.status = 'ollama_not_installed' - } } else { const externalOllamaGpu = await this.getExternalOllamaGpuInfo() if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) + useProbeControllers(toControllers(externalOllamaGpu)) gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true + } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') { + gpuHealth.status = 'ollama_not_installed' } else { gpuHealth.status = 'passthrough_failed' logger.warn( @@ -618,38 +596,41 @@ export class SystemService { ) } } - } else { + } else if (!ollama) { // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative. // Distinguish "Ollama not running" from "Ollama running but no GPU log line". - const containers = await this.dockerService.docker.listContainers({ all: false }) - const ollamaRunning = containers.some((c) => - c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`) - ) - if (!ollamaRunning) { - const externalOllamaGpu = await this.getExternalOllamaGpuInfo() - if (externalOllamaGpu) { - graphics.controllers = externalOllamaGpu.map((gpu) => ({ - model: gpu.model, - vendor: gpu.vendor, - bus: '', - vram: gpu.vram, - vramDynamic: false, - })) - gpuHealth.status = 'ok' - gpuHealth.ollamaGpuAccessible = true - } else { - gpuHealth.status = 'ollama_not_installed' - } + const externalOllamaGpu = await this.getExternalOllamaGpuInfo() + if (externalOllamaGpu) { + useProbeControllers(toControllers(externalOllamaGpu)) + gpuHealth.status = 'ok' + gpuHealth.ollamaGpuAccessible = true } else { - gpuHealth.status = 'passthrough_failed' - logger.warn( - 'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed' - ) + gpuHealth.status = 'ollama_not_installed' } + } else { + gpuHealth.status = 'passthrough_failed' + logger.warn( + 'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed' + ) + } + + // Tell the UI whether a reinstall would help or an HSA override is needed. + // Reinstalling an AMD container whose config already matches rebuilds the + // same CPU-bound container, so the banner must not offer that as the fix. + if (gpuHealth.status === 'passthrough_failed' && expectedVendor === 'amd' && ollama) { + Object.assign( + gpuHealth, + diagnoseAmdCpuFallback( + ollama.inspect, + ollamaLogs, + await this.dockerService.getAmdHsaOverride({ quiet: true }) + ) + ) } } } else { - // si.graphics() returned controllers (host install, not Docker) — GPU is working + // si.graphics() returned usable controllers and no GPU runtime is expected + // (host install, not Docker). Still unprobed, see #1325. gpuHealth.status = 'ok' gpuHealth.ollamaGpuAccessible = true } diff --git a/admin/app/validators/settings.ts b/admin/app/validators/settings.ts index 96d12600..2de84dbd 100644 --- a/admin/app/validators/settings.ts +++ b/admin/app/validators/settings.ts @@ -118,6 +118,19 @@ export function validateSettingValue(key: KVStoreKey, value: unknown): string | } return null } + case 'ai.amdHsaOverride': { + // Lands in the Ollama container env as HSA_OVERRIDE_GFX_VERSION, so only + // the forms DockerService._resolveAmdHsaOverride honours are accepted. + // Empty clears it and 'none' disables the gfx-marker default. + const raw = typeof value === 'string' ? value.trim().toLowerCase() : value + if (raw === '' || raw === undefined || raw === null || raw === 'none') { + return null + } + if (typeof raw !== 'string' || !/^\d{1,2}\.\d{1,2}\.\d{1,2}$/.test(raw)) { + return 'GFX override must be a version like "11.0.0", "none", or empty to use the detected default.' + } + return null + } default: return null } diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 6953de1d..bb5eb06d 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -15,6 +15,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [ 'ai.contextWindow', 'ai.keepAlive', 'ai.responseStyle', + 'ai.amdHsaOverride', 'rag.defaultIngestPolicy', 'rag.enabled', 'rag.minRelevance', diff --git a/admin/inertia/components/GpuPassthroughAlert.tsx b/admin/inertia/components/GpuPassthroughAlert.tsx new file mode 100644 index 00000000..f0c72387 --- /dev/null +++ b/admin/inertia/components/GpuPassthroughAlert.tsx @@ -0,0 +1,217 @@ +import { useState } from 'react' +import Alert from '~/components/Alert' +import StyledModal from '~/components/StyledModal' +import Input from '~/components/inputs/Input' +import { useNotifications } from '~/context/NotificationContext' +import { useModals } from '~/context/ModalContext' +import api from '~/lib/api' +import type { GpuHealthStatus } from '../../types/system' + +const DISMISS_KEY = 'nomad:gpu-banner-dismissed' +const HSA_OVERRIDE_PATTERN = /^\d{1,2}\.\d{1,2}\.\d{1,2}$/ + +/** + * Banner for gpuHealth.status === 'passthrough_failed'. + * + * NVIDIA, and AMD whose container differs from what a reinstall would create, get + * the one-click reinstall. AMD whose container already matches gets an HSA override + * prompt instead: reinstalling it rebuilds the same CPU-bound container (#1344). + */ +export default function GpuPassthroughAlert({ + gpuHealth, + assistantName, + className, +}: { + gpuHealth: GpuHealthStatus | undefined + assistantName: string + className?: string +}) { + const { addNotification } = useNotifications() + const { openModal, closeAllModals } = useModals() + const [reinstalling, setReinstalling] = useState(false) + const [dismissed, setDismissed] = useState(() => { + try { + return localStorage.getItem(DISMISS_KEY) === 'true' + } catch { + return false + } + }) + + if (gpuHealth?.status !== 'passthrough_failed' || dismissed) return null + + const isAmd = gpuHealth.gpuVendor === 'amd' + const needsHsaOverride = isAmd && gpuHealth.amdReinstallWouldChange === false + + const handleDismiss = () => { + setDismissed(true) + try { + localStorage.setItem(DISMISS_KEY, 'true') + } catch {} + } + + const reinstall = async () => { + setReinstalling(true) + try { + const response = await api.forceReinstallService('nomad_ollama') + if (!response || !response.success) { + throw new Error(response?.message || 'Force reinstall failed') + } + addNotification({ + message: `${assistantName} is being reinstalled with GPU support. This page will reload shortly.`, + type: 'success', + }) + try { + localStorage.removeItem(DISMISS_KEY) + } catch {} + setTimeout(() => window.location.reload(), 5000) + } catch (error) { + addNotification({ + message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`, + type: 'error', + }) + setReinstalling(false) + } + } + + const openReinstallModal = () => { + openModal( + { + closeAllModals() + reinstall() + }} + onCancel={closeAllModals} + open={true} + confirmText="Reinstall" + cancelText="Cancel" + > +

+ This will recreate the {assistantName} container with GPU support enabled. Your + downloaded models will be preserved. The service will be briefly unavailable during + reinstall. +

+ , + 'gpu-health-force-reinstall-modal' + ) + } + + const openHsaOverrideModal = () => { + openModal( + { + closeAllModals() + reinstall() + }} + />, + 'gpu-health-hsa-override-modal' + ) + } + + const vendorName = isAmd ? 'an AMD' : 'an NVIDIA' + const message = needsHsaOverride + ? `Your system has an AMD GPU, but ${assistantName} is running on CPU only. Reinstalling won't fix this: the GPU${gpuHealth.amdGfxTarget ? ` (${gpuHealth.amdGfxTarget})` : ''} isn't on ROCm's supported list and needs a GFX version override.` + : `Your system has ${vendorName} GPU, but ${assistantName} can't access it. AI is running on CPU only, which is significantly slower.` + + return ( + + ) +} + +function HsaOverrideModal({ + gpuHealth, + assistantName, + onCancel, + onSaved, +}: { + gpuHealth: GpuHealthStatus + assistantName: string + onCancel: () => void + onSaved: () => void +}) { + const { addNotification } = useNotifications() + const [value, setValue] = useState(gpuHealth.suggestedHsaOverride ?? '') + const [saving, setSaving] = useState(false) + const valid = HSA_OVERRIDE_PATTERN.test(value.trim()) + + const save = async () => { + setSaving(true) + try { + const response = await api.updateSetting('ai.amdHsaOverride', value.trim()) + // catchInternal already notified on a request failure and returned undefined. + if (!response) { + setSaving(false) + return + } + if (!response.success) throw new Error(response.message) + onSaved() + } catch (error) { + addNotification({ + message: `Failed to save the GFX override: ${error instanceof Error ? error.message : 'Unknown error'}`, + type: 'error', + }) + setSaving(false) + } + } + + return ( + +
+

+ ROCm ships GPU kernels for a fixed list of AMD chips. Integrated Radeon GPUs outside + that list run on CPU unless ROCm treats them as a supported chip, set through + HSA_OVERRIDE_GFX_VERSION. +

+
    +
  • Radeon 780M / 760M (gfx1103): 11.0.0
    • +
  • Radeon 680M / 660M and other RDNA 2 iGPUs (gfx1031 to gfx1036): 10.3.0
    • +
+

+ Detected GPU: {gpuHealth.amdGfxTarget ?? 'not reported by Ollama'}. Current override:{' '} + {gpuHealth.currentHsaOverride ?? 'none'}. + {gpuHealth.suggestedHsaOverride + ? ` Recommended for this GPU: ${gpuHealth.suggestedHsaOverride}.` + : ''} +

+ setValue(e.target.value)} + error={value.trim() !== '' && !valid} + helpText={`Saving recreates the ${assistantName} container with this override. Downloaded models are preserved.`} + /> +
+ + ) +} diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index df703c3f..e9681df4 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -23,6 +23,7 @@ import { formatBytes } from '~/lib/util' import useDebounce from '~/hooks/useDebounce' import ActiveModelDownloads from '~/components/ActiveModelDownloads' import { useSystemInfo } from '~/hooks/useSystemInfo' +import GpuPassthroughAlert from '~/components/GpuPassthroughAlert' export default function ModelsPage(props: { models: { @@ -41,62 +42,6 @@ export default function ModelsPage(props: { const { data: systemInfo } = useSystemInfo({}) const queryClient = useQueryClient() - const [gpuBannerDismissed, setGpuBannerDismissed] = useState(() => { - try { - return localStorage.getItem('nomad:gpu-banner-dismissed') === 'true' - } catch { - return false - } - }) - const [reinstalling, setReinstalling] = useState(false) - - const handleDismissGpuBanner = () => { - setGpuBannerDismissed(true) - try { - localStorage.setItem('nomad:gpu-banner-dismissed', 'true') - } catch {} - } - - const handleForceReinstallOllama = () => { - openModal( - { - closeAllModals() - setReinstalling(true) - try { - const response = await api.forceReinstallService('nomad_ollama') - if (!response || !response.success) { - throw new Error(response?.message || 'Force reinstall failed') - } - addNotification({ - message: `${aiAssistantName} is being reinstalled with GPU support. This page will reload shortly.`, - type: 'success', - }) - try { localStorage.removeItem('nomad:gpu-banner-dismissed') } catch {} - setTimeout(() => window.location.reload(), 5000) - } catch (error) { - addNotification({ - message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`, - type: 'error', - }) - setReinstalling(false) - } - }} - onCancel={closeAllModals} - open={true} - confirmText="Reinstall" - cancelText="Cancel" - > -

- This will recreate the {aiAssistantName} container with GPU support enabled. - Your downloaded models will be preserved. The service will be briefly - unavailable during reinstall. -

- , - 'gpu-health-force-reinstall-modal' - ) - } const [chatSuggestionsEnabled, setChatSuggestionsEnabled] = useState( props.models.settings.chatSuggestionsEnabled ) @@ -345,24 +290,11 @@ export default function ModelsPage(props: { className="!mt-6" /> )} - {isInstalled && systemInfo?.gpuHealth?.status === 'passthrough_failed' && !gpuBannerDismissed && ( - )} diff --git a/admin/inertia/pages/settings/system.tsx b/admin/inertia/pages/settings/system.tsx index 1a13b52f..895c9f88 100644 --- a/admin/inertia/pages/settings/system.tsx +++ b/admin/inertia/pages/settings/system.tsx @@ -1,4 +1,3 @@ -import { useState } from 'react' import { Head } from '@inertiajs/react' import SettingsLayout from '~/layouts/SettingsLayout' import { SystemInformationResponse } from '../../../types/system' @@ -8,11 +7,8 @@ import CircularGauge from '~/components/systeminfo/CircularGauge' import HorizontalBarChart from '~/components/HorizontalBarChart' import InfoCard from '~/components/systeminfo/InfoCard' import Alert from '~/components/Alert' -import StyledModal from '~/components/StyledModal' +import GpuPassthroughAlert from '~/components/GpuPassthroughAlert' import { useSystemInfo } from '~/hooks/useSystemInfo' -import { useNotifications } from '~/context/NotificationContext' -import { useModals } from '~/context/ModalContext' -import api from '~/lib/api' import StatusCard from '~/components/systeminfo/StatusCard' import { IconCpu, IconDatabase, IconServer, IconDeviceDesktop, IconComponents } from '@tabler/icons-react' @@ -22,65 +18,6 @@ export default function SettingsPage(props: { const { data: info } = useSystemInfo({ initialData: props.system.info, }) - const { addNotification } = useNotifications() - const { openModal, closeAllModals } = useModals() - - const [gpuBannerDismissed, setGpuBannerDismissed] = useState(() => { - try { - return localStorage.getItem('nomad:gpu-banner-dismissed') === 'true' - } catch { - return false - } - }) - const [reinstalling, setReinstalling] = useState(false) - - const handleDismissGpuBanner = () => { - setGpuBannerDismissed(true) - try { - localStorage.setItem('nomad:gpu-banner-dismissed', 'true') - } catch {} - } - - const handleForceReinstallOllama = () => { - openModal( - { - closeAllModals() - setReinstalling(true) - try { - const response = await api.forceReinstallService('nomad_ollama') - if (!response || !response.success) { - throw new Error(response?.message || 'Force reinstall failed') - } - addNotification({ - message: 'AI Assistant is being reinstalled with GPU support. This page will reload shortly.', - type: 'success', - }) - try { localStorage.removeItem('nomad:gpu-banner-dismissed') } catch {} - setTimeout(() => window.location.reload(), 5000) - } catch (error) { - addNotification({ - message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`, - type: 'error', - }) - setReinstalling(false) - } - }} - onCancel={closeAllModals} - open={true} - confirmText="Reinstall" - cancelText="Cancel" - > -

- This will recreate the AI Assistant container with GPU support enabled. - Your downloaded models will be preserved. The service will be briefly - unavailable during reinstall. -

- , - 'gpu-health-force-reinstall-modal' - ) - } // Use (total - available) to reflect actual memory pressure. // mem.used includes reclaimable buff/cache on Linux, which inflates the number. @@ -203,25 +140,9 @@ export default function SettingsPage(props: { }, ]} /> - {info?.gpuHealth?.status === 'passthrough_failed' && !gpuBannerDismissed && ( + {info?.gpuHealth?.status === 'passthrough_failed' && (
- +
)} {info?.graphics?.controllers && info.graphics.controllers.length > 0 && ( diff --git a/admin/providers/gpu_passthrough_remediation_provider.ts b/admin/providers/gpu_passthrough_remediation_provider.ts index d60cf92e..2263f129 100644 --- a/admin/providers/gpu_passthrough_remediation_provider.ts +++ b/admin/providers/gpu_passthrough_remediation_provider.ts @@ -1,14 +1,14 @@ import logger from '@adonisjs/core/services/logger' import type { ApplicationService } from '@adonisjs/core/types' -import type Docker from 'dockerode' /** - * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart. + * Detects Ollama's silent CPU fallback after admin / host restart, and auto-remediates + * it on NVIDIA. * * Detects the condition from Ollama's own "inference compute" startup log line: - * if the NVIDIA container runtime is registered with Docker but Ollama still - * loaded a CPU-only backend, passthrough is broken and nomad_ollama is recreated. - * This single signal covers two failure modes: + * if Ollama is expected on a GPU (see resolveExpectedGpuVendor) but loaded a CPU-only + * backend, passthrough is broken. On NVIDIA, nomad_ollama is recreated. + * This single signal covers two NVIDIA failure modes: * 1. nomad_ollama was created CPU-only because the nvidia runtime was registered * only AFTER the AI Assistant was first installed (DockerService attaches a * GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at install @@ -20,16 +20,19 @@ import type Docker from 'dockerode' * PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" banner; this * provider performs that click automatically on admin boot. * + * AMD is detect-only. The usual cause there is a missing HSA override for an iGPU + * outside ROCm's allowlist (e.g. gfx1103), which a reinstall rebuilds unchanged. The + * provider logs whether a reinstall would change the container and leaves the + * reinstall to the user; SystemService surfaces the same state as passthrough_failed. + * * Guards: - * - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override - * handling in PR #804) and is left to the user. * - One-shot per admin boot. The provider runs once on startup; if the recreate * itself fails the banner remains as a fallback. * - Cooldown: will not auto-reinstall more than once within * AUTO_REMEDIATE_COOLDOWN_MS, to avoid a reinstall loop when the GPU cannot be * accelerated (e.g. an architecture Ollama has no kernels for). * - Opt-out via KV `ai.autoFixGpuPassthrough = false`. - * - Skipped entirely when no NVIDIA runtime is registered with Docker. + * - Skipped entirely when nomad_ollama is not expected on a GPU. */ export default class GpuPassthroughRemediationProvider { constructor(protected app: ApplicationService) {} @@ -55,14 +58,6 @@ export default class GpuPassthroughRemediationProvider { const docker = new Docker({ socketPath: '/var/run/docker.sock' }) const dockerInfo = await docker.info() const runtimes = dockerInfo.Runtimes || {} - const hasNvidiaRuntime = 'nvidia' in runtimes - - if (!hasNvidiaRuntime) { - logger.info( - '[GpuPassthroughRemediationProvider] No NVIDIA runtime registered — skipping.' - ) - return - } const containers = await docker.listContainers({ all: false }) const ollama = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) @@ -74,20 +69,43 @@ export default class GpuPassthroughRemediationProvider { return } + const container = docker.getContainer(ollama.Id) + const inspect = await container.inspect() + const { + classifyOllamaComputeBackend, + diffAmdOllamaConfig, + readOllamaStartupLogs, + resolveExpectedGpuVendor, + } = await import('#services/ollama_compute') + + // The container's own GPU config decides the vendor. The KV AMD type only + // matters when no container exists, which the return above rules out. + const vendor = resolveExpectedGpuVendor({ + hasNvidiaRuntime: 'nvidia' in runtimes, + amdConfigured: false, + container: inspect, + }) + + if (!vendor) { + logger.info( + '[GpuPassthroughRemediationProvider] nomad_ollama is not configured for a GPU — skipping.' + ) + return + } + // Probe: read Ollama's own "inference compute" startup line from its logs. // This is the ground truth for whether Ollama loaded a GPU backend, and it // catches every failure mode: // - nomad_ollama created CPU-only because the nvidia runtime was registered // only AFTER first install (Ollama logs library=cpu), and // - DeviceRequests present but the toolkit binding tore after a recreate, - // where Ollama silently falls back to CPU (also library=cpu). + // where Ollama silently falls back to CPU (also library=cpu), and + // - a ROCm container on an AMD GPU that ROCm could not initialize. // The previous implementation probed by exec'ing `nvidia-smi` inside the // container, but a CPU-only container does not ship nvidia-smi, so the exec // returned an error string that the alphabetic-output check mistook for // "healthy" — a false negative that suppressed all auto-remediation. - const container = docker.getContainer(ollama.Id) - const { classifyOllamaComputeBackend } = await import('#services/ollama_compute') - const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container)) + const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container, inspect)) if (backend === 'gpu') { logger.info( @@ -103,6 +121,43 @@ export default class GpuPassthroughRemediationProvider { return } + if (vendor === 'amd') { + const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration') + if (String(amdEnabledRaw) === 'false') { + logger.info( + '[GpuPassthroughRemediationProvider] Ollama is on CPU and AMD acceleration is disabled via ai.amdGpuAcceleration — skipping.' + ) + return + } + + const hsaOverride = await new DockerService().getAmdHsaOverride() + const drift = diffAmdOllamaConfig(inspect, hsaOverride) + if (drift.length > 0) { + logger.warn( + '[GpuPassthroughRemediationProvider] AMD GPU configured but Ollama fell back to CPU. ' + + `A reinstall would change the container (${drift.join('; ')}). ` + + 'Use "Fix: Reinstall AI Assistant" to apply it.' + ) + } else { + logger.warn( + '[GpuPassthroughRemediationProvider] AMD GPU configured but Ollama fell back to CPU, and a ' + + 'reinstall would rebuild the same container. If this is an iGPU outside the ROCm ' + + 'allowlist, use "Fix: Set GFX Override" in Settings (e.g. 11.0.0 for a 780M/gfx1103).' + ) + } + return + } + + // NVIDIA DeviceRequests without a registered runtime: a reinstall would come up + // CPU-only, since DockerService attaches the GPU only when the runtime is present. + if (!('nvidia' in runtimes)) { + logger.warn( + '[GpuPassthroughRemediationProvider] nomad_ollama requests an NVIDIA GPU but no NVIDIA runtime ' + + 'is registered with Docker. Reinstall the NVIDIA Container Toolkit; skipping auto-reinstall.' + ) + return + } + // backend === 'cpu': an NVIDIA runtime is registered but Ollama still runs on // CPU. Cooldown: don't auto-reinstall more than once within the window. A // reinstall that fails to move Ollama onto the GPU (e.g. an architecture @@ -151,39 +206,3 @@ export default class GpuPassthroughRemediationProvider { * provider. Bounds a reinstall loop when the GPU cannot be accelerated. */ const AUTO_REMEDIATE_COOLDOWN_MS = 60 * 60 * 1000 // 1 hour - -/** - * Read nomad_ollama's logs from its startup window (the five minutes after it - * booted), where the "inference compute" line is emitted. Mirrors the windowing - * in SystemService.getOllamaInferenceComputeFromLogs. Returns '' on any error so - * the caller classifies the backend as 'unknown' rather than throwing. - */ -async function readOllamaStartupLogs(container: Docker.Container): Promise { - try { - const inspect = await container.inspect() - const startedAtRaw = inspect?.State?.StartedAt - const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN - - const logsOpts: { - stdout: true - stderr: true - follow: false - since?: number - until?: number - tail?: number - } = { stdout: true, stderr: true, follow: false } - - if (Number.isFinite(startedAtMs) && startedAtMs > 0) { - const startedAtSec = Math.floor(startedAtMs / 1000) - logsOpts.since = startedAtSec - logsOpts.until = startedAtSec + 300 - } else { - logsOpts.tail = 500 - } - - const buf = (await container.logs(logsOpts)) as unknown as Buffer - return buf.toString('utf8') - } catch { - return '' - } -} diff --git a/admin/tests/unit/ollama_compute.spec.ts b/admin/tests/unit/ollama_compute.spec.ts index c3d739c1..d47e9758 100644 --- a/admin/tests/unit/ollama_compute.spec.ts +++ b/admin/tests/unit/ollama_compute.spec.ts @@ -1,7 +1,14 @@ import * as assert from 'node:assert/strict' import { test } from 'node:test' -import { classifyOllamaComputeBackend } from '../../app/services/ollama_compute.js' +import { + classifyOllamaComputeBackend, + diagnoseAmdCpuFallback, + diffAmdOllamaConfig, + parseAmdGfxTargetFromLogs, + parseOllamaGpuFromLogs, + resolveExpectedGpuVendor, +} from '../../app/services/ollama_compute.js' test('CUDA inference line is classified as gpu', () => { const log = @@ -43,3 +50,183 @@ test('ROCm and Vulkan are gpu backends', () => { 'gpu' ) }) + +test('parseOllamaGpuFromLogs takes the GPU line even when the CPU line comes last', () => { + const log = + 'msg="inference compute" id=0 library=ROCm compute=gfx1150 name=ROCm0 description="AMD Radeon 890M" total="16.0 GiB"\n' + + 'msg="inference compute" id=cpu library=cpu name=cpu' + assert.deepEqual(parseOllamaGpuFromLogs(log), { + library: 'ROCm', + name: 'AMD Radeon 890M', + vramMiB: 16384, + }) +}) + +test('parseOllamaGpuFromLogs recognizes Vulkan and falls back to a generic name', () => { + assert.deepEqual(parseOllamaGpuFromLogs('msg="inference compute" id=0 library=Vulkan name=V0'), { + library: 'Vulkan', + name: 'GPU', + vramMiB: 0, + }) +}) + +test('parseOllamaGpuFromLogs returns null for a CPU-only log', () => { + assert.equal(parseOllamaGpuFromLogs('msg="inference compute" id=cpu library=cpu name=cpu'), null) +}) + +const ROCM_CONTAINER = { + Config: { + Image: 'ollama/ollama:rocm', + Env: ['OLLAMA_NO_CLOUD=1', 'HSA_OVERRIDE_GFX_VERSION=11.0.0', 'OLLAMA_IGPU_ENABLE=1'], + }, + HostConfig: { + Devices: [{ PathOnHost: '/dev/kfd' }, { PathOnHost: '/dev/dri' }], + }, +} + +const CPU_CONTAINER = { + Config: { Image: 'ollama/ollama:0.18.2', Env: ['OLLAMA_NO_CLOUD=1'] }, + HostConfig: {}, +} + +test('a ROCm container is expected on AMD with no Docker runtime registered (#1344)', () => { + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container: ROCM_CONTAINER }), + 'amd' + ) +}) + +test('a /dev/kfd device alone marks the container as AMD', () => { + const container = { + Config: { Image: 'ollama/ollama:0.18.2' }, + HostConfig: { Devices: [{ PathOnHost: '/dev/kfd' }] }, + } + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }), + 'amd' + ) +}) + +test('a versioned ROCm tag marks the container as AMD, a rocm repo path does not', () => { + const tagged = { Config: { Image: 'ollama/ollama:0.12.0-rocm' }, HostConfig: {} } + const repoOnly = { Config: { Image: 'rocm/ollama:latest' }, HostConfig: {} } + const input = { hasNvidiaRuntime: false, amdConfigured: false } + assert.equal(resolveExpectedGpuVendor({ ...input, container: tagged }), 'amd') + assert.equal(resolveExpectedGpuVendor({ ...input, container: repoOnly }), null) +}) + +test('/dev/dri alone is not treated as AMD', () => { + const container = { + Config: { Image: 'ollama/ollama:0.18.2' }, + HostConfig: { Devices: [{ PathOnHost: '/dev/dri' }] }, + } + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }), + null + ) +}) + +test('an nvidia DeviceRequest is expected on NVIDIA even without the runtime', () => { + const container = { + Config: { Image: 'ollama/ollama:0.18.2' }, + HostConfig: { DeviceRequests: [{ Driver: 'nvidia' }] }, + } + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }), + 'nvidia' + ) +}) + +test('a CPU container is expected on NVIDIA when the runtime was registered after install', () => { + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: true, amdConfigured: false, container: CPU_CONTAINER }), + 'nvidia' + ) +}) + +test('a CPU container on an AMD box is trusted as deliberate (#1232 fallback)', () => { + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: true, container: CPU_CONTAINER }), + null + ) +}) + +test('the AMD GPU type counts when no container exists', () => { + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: true, container: null }), + 'amd' + ) + assert.equal( + resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container: null }), + null + ) +}) + +test('diffAmdOllamaConfig is empty when a reinstall would rebuild the same container', () => { + assert.deepEqual(diffAmdOllamaConfig(ROCM_CONTAINER, '11.0.0'), []) +}) + +test('diffAmdOllamaConfig reports an HSA override the container is missing', () => { + const container = { + ...ROCM_CONTAINER, + Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] }, + } + assert.deepEqual(diffAmdOllamaConfig(container, '11.0.0'), [ + 'HSA_OVERRIDE_GFX_VERSION is unset, expected 11.0.0', + ]) +}) + +test('diffAmdOllamaConfig reports a stale override when none is wanted', () => { + assert.deepEqual(diffAmdOllamaConfig(ROCM_CONTAINER, null), [ + 'HSA_OVERRIDE_GFX_VERSION is 11.0.0, expected unset', + ]) +}) + +test('diffAmdOllamaConfig reports missing image, devices and iGPU flag', () => { + assert.deepEqual(diffAmdOllamaConfig(CPU_CONTAINER, null), [ + 'image is ollama/ollama:0.18.2, not the ROCm build', + '/dev/kfd is not passed through', + '/dev/dri is not passed through', + 'OLLAMA_IGPU_ENABLE=1 is not set', + ]) +}) + +const UNSUPPORTED_GFX1103_LOG = + 'level=WARN source=amd_linux.go msg="amdgpu is not supported (supported types:[gfx1030 gfx1100])" gpu_type=gfx1103 gpu=0 library=/usr/lib/ollama/rocm\n' + + 'msg="inference compute" id=cpu library=cpu name=cpu' + +test('parseAmdGfxTargetFromLogs reads the gfx target Ollama rejected', () => { + assert.equal(parseAmdGfxTargetFromLogs(UNSUPPORTED_GFX1103_LOG), 'gfx1103') + assert.equal(parseAmdGfxTargetFromLogs('msg="inference compute" id=cpu library=cpu'), null) +}) + +test('diagnoseAmdCpuFallback on the #1344 box: matching config, override needed', () => { + const container = { + ...ROCM_CONTAINER, + Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] }, + } + // No gfx marker and no KV override, so a reinstall resolves no override either. + assert.deepEqual(diagnoseAmdCpuFallback(container, UNSUPPORTED_GFX1103_LOG, null), { + amdReinstallWouldChange: false, + amdGfxTarget: 'gfx1103', + suggestedHsaOverride: '11.0.0', + }) +}) + +test('diagnoseAmdCpuFallback flags a reinstall when a newly set override is missing', () => { + const container = { + ...ROCM_CONTAINER, + Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] }, + } + assert.equal( + diagnoseAmdCpuFallback(container, UNSUPPORTED_GFX1103_LOG, '11.0.0').amdReinstallWouldChange, + true + ) +}) + +test('diagnoseAmdCpuFallback reports the current override and omits unknown fields', () => { + assert.deepEqual(diagnoseAmdCpuFallback(ROCM_CONTAINER, '', '11.0.0'), { + amdReinstallWouldChange: false, + currentHsaOverride: '11.0.0', + }) +}) diff --git a/admin/tests/unit/settings_amd_hsa_override.spec.ts b/admin/tests/unit/settings_amd_hsa_override.spec.ts new file mode 100644 index 00000000..d0ea6333 --- /dev/null +++ b/admin/tests/unit/settings_amd_hsa_override.spec.ts @@ -0,0 +1,16 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { validateSettingValue } from '../../app/validators/settings.js' + +test('ai.amdHsaOverride accepts a gfx version, none, or empty', () => { + for (const value of ['11.0.0', '10.3.0', ' 11.0.0 ', 'none', 'NONE', '', null, undefined]) { + assert.equal(validateSettingValue('ai.amdHsaOverride', value), null, String(value)) + } +}) + +test('ai.amdHsaOverride rejects anything that is not a version', () => { + for (const value of ['11', '11.0', 'gfx1103', '11.0.0; rm -rf /', '11.0.0\nFOO=1', 11, true]) { + assert.notEqual(validateSettingValue('ai.amdHsaOverride', value), null, String(value)) + } +}) diff --git a/admin/types/system.ts b/admin/types/system.ts index fb7dfb88..3f9893df 100644 --- a/admin/types/system.ts +++ b/admin/types/system.ts @@ -6,6 +6,18 @@ export type GpuHealthStatus = { hasRocmRuntime: boolean ollamaGpuAccessible: boolean gpuVendor?: 'nvidia' | 'amd' + /** + * AMD passthrough_failed only. Whether a reinstall would change the running + * container. When false, a reinstall rebuilds the same CPU-bound container and + * the fix is an HSA override (ai.amdHsaOverride). + */ + amdReinstallWouldChange?: boolean + /** gfx target Ollama reported for an unsupported AMD GPU, e.g. 'gfx1103'. */ + amdGfxTarget?: string + /** HSA_OVERRIDE_GFX_VERSION the running container has, if any. */ + currentHsaOverride?: string + /** Override the gfx mapping recommends for amdGfxTarget, if it knows one. */ + suggestedHsaOverride?: string } export type SystemInformationResponse = {