fix(gpu): detect Ollama CPU fallback on AMD and offer the right fix

GpuPassthroughRemediationProvider returned early unless the NVIDIA
runtime was registered with Docker. AMD doesn't register one, so the
log-based CPU-fallback detection added in v1.35.0-rc.1 never ran on
AMD boxes. On a Phoenix1 iGPU running ollama:rocm with /dev/kfd and
/dev/dri passed through, Ollama logged library=cpu while gpuHealth
reported status "ok".

The gpuHealth side had its own bug. hasNvidiaRuntime and
hasRocmRuntime were set only inside the lspci probe branch. When lspci
returned a plausible controller, the else branch reported "ok" without
probing and left both flags false. A blind reinstall would not have
helped either: gfx1103 needs HSA_OVERRIDE_GFX_VERSION=11.0.0, and
without the gfx marker or ai.amdHsaOverride a reinstall rebuilds the
same CPU-bound container.

- Add resolveExpectedGpuVendor(). It reads the nomad_ollama container
  config first (nvidia DeviceRequest, ROCm image tag, /dev/kfd), then
  the nvidia runtime, then the AMD GPU type when no container exists.
  /dev/dri alone is not treated as AMD. A CPU container on an AMD box
  is trusted as deliberate (#1232 fallback, acceleration disabled).
- Gate the provider on the expected vendor. NVIDIA auto-reinstall is
  unchanged but now skips when the container requests a GPU with no
  runtime registered. AMD is detect-only: the provider logs whether a
  reinstall would change the container and never reinstalls itself.
- In gpuHealth, set the runtime flags up front and let Ollama's
  startup log decide health whenever a GPU is expected, whatever lspci
  returned. A CPU-only log now reports passthrough_failed and skips
  the nvidia-smi exec. lspci stays the display source when clean.
- Parse the first GPU "inference compute" line instead of the last
  line, which could be the CPU fallback device, and recognize Vulkan.
  Move the startup-log reader into ollama_compute so both callers
  share it.
- Report amdReinstallWouldChange, amdGfxTarget, currentHsaOverride and
  suggestedHsaOverride on AMD passthrough_failed. The gfx target is
  parsed from Ollama's "amdgpu is not supported" line (best-effort).
- Add GpuPassthroughAlert, shared by Settings > System and Settings >
  Models. When a reinstall would rebuild the same AMD container, it
  offers "Fix: Set GFX Override" instead of a reinstall. The dialog
  saves ai.amdHsaOverride and then reinstalls.
- Expose ai.amdHsaOverride through the settings API. The value lands
  in the container env, so only x.y.z, "none" or empty is accepted.
- Silence the HSA override resolver's logging on the gpuHealth poll.

The #1325 case where no GPU runtime is expected and lspci reports a
controller still returns "ok" unprobed.

Fixes #1344
Refs #1325, #1232

admin/app/services/docker_service.ts

@@ -1550,6 +1550,15 @@ export class DockerService {
     }
   }
 
+  /**
+   * The HSA_OVERRIDE_GFX_VERSION an AMD install or reinstall would apply right now.
+   * Exposed so GpuPassthroughRemediationProvider can tell whether a reinstall would
+   * change the running container.
+   */
+  async getAmdHsaOverride(options: { quiet?: boolean } = {}): Promise<string | null> {
+    return this._resolveAmdHsaOverride(options)
+  }
+
   /**
    * Resolve the HSA_OVERRIDE_GFX_VERSION value for the host's AMD GPU.
    *
@@ -1589,32 +1598,32 @@ export class DockerService {
     return pepper
   }
 
-  private async _resolveAmdHsaOverride(): Promise<string | null> {
+  private async _resolveAmdHsaOverride({ quiet = false }: { quiet?: boolean } = {}): Promise<string | null> {
     const manualRaw = await KVStore.getValue('ai.amdHsaOverride')
     if (manualRaw !== null && manualRaw !== undefined && String(manualRaw).trim() !== '') {
       const manual = String(manualRaw).trim().toLowerCase()
       if (manual === 'none' || manual === 'off' || manual === 'false') {
-        logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride')
+        if (!quiet) logger.info('[DockerService] HSA override disabled via ai.amdHsaOverride')
         return null
       }
       if (/^\d+\.\d+\.\d+$/.test(manual)) {
-        logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`)
+        if (!quiet) logger.info(`[DockerService] HSA override forced to ${manual} via ai.amdHsaOverride`)
         return manual
       }
-      logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`)
+      if (!quiet) logger.warn(`[DockerService] Ignoring invalid ai.amdHsaOverride value: ${manualRaw}`)
     }
 
     try {
       const gfx = (await readFile('/app/storage/.nomad-amd-gfx', 'utf8')).trim()
       const mapped = this._mapGfxToHsaOverride(gfx)
-      logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`)
+      if (!quiet) logger.info(`[DockerService] AMD gfx marker '${gfx}' → HSA override ${mapped ?? 'none'}`)
       return mapped
     } catch {
       // Marker absent — most likely an existing install upgraded without re-running
       // install_nomad.sh. Fall through to the default.
     }
 
-    logger.warn(
+    if (!quiet) logger.warn(
       '[DockerService] AMD GPU configured but no gfx marker (/app/storage/.nomad-amd-gfx) and no ' +
         'ai.amdHsaOverride KV; relying on native ROCm discovery. iGPUs not on the bundled rocblas ' +
         'allowlist (e.g. 780M/gfx1103, 680M/gfx1035) will silently fall back to CPU. Set the ' +

admin/app/services/ollama_compute.ts

@@ -1,8 +1,13 @@
+import type Docker from 'dockerode'
+import { mapGfxToHsaOverride } from '../utils/amd_hsa_override.js'
+
 /**
  * Pure helpers for classifying which compute backend Ollama actually loaded,
  * parsed from its container logs. Extracted so both SystemService (GPU health
  * display) and GpuPassthroughRemediationProvider (auto-reinstall decision) share
  * one source of truth, and so the logic is unit-testable without Docker.
+ * readOllamaStartupLogs is the one Docker-facing helper, kept here so both
+ * callers read the same log window.
  *
  * Ollama writes one `inference compute` log line per detected device a few seconds
  * after startup, e.g.:
@@ -39,3 +44,200 @@ export function classifyOllamaComputeBackend(logText: string): OllamaComputeBack
 
   return 'unknown'
 }
+
+export type OllamaGpuLibrary = 'CUDA' | 'ROCm' | 'Vulkan'
+
+export type OllamaGpuInfo = {
+  library: OllamaGpuLibrary
+  name: string
+  vramMiB: number
+}
+
+/**
+ * Parse the GPU device Ollama reported at startup, or null when no GPU line is
+ * present. Takes the first GPU line rather than the last `inference compute`
+ * line: Ollama also lists a CPU fallback device, and when that line lands last
+ * a working GPU would otherwise read as no GPU at all.
+ */
+export function parseOllamaGpuFromLogs(logText: string): OllamaGpuInfo | null {
+  const line = logText
+    .split('\n')
+    .find((l) => l.includes(INFERENCE_LINE_MARKER) && GPU_LIBRARY_RE.test(l))
+  if (!line) return null
+
+  const library = line.match(GPU_LIBRARY_RE)![1] as OllamaGpuLibrary
+  const descMatch = line.match(/description="([^"]+)"/)
+  const totalMatch = line.match(/total="([0-9.]+)\s*GiB"/)
+  const fallbackName = library === 'CUDA' ? 'NVIDIA GPU' : library === 'ROCm' ? 'AMD GPU' : 'GPU'
+
+  return {
+    library,
+    name: descMatch?.[1] || fallbackName,
+    vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0,
+  }
+}
+
+export type GpuVendor = 'nvidia' | 'amd'
+
+/**
+ * The subset of `docker inspect nomad_ollama` the GPU checks read. Structural so
+ * dockerode's ContainerInspectInfo satisfies it and tests can pass plain objects.
+ */
+export type OllamaContainerSnapshot = {
+  Config?: { Image?: string; Env?: string[] | null }
+  HostConfig?: {
+    Devices?: Array<{ PathOnHost?: string }> | null
+    DeviceRequests?: Array<{ Driver?: string }> | null
+  }
+}
+
+function hasDevice(container: OllamaContainerSnapshot, path: string): boolean {
+  return (container.HostConfig?.Devices ?? []).some((d) => d.PathOnHost === path)
+}
+
+// Matches rocm in the tag only (ollama/ollama:rocm, :0.12.0-rocm), not the repo path.
+function isRocmImage(image: string | undefined): boolean {
+  return /:[^/:]*rocm[^/:]*$/i.test(image ?? '')
+}
+
+/**
+ * Which GPU vendor, if any, Ollama is expected to run on.
+ *
+ * The container's own config comes first because it records what the install
+ * actually set up, independent of how the host is detected today:
+ *   - an nvidia DeviceRequest means NVIDIA
+ *   - a ROCm image tag or a /dev/kfd device means AMD (/dev/dri alone is not
+ *     AMD-specific, Intel and Vulkan setups pass it too)
+ * A registered nvidia runtime comes next, which covers a container created
+ * CPU-only before the runtime was added.
+ *
+ * The persisted AMD GPU type counts only when no container exists. An AMD
+ * container without ROCm config is deliberate: either acceleration was turned
+ * off, or the daemon rejected /dev/kfd and the install fell back to CPU (#1232).
+ * Flagging it would offer a reinstall that rebuilds the same CPU container. AMD
+ * never registers a Docker runtime, so gating on the runtime alone skips every
+ * AMD box (#1344).
+ */
+export function resolveExpectedGpuVendor(input: {
+  hasNvidiaRuntime: boolean
+  amdConfigured: boolean
+  container: OllamaContainerSnapshot | null
+}): GpuVendor | null {
+  const { container } = input
+  if (container) {
+    if ((container.HostConfig?.DeviceRequests ?? []).some((r) => r.Driver === 'nvidia')) {
+      return 'nvidia'
+    }
+    if (isRocmImage(container.Config?.Image) || hasDevice(container, '/dev/kfd')) {
+      return 'amd'
+    }
+  }
+  if (input.hasNvidiaRuntime) return 'nvidia'
+  if (!container && input.amdConfigured) return 'amd'
+  return null
+}
+
+/**
+ * Where a running AMD nomad_ollama container differs from what a reinstall would
+ * create. An empty list means a reinstall would rebuild the same container, so it
+ * cannot move Ollama off the CPU and the fix lies elsewhere (usually the HSA
+ * override for an iGPU missing from ROCm's allowlist).
+ */
+export function diffAmdOllamaConfig(
+  container: OllamaContainerSnapshot,
+  desiredHsaOverride: string | null
+): string[] {
+  const drift: string[] = []
+  const env = container.Config?.Env ?? []
+
+  if (!isRocmImage(container.Config?.Image)) {
+    drift.push(`image is ${container.Config?.Image ?? 'unknown'}, not the ROCm build`)
+  }
+  if (!hasDevice(container, '/dev/kfd')) drift.push('/dev/kfd is not passed through')
+  if (!hasDevice(container, '/dev/dri')) drift.push('/dev/dri is not passed through')
+
+  const currentOverride =
+    env.find((e) => e.startsWith('HSA_OVERRIDE_GFX_VERSION='))?.split('=')[1] ?? null
+  if (currentOverride !== desiredHsaOverride) {
+    drift.push(
+      `HSA_OVERRIDE_GFX_VERSION is ${currentOverride ?? 'unset'}, expected ${desiredHsaOverride ?? 'unset'}`
+    )
+  }
+  if (!env.includes('OLLAMA_IGPU_ENABLE=1')) drift.push('OLLAMA_IGPU_ENABLE=1 is not set')
+
+  return drift
+}
+
+/**
+ * The gfx target Ollama named when it rejected an AMD GPU, e.g. 'gfx1103' from
+ *   msg="amdgpu is not supported (supported types:[gfx1030 ...])" gpu_type=gfx1103 ...
+ * Best-effort: the exact wording varies across Ollama releases, so callers treat
+ * null as "unknown" and fall back to asking the user.
+ */
+export function parseAmdGfxTargetFromLogs(logText: string): string | null {
+  return logText.match(/gpu_type=(gfx[0-9a-f]+)\b/i)?.[1]?.toLowerCase() ?? null
+}
+
+/**
+ * Why an AMD container expected on the GPU is running on CPU, in the shape
+ * GpuHealthStatus reports to the UI.
+ */
+export function diagnoseAmdCpuFallback(
+  container: OllamaContainerSnapshot,
+  logText: string,
+  desiredHsaOverride: string | null
+) {
+  const gfx = parseAmdGfxTargetFromLogs(logText)
+  const current = (container.Config?.Env ?? [])
+    .find((e) => e.startsWith('HSA_OVERRIDE_GFX_VERSION='))
+    ?.split('=')[1]
+  const suggested = gfx ? mapGfxToHsaOverride(gfx) : null
+  return {
+    amdReinstallWouldChange: diffAmdOllamaConfig(container, desiredHsaOverride).length > 0,
+    ...(gfx && { amdGfxTarget: gfx }),
+    ...(current && { currentHsaOverride: current }),
+    ...(suggested && { suggestedHsaOverride: suggested }),
+  }
+}
+
+/**
+ * Read nomad_ollama's logs from its startup window (the five minutes after it
+ * booted), where the "inference compute" line is emitted. A tail:N read is
+ * fragile here: under embedding load the container writes >1000 lines/min,
+ * which pushes the line out of any reasonable tail within minutes, while the
+ * startup window is bounded and never ages out. Falls back to tail:500 when
+ * StartedAt is missing. Returns '' on any error so callers classify the backend
+ * as 'unknown' rather than throwing.
+ */
+export async function readOllamaStartupLogs(
+  container: Docker.Container,
+  inspect?: { State?: { StartedAt?: string } }
+): Promise<string> {
+  try {
+    const state = inspect ?? (await container.inspect())
+    const startedAtRaw = state?.State?.StartedAt
+    const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
+
+    const logsOpts: {
+      stdout: true
+      stderr: true
+      follow: false
+      since?: number
+      until?: number
+      tail?: number
+    } = { stdout: true, stderr: true, follow: false }
+
+    if (Number.isFinite(startedAtMs) && startedAtMs > 0) {
+      const startedAtSec = Math.floor(startedAtMs / 1000)
+      logsOpts.since = startedAtSec
+      logsOpts.until = startedAtSec + 300
+    } else {
+      logsOpts.tail = 500
+    }
+
+    const buf = (await container.logs(logsOpts)) as unknown as Buffer
+    return buf.toString('utf8')
+  } catch {
+    return ''
+  }
+}

admin/app/services/system_service.ts

@@ -22,6 +22,14 @@ import KVStore from '#models/kv_store'
 import { KV_STORE_SCHEMA, KVStoreKey } from '../../types/kv_store.js'
 import { isNewerVersion } from '../utils/version.js'
 import { isUnresolvedGpuModel } from '../utils/gpu_model.js'
+import {
+  classifyOllamaComputeBackend,
+  diagnoseAmdCpuFallback,
+  parseOllamaGpuFromLogs,
+  readOllamaStartupLogs,
+  resolveExpectedGpuVendor,
+  type OllamaGpuInfo,
+} from '#services/ollama_compute'
 import { invalidateAssistantNameCache } from '../../config/inertia.js'
 import { invalidateMinRelevanceCache } from '../utils/rag_relevance.js'
 import { invalidateResponseStyleCache } from '../utils/response_style.js'
@@ -103,6 +111,20 @@ export class SystemService {
     return false
   }
 
+  /**
+   * The running nomad_ollama container and its inspect data, or null when it is
+   * not running. The inspect data records the GPU config the install actually
+   * applied (DeviceRequests, /dev/kfd, the ROCm image), which is what the GPU
+   * health check keys on.
+   */
+  private async _findRunningOllamaContainer() {
+    const containers = await this.dockerService.docker.listContainers({ all: false })
+    const summary = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
+    if (!summary) return null
+    const container = this.dockerService.docker.getContainer(summary.Id)
+    return { container, inspect: await container.inspect() }
+  }
+
   /**
    * Probe Ollama startup logs for the canonical "inference compute" line that records
    * which compute backend was selected. This catches silent CPU fallback (e.g. when
@@ -114,70 +136,11 @@ export class SystemService {
    *   - the line has not been emitted (Ollama still starting up)
    *   - logs show CPU-only operation (no GPU detected)
    */
-  async getOllamaInferenceComputeFromLogs(): Promise<{
-    library: 'CUDA' | 'ROCm'
-    name: string
-    vramMiB: number
-  } | null> {
+  async getOllamaInferenceComputeFromLogs(): Promise<OllamaGpuInfo | null> {
     try {
-      const containers = await this.dockerService.docker.listContainers({ all: false })
-      const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
-      if (!ollamaContainer) return null
-
-      const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
-
-      // Read logs only from the first 5 minutes after container start. The
-      // "inference compute" line is written once during Ollama's GPU discovery
-      // phase, within seconds of startup. Using tail:N here is fragile: under
-      // active embedding workloads we've seen >1000 lines/min, which pushes the
-      // line past any reasonable tail in minutes. Pinning to the startup window
-      // is bounded (~5 min of logs regardless of container uptime) and never
-      // ages out.
-      //
-      // Fall back to the previous tail:500 strategy if StartedAt is missing or
-      // unparseable — we can't construct a since/until window without it, but
-      // tail:500 is still useful when the container just started and the line
-      // is still recent.
-      const inspect = await container.inspect()
-      const startedAtRaw = inspect?.State?.StartedAt
-      const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
-      const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0
-
-      const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = {
-        stdout: true,
-        stderr: true,
-        follow: false,
-      }
-      if (hasValidStartedAt) {
-        const startedAtSec = Math.floor(startedAtMs / 1000)
-        logsOpts.since = startedAtSec
-        logsOpts.until = startedAtSec + 300 // 5-minute window
-      } else {
-        logger.warn(
-          `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe`
-        )
-        logsOpts.tail = 500
-      }
-      const buf = (await container.logs(logsOpts)) as unknown as Buffer
-      const logs = buf.toString('utf8')
-
-      const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"'))
-      if (lines.length === 0) return null
-
-      const lastLine = lines[lines.length - 1]
-      const libraryMatch = lastLine.match(/library=(CUDA|ROCm)/)
-      if (!libraryMatch) return null
-
-      const descMatch = lastLine.match(/description="([^"]+)"/)
-      const totalMatch = lastLine.match(/total="([0-9.]+)\s*GiB"/)
-
-      return {
-        library: libraryMatch[1] as 'CUDA' | 'ROCm',
-        name:
-          descMatch?.[1] ||
-          (libraryMatch[1] === 'CUDA' ? 'NVIDIA GPU' : 'AMD GPU'),
-        vramMiB: totalMatch ? Math.round(Number.parseFloat(totalMatch[1]) * 1024) : 0,
-      }
+      const ollama = await this._findRunningOllamaContainer()
+      if (!ollama) return null
+      return parseOllamaGpuFromLogs(await readOllamaStartupLogs(ollama.container, ollama.inspect))
     } catch (error) {
       logger.warn(
         `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}`
@@ -526,91 +489,106 @@ export class SystemService {
 
         // Run the probes when controllers are empty (common inside Docker),
         // when lspci gave us bogus discrete-GPU BAR0 values that need replacing,
-        // or when it named a card it couldn't resolve.
-        if (
+        // or when it named a card it couldn't resolve. Only in those cases do
+        // probe results replace the lspci controllers; a clean lspci reading
+        // stays as the display source.
+        const lspciNeedsProbe =
           !graphics.controllers ||
           graphics.controllers.length === 0 ||
           hasLspciBogusDgpuVram ||
           hasUnresolvedGpuName
-        ) {
-          const runtimes = dockerInfo.Runtimes || {}
-          gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes
+        const useProbeControllers = (controllers: typeof graphics.controllers) => {
+          if (lspciNeedsProbe) graphics.controllers = controllers
+        }
+        const toControllers = (gpus: Array<{ vendor: string; model: string; vram: number }>) =>
+          gpus.map((gpu) => ({
+            model: gpu.model,
+            vendor: gpu.vendor,
+            bus: '',
+            vram: gpu.vram,
+            vramDynamic: false,
+          }))
 
-          // AMD doesn't register a Docker runtime. Detection sources, in priority order:
-          //   1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
-          //   2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
-          // The marker file matters because the System page should reflect AMD presence
-          // even before AI Assistant has been installed for the first time.
-          let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
-          if (!savedGpuType) {
-            try {
-              savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
-            } catch {}
-          }
-          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
-          const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
-          gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled
+        // Runtime flags are reported whichever path runs below. They used to be set
+        // only inside the probe branch, so a box whose lspci reading looked clean
+        // reported hasRocmRuntime: false even with an AMD GPU configured (#1344).
+        const runtimes = dockerInfo.Runtimes || {}
+        gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes
 
-          if (gpuHealth.hasNvidiaRuntime || gpuHealth.hasRocmRuntime) {
-            gpuHealth.gpuVendor = gpuHealth.hasNvidiaRuntime ? 'nvidia' : 'amd'
+        // AMD doesn't register a Docker runtime. Detection sources, in priority order:
+        //   1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
+        //   2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
+        // The marker file matters because the System page should reflect AMD presence
+        // even before AI Assistant has been installed for the first time.
+        let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
+        if (!savedGpuType) {
+          try {
+            savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
+          } catch {}
+        }
+        const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+        const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
+        gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled
+
+        // When Ollama is expected on a GPU, its startup log decides health whatever
+        // lspci returned. A clean lspci reading says a GPU exists, not that Ollama
+        // uses it: an AMD iGPU silently on CPU used to land in the else branch below
+        // and report 'ok' (#1344).
+        const ollama = await this._findRunningOllamaContainer().catch(() => null)
+        const expectedVendor = resolveExpectedGpuVendor({
+          hasNvidiaRuntime: gpuHealth.hasNvidiaRuntime,
+          amdConfigured: gpuHealth.hasRocmRuntime,
+          container: ollama?.inspect ?? null,
+        })
+
+        if (expectedVendor || lspciNeedsProbe) {
+          if (expectedVendor) {
+            gpuHealth.gpuVendor = expectedVendor
 
             // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback
-            const logInfo = await this.getOllamaInferenceComputeFromLogs()
+            const ollamaLogs = ollama
+              ? await readOllamaStartupLogs(ollama.container, ollama.inspect)
+              : ''
+            const logInfo = parseOllamaGpuFromLogs(ollamaLogs)
             if (logInfo) {
-              graphics.controllers = [
+              useProbeControllers([
                 {
                   model: logInfo.name,
-                  vendor: logInfo.library === 'CUDA' ? 'NVIDIA' : 'AMD',
+                  vendor:
+                    logInfo.library === 'CUDA' ? 'NVIDIA' : logInfo.library === 'ROCm' ? 'AMD' : '',
                   bus: '',
                   vram: logInfo.vramMiB,
                   vramDynamic: false,
                 },
-              ]
+              ])
               gpuHealth.status = 'ok'
               gpuHealth.ollamaGpuAccessible = true
-            } else if (gpuHealth.hasNvidiaRuntime) {
+            } else if (classifyOllamaComputeBackend(ollamaLogs) === 'cpu') {
+              // Ollama reported its devices and none was a GPU. Definitive for both
+              // vendors, so skip the nvidia-smi exec, which cannot run in a CPU-only
+              // container anyway.
+              gpuHealth.status = 'passthrough_failed'
+              logger.warn(
+                `${expectedVendor === 'amd' ? 'AMD' : 'NVIDIA'} GPU expected but Ollama reported a CPU-only backend at startup`
+              )
+            } else if (expectedVendor === 'nvidia') {
               // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when
               // the log parser hasn't seen a startup line yet (e.g. log rotation,
               // very fresh container). Distinguishes "no Ollama container" from
               // "container exists but GPU broken".
               const nvidiaInfo = await this.getNvidiaSmiInfo()
               if (Array.isArray(nvidiaInfo)) {
-                graphics.controllers = nvidiaInfo.map((gpu) => ({
-                  model: gpu.model,
-                  vendor: gpu.vendor,
-                  bus: '',
-                  vram: gpu.vram,
-                  vramDynamic: false,
-                }))
+                useProbeControllers(toControllers(nvidiaInfo))
                 gpuHealth.status = 'ok'
                 gpuHealth.ollamaGpuAccessible = true
-              } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
-                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
-                if (externalOllamaGpu) {
-                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
-                    model: gpu.model,
-                    vendor: gpu.vendor,
-                    bus: '',
-                    vram: gpu.vram,
-                    vramDynamic: false,
-                  }))
-                  gpuHealth.status = 'ok'
-                  gpuHealth.ollamaGpuAccessible = true
-                } else {
-                  gpuHealth.status = 'ollama_not_installed'
-                }
               } else {
                 const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
                 if (externalOllamaGpu) {
-                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
-                    model: gpu.model,
-                    vendor: gpu.vendor,
-                    bus: '',
-                    vram: gpu.vram,
-                    vramDynamic: false,
-                  }))
+                  useProbeControllers(toControllers(externalOllamaGpu))
                   gpuHealth.status = 'ok'
                   gpuHealth.ollamaGpuAccessible = true
+                } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
+                  gpuHealth.status = 'ollama_not_installed'
                 } else {
                   gpuHealth.status = 'passthrough_failed'
                   logger.warn(
@@ -618,38 +596,41 @@ export class SystemService {
                   )
                 }
               }
-            } else {
+            } else if (!ollama) {
               // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative.
               // Distinguish "Ollama not running" from "Ollama running but no GPU log line".
-              const containers = await this.dockerService.docker.listContainers({ all: false })
-              const ollamaRunning = containers.some((c) =>
-                c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)
-              )
-              if (!ollamaRunning) {
-                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
-                if (externalOllamaGpu) {
-                  graphics.controllers = externalOllamaGpu.map((gpu) => ({
-                    model: gpu.model,
-                    vendor: gpu.vendor,
-                    bus: '',
-                    vram: gpu.vram,
-                    vramDynamic: false,
-                  }))
-                  gpuHealth.status = 'ok'
-                  gpuHealth.ollamaGpuAccessible = true
-                } else {
-                  gpuHealth.status = 'ollama_not_installed'
-                }
+              const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
+              if (externalOllamaGpu) {
+                useProbeControllers(toControllers(externalOllamaGpu))
+                gpuHealth.status = 'ok'
+                gpuHealth.ollamaGpuAccessible = true
               } else {
-                gpuHealth.status = 'passthrough_failed'
-                logger.warn(
-                  'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
-                )
+                gpuHealth.status = 'ollama_not_installed'
               }
+            } else {
+              gpuHealth.status = 'passthrough_failed'
+              logger.warn(
+                'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
+              )
+            }
+
+            // Tell the UI whether a reinstall would help or an HSA override is needed.
+            // Reinstalling an AMD container whose config already matches rebuilds the
+            // same CPU-bound container, so the banner must not offer that as the fix.
+            if (gpuHealth.status === 'passthrough_failed' && expectedVendor === 'amd' && ollama) {
+              Object.assign(
+                gpuHealth,
+                diagnoseAmdCpuFallback(
+                  ollama.inspect,
+                  ollamaLogs,
+                  await this.dockerService.getAmdHsaOverride({ quiet: true })
+                )
+              )
             }
           }
         } else {
-          // si.graphics() returned controllers (host install, not Docker) — GPU is working
+          // si.graphics() returned usable controllers and no GPU runtime is expected
+          // (host install, not Docker). Still unprobed, see #1325.
           gpuHealth.status = 'ok'
           gpuHealth.ollamaGpuAccessible = true
         }

admin/app/validators/settings.ts

@@ -118,6 +118,19 @@ export function validateSettingValue(key: KVStoreKey, value: unknown): string |
             }
             return null
         }
+        case 'ai.amdHsaOverride': {
+            // Lands in the Ollama container env as HSA_OVERRIDE_GFX_VERSION, so only
+            // the forms DockerService._resolveAmdHsaOverride honours are accepted.
+            // Empty clears it and 'none' disables the gfx-marker default.
+            const raw = typeof value === 'string' ? value.trim().toLowerCase() : value
+            if (raw === '' || raw === undefined || raw === null || raw === 'none') {
+                return null
+            }
+            if (typeof raw !== 'string' || !/^\d{1,2}\.\d{1,2}\.\d{1,2}$/.test(raw)) {
+                return 'GFX override must be a version like "11.0.0", "none", or empty to use the detected default.'
+            }
+            return null
+        }
         default:
             return null
     }

admin/constants/kv_store.ts

@@ -15,6 +15,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [
     'ai.contextWindow',
     'ai.keepAlive',
     'ai.responseStyle',
+    'ai.amdHsaOverride',
     'rag.defaultIngestPolicy',
     'rag.enabled',
     'rag.minRelevance',

admin/inertia/components/GpuPassthroughAlert.tsx

@@ -0,0 +1,217 @@
+import { useState } from 'react'
+import Alert from '~/components/Alert'
+import StyledModal from '~/components/StyledModal'
+import Input from '~/components/inputs/Input'
+import { useNotifications } from '~/context/NotificationContext'
+import { useModals } from '~/context/ModalContext'
+import api from '~/lib/api'
+import type { GpuHealthStatus } from '../../types/system'
+
+const DISMISS_KEY = 'nomad:gpu-banner-dismissed'
+const HSA_OVERRIDE_PATTERN = /^\d{1,2}\.\d{1,2}\.\d{1,2}$/
+
+/**
+ * Banner for gpuHealth.status === 'passthrough_failed'.
+ *
+ * NVIDIA, and AMD whose container differs from what a reinstall would create, get
+ * the one-click reinstall. AMD whose container already matches gets an HSA override
+ * prompt instead: reinstalling it rebuilds the same CPU-bound container (#1344).
+ */
+export default function GpuPassthroughAlert({
+  gpuHealth,
+  assistantName,
+  className,
+}: {
+  gpuHealth: GpuHealthStatus | undefined
+  assistantName: string
+  className?: string
+}) {
+  const { addNotification } = useNotifications()
+  const { openModal, closeAllModals } = useModals()
+  const [reinstalling, setReinstalling] = useState(false)
+  const [dismissed, setDismissed] = useState(() => {
+    try {
+      return localStorage.getItem(DISMISS_KEY) === 'true'
+    } catch {
+      return false
+    }
+  })
+
+  if (gpuHealth?.status !== 'passthrough_failed' || dismissed) return null
+
+  const isAmd = gpuHealth.gpuVendor === 'amd'
+  const needsHsaOverride = isAmd && gpuHealth.amdReinstallWouldChange === false
+
+  const handleDismiss = () => {
+    setDismissed(true)
+    try {
+      localStorage.setItem(DISMISS_KEY, 'true')
+    } catch {}
+  }
+
+  const reinstall = async () => {
+    setReinstalling(true)
+    try {
+      const response = await api.forceReinstallService('nomad_ollama')
+      if (!response || !response.success) {
+        throw new Error(response?.message || 'Force reinstall failed')
+      }
+      addNotification({
+        message: `${assistantName} is being reinstalled with GPU support. This page will reload shortly.`,
+        type: 'success',
+      })
+      try {
+        localStorage.removeItem(DISMISS_KEY)
+      } catch {}
+      setTimeout(() => window.location.reload(), 5000)
+    } catch (error) {
+      addNotification({
+        message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        type: 'error',
+      })
+      setReinstalling(false)
+    }
+  }
+
+  const openReinstallModal = () => {
+    openModal(
+      <StyledModal
+        title={`Reinstall ${assistantName}?`}
+        onConfirm={() => {
+          closeAllModals()
+          reinstall()
+        }}
+        onCancel={closeAllModals}
+        open={true}
+        confirmText="Reinstall"
+        cancelText="Cancel"
+      >
+        <p className="text-text-primary">
+          This will recreate the {assistantName} container with GPU support enabled. Your
+          downloaded models will be preserved. The service will be briefly unavailable during
+          reinstall.
+        </p>
+      </StyledModal>,
+      'gpu-health-force-reinstall-modal'
+    )
+  }
+
+  const openHsaOverrideModal = () => {
+    openModal(
+      <HsaOverrideModal
+        gpuHealth={gpuHealth}
+        assistantName={assistantName}
+        onCancel={closeAllModals}
+        onSaved={() => {
+          closeAllModals()
+          reinstall()
+        }}
+      />,
+      'gpu-health-hsa-override-modal'
+    )
+  }
+
+  const vendorName = isAmd ? 'an AMD' : 'an NVIDIA'
+  const message = needsHsaOverride
+    ? `Your system has an AMD GPU, but ${assistantName} is running on CPU only. Reinstalling won't fix this: the GPU${gpuHealth.amdGfxTarget ? ` (${gpuHealth.amdGfxTarget})` : ''} isn't on ROCm's supported list and needs a GFX version override.`
+    : `Your system has ${vendorName} GPU, but ${assistantName} can't access it. AI is running on CPU only, which is significantly slower.`
+
+  return (
+    <Alert
+      type="warning"
+      variant="bordered"
+      title={`GPU Not Accessible to ${assistantName}`}
+      message={message}
+      className={className}
+      dismissible={true}
+      onDismiss={handleDismiss}
+      buttonProps={{
+        children: needsHsaOverride ? 'Fix: Set GFX Override' : `Fix: Reinstall ${assistantName}`,
+        icon: needsHsaOverride ? 'IconTool' : 'IconRefresh',
+        variant: 'action',
+        size: 'sm',
+        onClick: needsHsaOverride ? openHsaOverrideModal : openReinstallModal,
+        loading: reinstalling,
+        disabled: reinstalling,
+      }}
+    />
+  )
+}
+
+function HsaOverrideModal({
+  gpuHealth,
+  assistantName,
+  onCancel,
+  onSaved,
+}: {
+  gpuHealth: GpuHealthStatus
+  assistantName: string
+  onCancel: () => void
+  onSaved: () => void
+}) {
+  const { addNotification } = useNotifications()
+  const [value, setValue] = useState(gpuHealth.suggestedHsaOverride ?? '')
+  const [saving, setSaving] = useState(false)
+  const valid = HSA_OVERRIDE_PATTERN.test(value.trim())
+
+  const save = async () => {
+    setSaving(true)
+    try {
+      const response = await api.updateSetting('ai.amdHsaOverride', value.trim())
+      // catchInternal already notified on a request failure and returned undefined.
+      if (!response) {
+        setSaving(false)
+        return
+      }
+      if (!response.success) throw new Error(response.message)
+      onSaved()
+    } catch (error) {
+      addNotification({
+        message: `Failed to save the GFX override: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        type: 'error',
+      })
+      setSaving(false)
+    }
+  }
+
+  return (
+    <StyledModal
+      title="Set AMD GFX Override"
+      onConfirm={save}
+      onCancel={onCancel}
+      open={true}
+      confirmText="Save and Reinstall"
+      cancelText="Cancel"
+      confirmLoading={saving}
+      confirmDisabled={!valid || saving}
+    >
+      <div className="space-y-4 text-text-primary">
+        <p>
+          ROCm ships GPU kernels for a fixed list of AMD chips. Integrated Radeon GPUs outside
+          that list run on CPU unless ROCm treats them as a supported chip, set through
+          HSA_OVERRIDE_GFX_VERSION.
+        </p>
+        <ul className="list-disc pl-5 space-y-1">
+          <li>Radeon 780M / 760M (gfx1103): 11.0.0</li>
+          <li>Radeon 680M / 660M and other RDNA 2 iGPUs (gfx1031 to gfx1036): 10.3.0</li>
+        </ul>
+        <p className="text-sm text-text-secondary">
+          Detected GPU: {gpuHealth.amdGfxTarget ?? 'not reported by Ollama'}. Current override:{' '}
+          {gpuHealth.currentHsaOverride ?? 'none'}.
+          {gpuHealth.suggestedHsaOverride
+            ? ` Recommended for this GPU: ${gpuHealth.suggestedHsaOverride}.`
+            : ''}
+        </p>
+        <Input
+          name="amdHsaOverride"
+          label="GFX version override"
+          placeholder="11.0.0"
+          value={value}
+          onChange={(e) => setValue(e.target.value)}
+          error={value.trim() !== '' && !valid}
+          helpText={`Saving recreates the ${assistantName} container with this override. Downloaded models are preserved.`}
+        />
+      </div>
+    </StyledModal>
+  )
+}

admin/inertia/pages/settings/models.tsx

@@ -23,6 +23,7 @@ import { formatBytes } from '~/lib/util'
 import useDebounce from '~/hooks/useDebounce'
 import ActiveModelDownloads from '~/components/ActiveModelDownloads'
 import { useSystemInfo } from '~/hooks/useSystemInfo'
+import GpuPassthroughAlert from '~/components/GpuPassthroughAlert'
 
 export default function ModelsPage(props: {
   models: {
@@ -41,62 +42,6 @@ export default function ModelsPage(props: {
   const { data: systemInfo } = useSystemInfo({})
   const queryClient = useQueryClient()
 
-  const [gpuBannerDismissed, setGpuBannerDismissed] = useState(() => {
-    try {
-      return localStorage.getItem('nomad:gpu-banner-dismissed') === 'true'
-    } catch {
-      return false
-    }
-  })
-  const [reinstalling, setReinstalling] = useState(false)
-
-  const handleDismissGpuBanner = () => {
-    setGpuBannerDismissed(true)
-    try {
-      localStorage.setItem('nomad:gpu-banner-dismissed', 'true')
-    } catch {}
-  }
-
-  const handleForceReinstallOllama = () => {
-    openModal(
-      <StyledModal
-        title="Reinstall AI Assistant?"
-        onConfirm={async () => {
-          closeAllModals()
-          setReinstalling(true)
-          try {
-            const response = await api.forceReinstallService('nomad_ollama')
-            if (!response || !response.success) {
-              throw new Error(response?.message || 'Force reinstall failed')
-            }
-            addNotification({
-              message: `${aiAssistantName} is being reinstalled with GPU support. This page will reload shortly.`,
-              type: 'success',
-            })
-            try { localStorage.removeItem('nomad:gpu-banner-dismissed') } catch {}
-            setTimeout(() => window.location.reload(), 5000)
-          } catch (error) {
-            addNotification({
-              message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`,
-              type: 'error',
-            })
-            setReinstalling(false)
-          }
-        }}
-        onCancel={closeAllModals}
-        open={true}
-        confirmText="Reinstall"
-        cancelText="Cancel"
-      >
-        <p className="text-text-primary">
-          This will recreate the {aiAssistantName} container with GPU support enabled.
-          Your downloaded models will be preserved. The service will be briefly
-          unavailable during reinstall.
-        </p>
-      </StyledModal>,
-      'gpu-health-force-reinstall-modal'
-    )
-  }
   const [chatSuggestionsEnabled, setChatSuggestionsEnabled] = useState(
     props.models.settings.chatSuggestionsEnabled
   )
@@ -345,24 +290,11 @@ export default function ModelsPage(props: {
               className="!mt-6"
             />
           )}
-          {isInstalled && systemInfo?.gpuHealth?.status === 'passthrough_failed' && !gpuBannerDismissed && (
-            <Alert
-              type="warning"
-              variant="bordered"
-              title="GPU Not Accessible"
-              message={`Your system has ${systemInfo?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but ${aiAssistantName} can't access it. AI is running on CPU only, which is significantly slower.`}
+          {isInstalled && (
+            <GpuPassthroughAlert
+              gpuHealth={systemInfo?.gpuHealth}
+              assistantName={aiAssistantName}
               className="!mt-6"
-              dismissible={true}
-              onDismiss={handleDismissGpuBanner}
-              buttonProps={{
-                children: `Fix: Reinstall ${aiAssistantName}`,
-                icon: 'IconRefresh',
-                variant: 'action',
-                size: 'sm',
-                onClick: handleForceReinstallOllama,
-                loading: reinstalling,
-                disabled: reinstalling,
-              }}
             />
           )}
 

admin/inertia/pages/settings/system.tsx

@@ -1,4 +1,3 @@
-import { useState } from 'react'
 import { Head } from '@inertiajs/react'
 import SettingsLayout from '~/layouts/SettingsLayout'
 import { SystemInformationResponse } from '../../../types/system'
@@ -8,11 +7,8 @@ import CircularGauge from '~/components/systeminfo/CircularGauge'
 import HorizontalBarChart from '~/components/HorizontalBarChart'
 import InfoCard from '~/components/systeminfo/InfoCard'
 import Alert from '~/components/Alert'
-import StyledModal from '~/components/StyledModal'
+import GpuPassthroughAlert from '~/components/GpuPassthroughAlert'
 import { useSystemInfo } from '~/hooks/useSystemInfo'
-import { useNotifications } from '~/context/NotificationContext'
-import { useModals } from '~/context/ModalContext'
-import api from '~/lib/api'
 import StatusCard from '~/components/systeminfo/StatusCard'
 import { IconCpu, IconDatabase, IconServer, IconDeviceDesktop, IconComponents } from '@tabler/icons-react'
 
@@ -22,65 +18,6 @@ export default function SettingsPage(props: {
   const { data: info } = useSystemInfo({
     initialData: props.system.info,
   })
-  const { addNotification } = useNotifications()
-  const { openModal, closeAllModals } = useModals()
-
-  const [gpuBannerDismissed, setGpuBannerDismissed] = useState(() => {
-    try {
-      return localStorage.getItem('nomad:gpu-banner-dismissed') === 'true'
-    } catch {
-      return false
-    }
-  })
-  const [reinstalling, setReinstalling] = useState(false)
-
-  const handleDismissGpuBanner = () => {
-    setGpuBannerDismissed(true)
-    try {
-      localStorage.setItem('nomad:gpu-banner-dismissed', 'true')
-    } catch {}
-  }
-
-  const handleForceReinstallOllama = () => {
-    openModal(
-      <StyledModal
-        title="Reinstall AI Assistant?"
-        onConfirm={async () => {
-          closeAllModals()
-          setReinstalling(true)
-          try {
-            const response = await api.forceReinstallService('nomad_ollama')
-            if (!response || !response.success) {
-              throw new Error(response?.message || 'Force reinstall failed')
-            }
-            addNotification({
-              message: 'AI Assistant is being reinstalled with GPU support. This page will reload shortly.',
-              type: 'success',
-            })
-            try { localStorage.removeItem('nomad:gpu-banner-dismissed') } catch {}
-            setTimeout(() => window.location.reload(), 5000)
-          } catch (error) {
-            addNotification({
-              message: `Failed to reinstall: ${error instanceof Error ? error.message : 'Unknown error'}`,
-              type: 'error',
-            })
-            setReinstalling(false)
-          }
-        }}
-        onCancel={closeAllModals}
-        open={true}
-        confirmText="Reinstall"
-        cancelText="Cancel"
-      >
-        <p className="text-text-primary">
-          This will recreate the AI Assistant container with GPU support enabled.
-          Your downloaded models will be preserved. The service will be briefly
-          unavailable during reinstall.
-        </p>
-      </StyledModal>,
-      'gpu-health-force-reinstall-modal'
-    )
-  }
 
   // Use (total - available) to reflect actual memory pressure.
   // mem.used includes reclaimable buff/cache on Linux, which inflates the number.
@@ -203,25 +140,9 @@ export default function SettingsPage(props: {
                   },
                 ]}
               />
-              {info?.gpuHealth?.status === 'passthrough_failed' && !gpuBannerDismissed && (
+              {info?.gpuHealth?.status === 'passthrough_failed' && (
                 <div className="lg:col-span-2">
-                  <Alert
-                    type="warning"
-                    variant="bordered"
-                    title="GPU Not Accessible to AI Assistant"
-                    message={`Your system has ${info?.gpuHealth?.gpuVendor === 'amd' ? 'an AMD' : 'an NVIDIA'} GPU, but the AI Assistant can't access it. AI is running on CPU only, which is significantly slower.`}
-                    dismissible={true}
-                    onDismiss={handleDismissGpuBanner}
-                    buttonProps={{
-                      children: 'Fix: Reinstall AI Assistant',
-                      icon: 'IconRefresh',
-                      variant: 'action',
-                      size: 'sm',
-                      onClick: handleForceReinstallOllama,
-                      loading: reinstalling,
-                      disabled: reinstalling,
-                    }}
-                  />
+                  <GpuPassthroughAlert gpuHealth={info.gpuHealth} assistantName="AI Assistant" />
                 </div>
               )}
               {info?.graphics?.controllers && info.graphics.controllers.length > 0 && (

admin/providers/gpu_passthrough_remediation_provider.ts

@@ -1,14 +1,14 @@
 import logger from '@adonisjs/core/services/logger'
 import type { ApplicationService } from '@adonisjs/core/types'
-import type Docker from 'dockerode'
 
 /**
- * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart.
+ * Detects Ollama's silent CPU fallback after admin / host restart, and auto-remediates
+ * it on NVIDIA.
  *
  * Detects the condition from Ollama's own "inference compute" startup log line:
- * if the NVIDIA container runtime is registered with Docker but Ollama still
- * loaded a CPU-only backend, passthrough is broken and nomad_ollama is recreated.
- * This single signal covers two failure modes:
+ * if Ollama is expected on a GPU (see resolveExpectedGpuVendor) but loaded a CPU-only
+ * backend, passthrough is broken. On NVIDIA, nomad_ollama is recreated.
+ * This single signal covers two NVIDIA failure modes:
  *   1. nomad_ollama was created CPU-only because the nvidia runtime was registered
  *      only AFTER the AI Assistant was first installed (DockerService attaches a
  *      GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at install
@@ -20,16 +20,19 @@ import type Docker from 'dockerode'
  * PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" banner; this
  * provider performs that click automatically on admin boot.
  *
+ * AMD is detect-only. The usual cause there is a missing HSA override for an iGPU
+ * outside ROCm's allowlist (e.g. gfx1103), which a reinstall rebuilds unchanged. The
+ * provider logs whether a reinstall would change the container and leaves the
+ * reinstall to the user; SystemService surfaces the same state as passthrough_failed.
+ *
  * Guards:
- *   - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override
- *     handling in PR #804) and is left to the user.
  *   - One-shot per admin boot. The provider runs once on startup; if the recreate
  *     itself fails the banner remains as a fallback.
  *   - Cooldown: will not auto-reinstall more than once within
  *     AUTO_REMEDIATE_COOLDOWN_MS, to avoid a reinstall loop when the GPU cannot be
  *     accelerated (e.g. an architecture Ollama has no kernels for).
  *   - Opt-out via KV `ai.autoFixGpuPassthrough = false`.
- *   - Skipped entirely when no NVIDIA runtime is registered with Docker.
+ *   - Skipped entirely when nomad_ollama is not expected on a GPU.
  */
 export default class GpuPassthroughRemediationProvider {
   constructor(protected app: ApplicationService) {}
@@ -55,14 +58,6 @@ export default class GpuPassthroughRemediationProvider {
         const docker = new Docker({ socketPath: '/var/run/docker.sock' })
         const dockerInfo = await docker.info()
         const runtimes = dockerInfo.Runtimes || {}
-        const hasNvidiaRuntime = 'nvidia' in runtimes
-
-        if (!hasNvidiaRuntime) {
-          logger.info(
-            '[GpuPassthroughRemediationProvider] No NVIDIA runtime registered — skipping.'
-          )
-          return
-        }
 
         const containers = await docker.listContainers({ all: false })
         const ollama = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
@@ -74,20 +69,43 @@ export default class GpuPassthroughRemediationProvider {
           return
         }
 
+        const container = docker.getContainer(ollama.Id)
+        const inspect = await container.inspect()
+        const {
+          classifyOllamaComputeBackend,
+          diffAmdOllamaConfig,
+          readOllamaStartupLogs,
+          resolveExpectedGpuVendor,
+        } = await import('#services/ollama_compute')
+
+        // The container's own GPU config decides the vendor. The KV AMD type only
+        // matters when no container exists, which the return above rules out.
+        const vendor = resolveExpectedGpuVendor({
+          hasNvidiaRuntime: 'nvidia' in runtimes,
+          amdConfigured: false,
+          container: inspect,
+        })
+
+        if (!vendor) {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] nomad_ollama is not configured for a GPU — skipping.'
+          )
+          return
+        }
+
         // Probe: read Ollama's own "inference compute" startup line from its logs.
         // This is the ground truth for whether Ollama loaded a GPU backend, and it
         // catches every failure mode:
         //   - nomad_ollama created CPU-only because the nvidia runtime was registered
         //     only AFTER first install (Ollama logs library=cpu), and
         //   - DeviceRequests present but the toolkit binding tore after a recreate,
-        //     where Ollama silently falls back to CPU (also library=cpu).
+        //     where Ollama silently falls back to CPU (also library=cpu), and
+        //   - a ROCm container on an AMD GPU that ROCm could not initialize.
         // The previous implementation probed by exec'ing `nvidia-smi` inside the
         // container, but a CPU-only container does not ship nvidia-smi, so the exec
         // returned an error string that the alphabetic-output check mistook for
         // "healthy" — a false negative that suppressed all auto-remediation.
-        const container = docker.getContainer(ollama.Id)
-        const { classifyOllamaComputeBackend } = await import('#services/ollama_compute')
-        const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container))
+        const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container, inspect))
 
         if (backend === 'gpu') {
           logger.info(
@@ -103,6 +121,43 @@ export default class GpuPassthroughRemediationProvider {
           return
         }
 
+        if (vendor === 'amd') {
+          const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
+          if (String(amdEnabledRaw) === 'false') {
+            logger.info(
+              '[GpuPassthroughRemediationProvider] Ollama is on CPU and AMD acceleration is disabled via ai.amdGpuAcceleration — skipping.'
+            )
+            return
+          }
+
+          const hsaOverride = await new DockerService().getAmdHsaOverride()
+          const drift = diffAmdOllamaConfig(inspect, hsaOverride)
+          if (drift.length > 0) {
+            logger.warn(
+              '[GpuPassthroughRemediationProvider] AMD GPU configured but Ollama fell back to CPU. ' +
+                `A reinstall would change the container (${drift.join('; ')}). ` +
+                'Use "Fix: Reinstall AI Assistant" to apply it.'
+            )
+          } else {
+            logger.warn(
+              '[GpuPassthroughRemediationProvider] AMD GPU configured but Ollama fell back to CPU, and a ' +
+                'reinstall would rebuild the same container. If this is an iGPU outside the ROCm ' +
+                'allowlist, use "Fix: Set GFX Override" in Settings (e.g. 11.0.0 for a 780M/gfx1103).'
+            )
+          }
+          return
+        }
+
+        // NVIDIA DeviceRequests without a registered runtime: a reinstall would come up
+        // CPU-only, since DockerService attaches the GPU only when the runtime is present.
+        if (!('nvidia' in runtimes)) {
+          logger.warn(
+            '[GpuPassthroughRemediationProvider] nomad_ollama requests an NVIDIA GPU but no NVIDIA runtime ' +
+              'is registered with Docker. Reinstall the NVIDIA Container Toolkit; skipping auto-reinstall.'
+          )
+          return
+        }
+
         // backend === 'cpu': an NVIDIA runtime is registered but Ollama still runs on
         // CPU. Cooldown: don't auto-reinstall more than once within the window. A
         // reinstall that fails to move Ollama onto the GPU (e.g. an architecture
@@ -151,39 +206,3 @@ export default class GpuPassthroughRemediationProvider {
  * provider. Bounds a reinstall loop when the GPU cannot be accelerated.
  */
 const AUTO_REMEDIATE_COOLDOWN_MS = 60 * 60 * 1000 // 1 hour
-
-/**
- * Read nomad_ollama's logs from its startup window (the five minutes after it
- * booted), where the "inference compute" line is emitted. Mirrors the windowing
- * in SystemService.getOllamaInferenceComputeFromLogs. Returns '' on any error so
- * the caller classifies the backend as 'unknown' rather than throwing.
- */
-async function readOllamaStartupLogs(container: Docker.Container): Promise<string> {
-  try {
-    const inspect = await container.inspect()
-    const startedAtRaw = inspect?.State?.StartedAt
-    const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
-
-    const logsOpts: {
-      stdout: true
-      stderr: true
-      follow: false
-      since?: number
-      until?: number
-      tail?: number
-    } = { stdout: true, stderr: true, follow: false }
-
-    if (Number.isFinite(startedAtMs) && startedAtMs > 0) {
-      const startedAtSec = Math.floor(startedAtMs / 1000)
-      logsOpts.since = startedAtSec
-      logsOpts.until = startedAtSec + 300
-    } else {
-      logsOpts.tail = 500
-    }
-
-    const buf = (await container.logs(logsOpts)) as unknown as Buffer
-    return buf.toString('utf8')
-  } catch {
-    return ''
-  }
-}

admin/tests/unit/ollama_compute.spec.ts

@@ -1,7 +1,14 @@
 import * as assert from 'node:assert/strict'
 import { test } from 'node:test'
 
-import { classifyOllamaComputeBackend } from '../../app/services/ollama_compute.js'
+import {
+  classifyOllamaComputeBackend,
+  diagnoseAmdCpuFallback,
+  diffAmdOllamaConfig,
+  parseAmdGfxTargetFromLogs,
+  parseOllamaGpuFromLogs,
+  resolveExpectedGpuVendor,
+} from '../../app/services/ollama_compute.js'
 
 test('CUDA inference line is classified as gpu', () => {
   const log =
@@ -43,3 +50,183 @@ test('ROCm and Vulkan are gpu backends', () => {
     'gpu'
   )
 })
+
+test('parseOllamaGpuFromLogs takes the GPU line even when the CPU line comes last', () => {
+  const log =
+    'msg="inference compute" id=0 library=ROCm compute=gfx1150 name=ROCm0 description="AMD Radeon 890M" total="16.0 GiB"\n' +
+    'msg="inference compute" id=cpu library=cpu name=cpu'
+  assert.deepEqual(parseOllamaGpuFromLogs(log), {
+    library: 'ROCm',
+    name: 'AMD Radeon 890M',
+    vramMiB: 16384,
+  })
+})
+
+test('parseOllamaGpuFromLogs recognizes Vulkan and falls back to a generic name', () => {
+  assert.deepEqual(parseOllamaGpuFromLogs('msg="inference compute" id=0 library=Vulkan name=V0'), {
+    library: 'Vulkan',
+    name: 'GPU',
+    vramMiB: 0,
+  })
+})
+
+test('parseOllamaGpuFromLogs returns null for a CPU-only log', () => {
+  assert.equal(parseOllamaGpuFromLogs('msg="inference compute" id=cpu library=cpu name=cpu'), null)
+})
+
+const ROCM_CONTAINER = {
+  Config: {
+    Image: 'ollama/ollama:rocm',
+    Env: ['OLLAMA_NO_CLOUD=1', 'HSA_OVERRIDE_GFX_VERSION=11.0.0', 'OLLAMA_IGPU_ENABLE=1'],
+  },
+  HostConfig: {
+    Devices: [{ PathOnHost: '/dev/kfd' }, { PathOnHost: '/dev/dri' }],
+  },
+}
+
+const CPU_CONTAINER = {
+  Config: { Image: 'ollama/ollama:0.18.2', Env: ['OLLAMA_NO_CLOUD=1'] },
+  HostConfig: {},
+}
+
+test('a ROCm container is expected on AMD with no Docker runtime registered (#1344)', () => {
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container: ROCM_CONTAINER }),
+    'amd'
+  )
+})
+
+test('a /dev/kfd device alone marks the container as AMD', () => {
+  const container = {
+    Config: { Image: 'ollama/ollama:0.18.2' },
+    HostConfig: { Devices: [{ PathOnHost: '/dev/kfd' }] },
+  }
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }),
+    'amd'
+  )
+})
+
+test('a versioned ROCm tag marks the container as AMD, a rocm repo path does not', () => {
+  const tagged = { Config: { Image: 'ollama/ollama:0.12.0-rocm' }, HostConfig: {} }
+  const repoOnly = { Config: { Image: 'rocm/ollama:latest' }, HostConfig: {} }
+  const input = { hasNvidiaRuntime: false, amdConfigured: false }
+  assert.equal(resolveExpectedGpuVendor({ ...input, container: tagged }), 'amd')
+  assert.equal(resolveExpectedGpuVendor({ ...input, container: repoOnly }), null)
+})
+
+test('/dev/dri alone is not treated as AMD', () => {
+  const container = {
+    Config: { Image: 'ollama/ollama:0.18.2' },
+    HostConfig: { Devices: [{ PathOnHost: '/dev/dri' }] },
+  }
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }),
+    null
+  )
+})
+
+test('an nvidia DeviceRequest is expected on NVIDIA even without the runtime', () => {
+  const container = {
+    Config: { Image: 'ollama/ollama:0.18.2' },
+    HostConfig: { DeviceRequests: [{ Driver: 'nvidia' }] },
+  }
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container }),
+    'nvidia'
+  )
+})
+
+test('a CPU container is expected on NVIDIA when the runtime was registered after install', () => {
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: true, amdConfigured: false, container: CPU_CONTAINER }),
+    'nvidia'
+  )
+})
+
+test('a CPU container on an AMD box is trusted as deliberate (#1232 fallback)', () => {
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: true, container: CPU_CONTAINER }),
+    null
+  )
+})
+
+test('the AMD GPU type counts when no container exists', () => {
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: true, container: null }),
+    'amd'
+  )
+  assert.equal(
+    resolveExpectedGpuVendor({ hasNvidiaRuntime: false, amdConfigured: false, container: null }),
+    null
+  )
+})
+
+test('diffAmdOllamaConfig is empty when a reinstall would rebuild the same container', () => {
+  assert.deepEqual(diffAmdOllamaConfig(ROCM_CONTAINER, '11.0.0'), [])
+})
+
+test('diffAmdOllamaConfig reports an HSA override the container is missing', () => {
+  const container = {
+    ...ROCM_CONTAINER,
+    Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] },
+  }
+  assert.deepEqual(diffAmdOllamaConfig(container, '11.0.0'), [
+    'HSA_OVERRIDE_GFX_VERSION is unset, expected 11.0.0',
+  ])
+})
+
+test('diffAmdOllamaConfig reports a stale override when none is wanted', () => {
+  assert.deepEqual(diffAmdOllamaConfig(ROCM_CONTAINER, null), [
+    'HSA_OVERRIDE_GFX_VERSION is 11.0.0, expected unset',
+  ])
+})
+
+test('diffAmdOllamaConfig reports missing image, devices and iGPU flag', () => {
+  assert.deepEqual(diffAmdOllamaConfig(CPU_CONTAINER, null), [
+    'image is ollama/ollama:0.18.2, not the ROCm build',
+    '/dev/kfd is not passed through',
+    '/dev/dri is not passed through',
+    'OLLAMA_IGPU_ENABLE=1 is not set',
+  ])
+})
+
+const UNSUPPORTED_GFX1103_LOG =
+  'level=WARN source=amd_linux.go msg="amdgpu is not supported (supported types:[gfx1030 gfx1100])" gpu_type=gfx1103 gpu=0 library=/usr/lib/ollama/rocm\n' +
+  'msg="inference compute" id=cpu library=cpu name=cpu'
+
+test('parseAmdGfxTargetFromLogs reads the gfx target Ollama rejected', () => {
+  assert.equal(parseAmdGfxTargetFromLogs(UNSUPPORTED_GFX1103_LOG), 'gfx1103')
+  assert.equal(parseAmdGfxTargetFromLogs('msg="inference compute" id=cpu library=cpu'), null)
+})
+
+test('diagnoseAmdCpuFallback on the #1344 box: matching config, override needed', () => {
+  const container = {
+    ...ROCM_CONTAINER,
+    Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] },
+  }
+  // No gfx marker and no KV override, so a reinstall resolves no override either.
+  assert.deepEqual(diagnoseAmdCpuFallback(container, UNSUPPORTED_GFX1103_LOG, null), {
+    amdReinstallWouldChange: false,
+    amdGfxTarget: 'gfx1103',
+    suggestedHsaOverride: '11.0.0',
+  })
+})
+
+test('diagnoseAmdCpuFallback flags a reinstall when a newly set override is missing', () => {
+  const container = {
+    ...ROCM_CONTAINER,
+    Config: { ...ROCM_CONTAINER.Config, Env: ['OLLAMA_NO_CLOUD=1', 'OLLAMA_IGPU_ENABLE=1'] },
+  }
+  assert.equal(
+    diagnoseAmdCpuFallback(container, UNSUPPORTED_GFX1103_LOG, '11.0.0').amdReinstallWouldChange,
+    true
+  )
+})
+
+test('diagnoseAmdCpuFallback reports the current override and omits unknown fields', () => {
+  assert.deepEqual(diagnoseAmdCpuFallback(ROCM_CONTAINER, '', '11.0.0'), {
+    amdReinstallWouldChange: false,
+    currentHsaOverride: '11.0.0',
+  })
+})

admin/tests/unit/settings_amd_hsa_override.spec.ts

@@ -0,0 +1,16 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { validateSettingValue } from '../../app/validators/settings.js'
+
+test('ai.amdHsaOverride accepts a gfx version, none, or empty', () => {
+  for (const value of ['11.0.0', '10.3.0', ' 11.0.0 ', 'none', 'NONE', '', null, undefined]) {
+    assert.equal(validateSettingValue('ai.amdHsaOverride', value), null, String(value))
+  }
+})
+
+test('ai.amdHsaOverride rejects anything that is not a version', () => {
+  for (const value of ['11', '11.0', 'gfx1103', '11.0.0; rm -rf /', '11.0.0\nFOO=1', 11, true]) {
+    assert.notEqual(validateSettingValue('ai.amdHsaOverride', value), null, String(value))
+  }
+})

admin/types/system.ts

@@ -6,6 +6,18 @@ export type GpuHealthStatus = {
   hasRocmRuntime: boolean
   ollamaGpuAccessible: boolean
   gpuVendor?: 'nvidia' | 'amd'
+  /**
+   * AMD passthrough_failed only. Whether a reinstall would change the running
+   * container. When false, a reinstall rebuilds the same CPU-bound container and
+   * the fix is an HSA override (ai.amdHsaOverride).
+   */
+  amdReinstallWouldChange?: boolean
+  /** gfx target Ollama reported for an unsupported AMD GPU, e.g. 'gfx1103'. */
+  amdGfxTarget?: string
+  /** HSA_OVERRIDE_GFX_VERSION the running container has, if any. */
+  currentHsaOverride?: string
+  /** Override the gfx mapping recommends for amdGfxTarget, if it knows one. */
+  suggestedHsaOverride?: string
 }
 
 export type SystemInformationResponse = {