From 9b90fcf9e875d922a05fa19ac2c6b5e304e5f39b Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Wed, 26 Aug 2026 10:41:33 -0700 Subject: [PATCH] fix(gpu): a missing /dev/kfd blocks the AI Assistant install entirely When an AMD GPU is detected, the install switches to the ROCm image and passes /dev/kfd and /dev/dri into the container. _discoverAMDDevices() returns both unconditionally, with no check that they exist. Detection reads PCI data, which says nothing about whether the ROCm kernel interface was ever loaded, so on a pre-ROCm card or a host without amdgpu the container is rejected: (HTTP code 500) server error - error gathering device information while adding custom device "/dev/kfd": no such file or directory and the install dies. There is an opt-out, ai.amdGpuAcceleration, but it is a backend-only KV key with no settings toggle, no route and no UI, so an affected user cannot install the AI Assistant at all through the interface even though CPU-only would work fine on their machine. Fall back to the CPU image instead, with a message explaining why. Two things about the shape of this fix, both measured rather than assumed: - It hooks start(), not createContainer(). The daemon accepts a container whose --device path does not exist and only resolves devices on start. Verified against Docker 29.6.2 over the socket: create returns 201, start returns 500 with the message above. - It retries rather than pre-checking. The admin runs in its own container, so stat()ing /dev/kfd describes the admin's namespace, not the host's. Verified on NOMAD6, a working ROCm box (Ollama running ollama/ollama:rocm with both devices attached): /dev/kfd is PRESENT on the host and ABSENT inside the admin container. A pre-check would therefore have stripped GPU passthrough from a machine where AMD acceleration works. The matcher keys on the device-gathering phrase rather than a bare "no such file or directory", which Docker also emits for missing bind-mount sources; falling back to CPU on one of those would hide a real failure behind a degraded install. Checked against all three formats the daemon/dockerode produce, and against a port-conflict message, to confirm it discriminates. The success path is unchanged: the create payload is built by one helper used by both attempts, and the fallback only runs after a start() failure that matches. Fixes #1232 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LtGWCVMPYn6qPWVYiycK74 --- admin/app/services/docker_service.ts | 98 ++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 8849b872..e5ebdba8 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -850,8 +850,10 @@ export class DockerService { 'creating', `Creating Docker container for service ${service.service_name}...` ) - const container = await this.docker.createContainer({ - Image: finalImage, + // Built once and reused, so the AMD fallback below cannot drift from the + // real payload as this config grows. + const buildCreateOptions = (image: string, hostConfig: any, env: string[]) => ({ + Image: image, name: service.service_name, Labels: { ...(containerConfig?.Labels ?? {}), @@ -859,10 +861,10 @@ export class DockerService { 'io.project-nomad.managed': 'true', }, ...(containerConfig?.User && { User: containerConfig.User }), - HostConfig: gpuHostConfig, + HostConfig: hostConfig, ...(containerConfig?.WorkingDir && { WorkingDir: containerConfig.WorkingDir }), ...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }), - Env: [...(containerConfig?.Env ?? []), ...ollamaEnv, ...appEnv], + Env: env, ...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}), // Ensure container is attached to the Nomad docker network in production ...(process.env.NODE_ENV === 'production' && { @@ -874,12 +876,76 @@ export class DockerService { }), }) + let container = await this.docker.createContainer( + buildCreateOptions(finalImage, gpuHostConfig, [ + ...(containerConfig?.Env ?? []), + ...ollamaEnv, + ...appEnv, + ]) + ) + this._broadcast( service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...` ) - await container.start() + try { + await container.start() + } catch (error: any) { + // An AMD GPU whose ROCm device nodes are absent must not block the install + // outright (#1232). GPU detection reads PCI data, so "an AMD GPU is present" + // and "ROCm is usable" are different questions, and only the daemon can + // answer the second. CPU-only Ollama works fine on these machines. + // + // This has to sit on start(), not createContainer(): the daemon accepts a + // container whose --device path does not exist (create returns 201) and only + // resolves devices when the container starts. Measured on Docker 29.6.2. + // + // Retrying rather than pre-checking is also deliberate. The admin runs in its + // own container, so stat()ing /dev/kfd here describes the admin's namespace, + // not the host's. Verified on NOMAD6, a working ROCm box: /dev/kfd is present + // on the host and absent inside the admin container, so a pre-check would + // strip GPU passthrough from a machine where it works. + if (!amdGpuConfigured || !this._isMissingDeviceError(error)) throw error + + logger.warn( + `[DockerService] AMD device passthrough rejected by the daemon (${error?.message}); ` + + `recreating ${service.service_name} CPU-only on ${service.container_image}` + ) + this._broadcast( + service.service_name, + 'gpu-config', + `AMD GPU detected, but this system has no usable ROCm device (/dev/kfd is missing, ` + + `which means the amdgpu/ROCm kernel driver is not loaded or does not support this GPU). ` + + `Installing CPU-only instead so the AI Assistant still works.` + ) + + // The created container carries the rejected device config, so it cannot be + // started as-is and has to go before the name can be reused. + await container.remove({ force: true }).catch((removeError: any) => { + logger.warn(`[DockerService] Could not remove the failed container: ${removeError?.message}`) + }) + + const { Devices, ...cpuHostConfig } = gpuHostConfig as any + // Drop the ROCm-only env along with the devices. HSA_OVERRIDE_GFX_VERSION and + // OLLAMA_IGPU_ENABLE mean nothing to the CPU build, and leaving them behind is + // misleading to anyone who later reads `docker inspect`. + const cpuOllamaEnv = ollamaEnv.filter( + (e) => !e.startsWith('HSA_OVERRIDE_GFX_VERSION=') && !e.startsWith('OLLAMA_IGPU_ENABLE=') + ) + amdGpuConfigured = false + // service.container_image is the CPU tag, already pulled earlier in this + // function before the AMD branch overrode finalImage to :rocm. + finalImage = service.container_image + container = await this.docker.createContainer( + buildCreateOptions(finalImage, cpuHostConfig, [ + ...(containerConfig?.Env ?? []), + ...cpuOllamaEnv, + ...appEnv, + ]) + ) + await container.start() + } this._broadcast( service.service_name, @@ -1581,6 +1647,28 @@ export class DockerService { ] } + /** + * Whether a container-create failure was the daemon refusing a `--device` path + * that does not exist on the host, e.g. + * + * (HTTP code 500) server error - error gathering device information while + * adding custom device "/dev/kfd": no such file or directory + * + * GPU *detection* reads PCI data, which says nothing about whether the ROCm + * kernel interface was ever loaded. An AMD card with no `/dev/kfd` is a + * perfectly normal machine — old pre-ROCm silicon, or amdgpu not loaded — and + * on those the create fails outright and the whole install dies (#1232). + * + * Deliberately keyed on the device-gathering phrase rather than a bare "no + * such file or directory", which Docker also emits for missing bind-mount + * sources and image layers. Falling back to CPU on one of those would hide a + * real failure behind a degraded install. + */ + private _isMissingDeviceError(error: any): boolean { + const raw: string = error?.message ?? String(error) + return /error gathering device information while adding custom device/i.test(raw) + } + /** * Update a service container to a new image version while preserving volumes and data. * Includes automatic rollback if the new container fails health checks.