fix(Docker): improve container state management

This commit is contained in:
Jake Turner 2026-01-16 05:46:55 +00:00 committed by Jake Turner
parent e1b1b187b0
commit 003902b84b
3 changed files with 27 additions and 37 deletions

View File

@ -2,7 +2,6 @@ import Service from '#models/service'
import Docker from 'dockerode' import Docker from 'dockerode'
import logger from '@adonisjs/core/services/logger' import logger from '@adonisjs/core/services/logger'
import { inject } from '@adonisjs/core' import { inject } from '@adonisjs/core'
import { ServiceStatus } from '../../types/services.js'
import transmit from '@adonisjs/transmit/services/main' import transmit from '@adonisjs/transmit/services/main'
import { doResumableDownloadWithRetry } from '../utils/downloads.js' import { doResumableDownloadWithRetry } from '../utils/downloads.js'
import { join } from 'path' import { join } from 'path'
@ -92,18 +91,16 @@ export class DockerService {
} }
} }
/**
* Fetches the status of all Docker containers related to Nomad services. (those prefixed with 'nomad_')
*/
async getServicesStatus(): Promise< async getServicesStatus(): Promise<
{ {
service_name: string service_name: string
status: ServiceStatus status: string
}[] }[]
> { > {
try { try {
const services = await Service.query().where('installed', true)
if (!services || services.length === 0) {
return []
}
const containers = await this.docker.listContainers({ all: true }) const containers = await this.docker.listContainers({ all: true })
const containerMap = new Map<string, Docker.ContainerInfo>() const containerMap = new Map<string, Docker.ContainerInfo>()
containers.forEach((container) => { containers.forEach((container) => {
@ -113,22 +110,9 @@ export class DockerService {
} }
}) })
const getStatus = (state: string): ServiceStatus => {
switch (state) {
case 'running':
return 'running'
case 'exited':
case 'created':
case 'paused':
return 'stopped'
default:
return 'unknown'
}
}
return Array.from(containerMap.entries()).map(([name, container]) => ({ return Array.from(containerMap.entries()).map(([name, container]) => ({
service_name: name, service_name: name,
status: getStatus(container.State), status: container.State,
})) }))
} catch (error) { } catch (error) {
console.error(`Error fetching services status: ${error.message}`) console.error(`Error fetching services status: ${error.message}`)
@ -189,13 +173,12 @@ export class DockerService {
// } // }
const containerConfig = this._parseContainerConfig(service.container_config) const containerConfig = this._parseContainerConfig(service.container_config)
// Execute installation asynchronously and handle cleanup // Execute installation asynchronously and handle cleanup
this._createContainer(service, containerConfig) this._createContainer(service, containerConfig).catch(async (error) => {
.catch(async (error) => { logger.error(`Installation failed for ${serviceName}: ${error.message}`)
logger.error(`Installation failed for ${serviceName}: ${error.message}`) await this._cleanupFailedInstallation(serviceName)
await this._cleanupFailedInstallation(serviceName) })
})
return { return {
success: true, success: true,
@ -416,7 +399,9 @@ export class DockerService {
this.activeInstallations.delete(serviceName) this.activeInstallations.delete(serviceName)
logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`) logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
} catch (error) { } catch (error) {
logger.error(`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`) logger.error(
`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`
)
} }
} }

View File

@ -223,8 +223,9 @@ export class SystemService {
/** /**
* Checks the current state of Docker containers against the database records and updates the database accordingly. * Checks the current state of Docker containers against the database records and updates the database accordingly.
* It will mark services as not installed if their corresponding containers are not running, and can also handle cleanup of any orphaned records. * It will mark services as not installed if their corresponding containers do not exist, regardless of their running state.
* Handles cases where a container might have been manually removed or is in an unexpected state, ensuring the database reflects the actual state of containers. * Handles cases where a container might have been manually removed, ensuring the database reflects the actual existence of containers.
* Containers that exist but are stopped, paused, or restarting will still be considered installed.
*/ */
private async _syncContainersWithDatabase() { private async _syncContainersWithDatabase() {
try { try {
@ -232,20 +233,25 @@ export class SystemService {
const serviceStatusList = await this.dockerService.getServicesStatus() const serviceStatusList = await this.dockerService.getServicesStatus()
for (const service of allServices) { for (const service of allServices) {
const status = serviceStatusList.find((s) => s.service_name === service.service_name) const containerExists = serviceStatusList.find(
(s) => s.service_name === service.service_name
)
if (service.installed) { if (service.installed) {
if (!status || status.status !== 'running') { // If marked as installed but container doesn't exist, mark as not installed
if (!containerExists) {
logger.warn( logger.warn(
`Service ${service.service_name} is marked as installed but container is not running. Marking as not installed.` `Service ${service.service_name} is marked as installed but container does not exist. Marking as not installed.`
) )
service.installed = false service.installed = false
service.installation_status = 'idle' service.installation_status = 'idle'
await service.save() await service.save()
} }
} else { } else {
if (status && status.status === 'running') { // If marked as not installed but container exists (any state), mark as installed
if (containerExists) {
logger.warn( logger.warn(
`Service ${service.service_name} is marked as not installed but container is running. Marking as installed.` `Service ${service.service_name} is marked as not installed but container exists. Marking as installed.`
) )
service.installed = true service.installed = true
service.installation_status = 'idle' service.installation_status = 'idle'

View File

@ -1,6 +1,5 @@
import Service from '#models/service' import Service from '#models/service'
export type ServiceStatus = 'unknown' | 'running' | 'stopped'
export type ServiceSlim = Pick< export type ServiceSlim = Pick<
Service, Service,
| 'id' | 'id'
@ -11,4 +10,4 @@ export type ServiceSlim = Pick<
| 'friendly_name' | 'friendly_name'
| 'description' | 'description'
| 'icon' | 'icon'
> & { status?: ServiceStatus } > & { status?: string }