mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-08 09:46:15 +02:00
fix(Docker): improve container state management
This commit is contained in:
parent
e1b1b187b0
commit
003902b84b
|
|
@ -2,7 +2,6 @@ import Service from '#models/service'
|
||||||
import Docker from 'dockerode'
|
import Docker from 'dockerode'
|
||||||
import logger from '@adonisjs/core/services/logger'
|
import logger from '@adonisjs/core/services/logger'
|
||||||
import { inject } from '@adonisjs/core'
|
import { inject } from '@adonisjs/core'
|
||||||
import { ServiceStatus } from '../../types/services.js'
|
|
||||||
import transmit from '@adonisjs/transmit/services/main'
|
import transmit from '@adonisjs/transmit/services/main'
|
||||||
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
|
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
|
|
@ -92,18 +91,16 @@ export class DockerService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches the status of all Docker containers related to Nomad services. (those prefixed with 'nomad_')
|
||||||
|
*/
|
||||||
async getServicesStatus(): Promise<
|
async getServicesStatus(): Promise<
|
||||||
{
|
{
|
||||||
service_name: string
|
service_name: string
|
||||||
status: ServiceStatus
|
status: string
|
||||||
}[]
|
}[]
|
||||||
> {
|
> {
|
||||||
try {
|
try {
|
||||||
const services = await Service.query().where('installed', true)
|
|
||||||
if (!services || services.length === 0) {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
const containers = await this.docker.listContainers({ all: true })
|
const containers = await this.docker.listContainers({ all: true })
|
||||||
const containerMap = new Map<string, Docker.ContainerInfo>()
|
const containerMap = new Map<string, Docker.ContainerInfo>()
|
||||||
containers.forEach((container) => {
|
containers.forEach((container) => {
|
||||||
|
|
@ -113,22 +110,9 @@ export class DockerService {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
const getStatus = (state: string): ServiceStatus => {
|
|
||||||
switch (state) {
|
|
||||||
case 'running':
|
|
||||||
return 'running'
|
|
||||||
case 'exited':
|
|
||||||
case 'created':
|
|
||||||
case 'paused':
|
|
||||||
return 'stopped'
|
|
||||||
default:
|
|
||||||
return 'unknown'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(containerMap.entries()).map(([name, container]) => ({
|
return Array.from(containerMap.entries()).map(([name, container]) => ({
|
||||||
service_name: name,
|
service_name: name,
|
||||||
status: getStatus(container.State),
|
status: container.State,
|
||||||
}))
|
}))
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error fetching services status: ${error.message}`)
|
console.error(`Error fetching services status: ${error.message}`)
|
||||||
|
|
@ -191,11 +175,10 @@ export class DockerService {
|
||||||
const containerConfig = this._parseContainerConfig(service.container_config)
|
const containerConfig = this._parseContainerConfig(service.container_config)
|
||||||
|
|
||||||
// Execute installation asynchronously and handle cleanup
|
// Execute installation asynchronously and handle cleanup
|
||||||
this._createContainer(service, containerConfig)
|
this._createContainer(service, containerConfig).catch(async (error) => {
|
||||||
.catch(async (error) => {
|
logger.error(`Installation failed for ${serviceName}: ${error.message}`)
|
||||||
logger.error(`Installation failed for ${serviceName}: ${error.message}`)
|
await this._cleanupFailedInstallation(serviceName)
|
||||||
await this._cleanupFailedInstallation(serviceName)
|
})
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
|
|
@ -416,7 +399,9 @@ export class DockerService {
|
||||||
this.activeInstallations.delete(serviceName)
|
this.activeInstallations.delete(serviceName)
|
||||||
logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
|
logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`)
|
logger.error(
|
||||||
|
`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -223,8 +223,9 @@ export class SystemService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Checks the current state of Docker containers against the database records and updates the database accordingly.
|
* Checks the current state of Docker containers against the database records and updates the database accordingly.
|
||||||
* It will mark services as not installed if their corresponding containers are not running, and can also handle cleanup of any orphaned records.
|
* It will mark services as not installed if their corresponding containers do not exist, regardless of their running state.
|
||||||
* Handles cases where a container might have been manually removed or is in an unexpected state, ensuring the database reflects the actual state of containers.
|
* Handles cases where a container might have been manually removed, ensuring the database reflects the actual existence of containers.
|
||||||
|
* Containers that exist but are stopped, paused, or restarting will still be considered installed.
|
||||||
*/
|
*/
|
||||||
private async _syncContainersWithDatabase() {
|
private async _syncContainersWithDatabase() {
|
||||||
try {
|
try {
|
||||||
|
|
@ -232,20 +233,25 @@ export class SystemService {
|
||||||
const serviceStatusList = await this.dockerService.getServicesStatus()
|
const serviceStatusList = await this.dockerService.getServicesStatus()
|
||||||
|
|
||||||
for (const service of allServices) {
|
for (const service of allServices) {
|
||||||
const status = serviceStatusList.find((s) => s.service_name === service.service_name)
|
const containerExists = serviceStatusList.find(
|
||||||
|
(s) => s.service_name === service.service_name
|
||||||
|
)
|
||||||
|
|
||||||
if (service.installed) {
|
if (service.installed) {
|
||||||
if (!status || status.status !== 'running') {
|
// If marked as installed but container doesn't exist, mark as not installed
|
||||||
|
if (!containerExists) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`Service ${service.service_name} is marked as installed but container is not running. Marking as not installed.`
|
`Service ${service.service_name} is marked as installed but container does not exist. Marking as not installed.`
|
||||||
)
|
)
|
||||||
service.installed = false
|
service.installed = false
|
||||||
service.installation_status = 'idle'
|
service.installation_status = 'idle'
|
||||||
await service.save()
|
await service.save()
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (status && status.status === 'running') {
|
// If marked as not installed but container exists (any state), mark as installed
|
||||||
|
if (containerExists) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`Service ${service.service_name} is marked as not installed but container is running. Marking as installed.`
|
`Service ${service.service_name} is marked as not installed but container exists. Marking as installed.`
|
||||||
)
|
)
|
||||||
service.installed = true
|
service.installed = true
|
||||||
service.installation_status = 'idle'
|
service.installation_status = 'idle'
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import Service from '#models/service'
|
import Service from '#models/service'
|
||||||
|
|
||||||
export type ServiceStatus = 'unknown' | 'running' | 'stopped'
|
|
||||||
export type ServiceSlim = Pick<
|
export type ServiceSlim = Pick<
|
||||||
Service,
|
Service,
|
||||||
| 'id'
|
| 'id'
|
||||||
|
|
@ -11,4 +10,4 @@ export type ServiceSlim = Pick<
|
||||||
| 'friendly_name'
|
| 'friendly_name'
|
||||||
| 'description'
|
| 'description'
|
||||||
| 'icon'
|
| 'icon'
|
||||||
> & { status?: ServiceStatus }
|
> & { status?: string }
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user