diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 9cf1564..a334402 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -1,71 +1,65 @@ -import Service from "#models/service"; -import Docker from "dockerode"; +import Service from '#models/service' +import Docker from 'dockerode' import drive from '@adonisjs/drive/services/main' -import axios from 'axios'; +import axios from 'axios' import logger from '@adonisjs/core/services/logger' -import { inject } from "@adonisjs/core"; -import { ServiceStatus } from "../../types/services.js"; -import transmit from "@adonisjs/transmit/services/main"; -import { Readable } from "stream"; -import { chmodRecursive, chownRecursive } from "../../util/files.js"; -import fs from 'fs' +import { inject } from '@adonisjs/core' +import { ServiceStatus } from '../../types/services.js' +import transmit from '@adonisjs/transmit/services/main' @inject() export class DockerService { - private docker: Docker; - public static KIWIX_SERVICE_NAME = 'nomad_kiwix_serve'; - public static OPENSTREETMAP_SERVICE_NAME = 'nomad_openstreetmap'; - public static OPENSTREETMAP_IMPORT_SERVICE_NAME = 'nomad_openstreetmap_import'; - public static OLLAMA_SERVICE_NAME = 'nomad_ollama'; - public static OPEN_WEBUI_SERVICE_NAME = 'nomad_open_webui'; - public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef'; - public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes'; - public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri'; - public static NOMAD_STORAGE_ABS_PATH = '/opt/project-nomad/storage'; + private docker: Docker + public static KIWIX_SERVICE_NAME = 'nomad_kiwix_serve' + public static OLLAMA_SERVICE_NAME = 'nomad_ollama' + public static OPEN_WEBUI_SERVICE_NAME = 'nomad_open_webui' + public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef' + public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes' + public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri' + public static NOMAD_STORAGE_ABS_PATH = '/opt/project-nomad/storage' constructor() { - this.docker = new Docker({ socketPath: '/var/run/docker.sock' }); + this.docker = new Docker({ socketPath: '/var/run/docker.sock' }) } - async affectContainer(serviceName: string, action: 'start' | 'stop' | 'restart'): Promise<{ success: boolean; message: string }> { + async affectContainer( + serviceName: string, + action: 'start' | 'stop' | 'restart' + ): Promise<{ success: boolean; message: string }> { try { - const service = await Service.query().where('service_name', serviceName).first(); + const service = await Service.query().where('service_name', serviceName).first() if (!service || !service.installed) { return { success: false, message: `Service ${serviceName} not found or not installed`, - }; + } } - const containers = await this.docker.listContainers({ all: true }); - const container = containers.find(c => c.Names.includes(`/${serviceName}`)); + const containers = await this.docker.listContainers({ all: true }) + const container = containers.find((c) => c.Names.includes(`/${serviceName}`)) if (!container) { return { success: false, message: `Container for service ${serviceName} not found`, - }; + } } - const dockerContainer = this.docker.getContainer(container.Id); + const dockerContainer = this.docker.getContainer(container.Id) if (action === 'stop') { - await dockerContainer.stop(); + await dockerContainer.stop() return { success: true, message: `Service ${serviceName} stopped successfully`, - }; + } } if (action === 'restart') { - await dockerContainer.restart(); - - if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { - await this._fixOSMPermissions(); - } + await dockerContainer.restart() return { success: true, message: `Service ${serviceName} restarted successfully`, - }; + } } if (action === 'start') { @@ -73,19 +67,15 @@ export class DockerService { return { success: true, message: `Service ${serviceName} is already running`, - }; + } } - await dockerContainer.start(); - - if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { - await this._fixOSMPermissions(); - } + await dockerContainer.start() return { success: true, message: `Service ${serviceName} started successfully`, - }; + } } return { @@ -93,71 +83,74 @@ export class DockerService { message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`, } } catch (error) { - logger.error(`Error starting service ${serviceName}: ${error.message}`); + logger.error(`Error starting service ${serviceName}: ${error.message}`) return { success: false, message: `Failed to start service ${serviceName}: ${error.message}`, - }; + } } } - async getServicesStatus(): Promise<{ - service_name: string; - status: ServiceStatus; - }[]> { + async getServicesStatus(): Promise< + { + service_name: string + status: ServiceStatus + }[] + > { try { - const services = await Service.query().where('installed', true); + const services = await Service.query().where('installed', true) if (!services || services.length === 0) { - return []; + return [] } - const containers = await this.docker.listContainers({ all: true }); - const containerMap = new Map(); - containers.forEach(container => { - const name = container.Names[0].replace('/', ''); + const containers = await this.docker.listContainers({ all: true }) + const containerMap = new Map() + containers.forEach((container) => { + const name = container.Names[0].replace('/', '') if (name.startsWith('nomad_')) { - containerMap.set(name, container); + containerMap.set(name, container) } - }); + }) const getStatus = (state: string): ServiceStatus => { switch (state) { case 'running': - return 'running'; + return 'running' case 'exited': case 'created': case 'paused': - return 'stopped'; + return 'stopped' default: - return 'unknown'; + return 'unknown' } - }; - + } return Array.from(containerMap.entries()).map(([name, container]) => ({ service_name: name, status: getStatus(container.State), - })); + })) } catch (error) { - console.error(`Error fetching services status: ${error.message}`); - return []; + console.error(`Error fetching services status: ${error.message}`) + return [] } } - async createContainerPreflight(serviceName: string): Promise<{ success: boolean; message: string }> { - const service = await Service.query().where('service_name', serviceName).first(); + async createContainerPreflight( + serviceName: string + ): Promise<{ success: boolean; message: string }> { + const service = await Service.query().where('service_name', serviceName).first() if (!service) { return { success: false, message: `Service ${serviceName} not found`, - }; + } } if (service.installed) { return { success: false, message: `Service ${serviceName} is already installed`, - }; + } } // Check if a service wasn't marked as installed but has an existing container @@ -173,8 +166,8 @@ export class DockerService { // } // } - const containerConfig = this._parseContainerConfig(service.container_config); - this._createContainer(service, containerConfig); // Don't await this method - we will use server-sent events to notify the client of progress + const containerConfig = this._parseContainerConfig(service.container_config) + this._createContainer(service, containerConfig) // Don't await this method - we will use server-sent events to notify the client of progress return { success: true, @@ -187,48 +180,75 @@ export class DockerService { * NOTE: This method should not be called directly. Instead, use `createContainerPreflight` to check prerequisites first * and return an HTTP response to the client, if needed. This method will then transmit server-sent events to the client * to notify them of the progress. - * @param serviceName - * @returns + * @param serviceName + * @returns */ - async _createContainer(service: Service & { dependencies?: Service[] }, containerConfig: any): Promise { + async _createContainer( + service: Service & { dependencies?: Service[] }, + containerConfig: any + ): Promise { try { - this._broadcast(service.service_name, 'initializing', ''); + this._broadcast(service.service_name, 'initializing', '') - let dependencies = []; + let dependencies = [] if (service.depends_on) { - const dependency = await Service.query().where('service_name', service.depends_on).first(); + const dependency = await Service.query().where('service_name', service.depends_on).first() if (dependency) { - dependencies.push(dependency); + dependencies.push(dependency) } } // First, check if the service has any dependencies that need to be installed first if (dependencies && dependencies.length > 0) { - this._broadcast(service.service_name, 'checking-dependencies', `Checking dependencies for service ${service.service_name}...`); + this._broadcast( + service.service_name, + 'checking-dependencies', + `Checking dependencies for service ${service.service_name}...` + ) for (const dependency of dependencies) { if (!dependency.installed) { - this._broadcast(service.service_name, 'dependency-not-installed', `Dependency service ${dependency.service_name} is not installed. Installing it first...`); - await this._createContainer(dependency, this._parseContainerConfig(dependency.container_config)); + this._broadcast( + service.service_name, + 'dependency-not-installed', + `Dependency service ${dependency.service_name} is not installed. Installing it first...` + ) + await this._createContainer( + dependency, + this._parseContainerConfig(dependency.container_config) + ) } else { - this._broadcast(service.service_name, 'dependency-installed', `Dependency service ${dependency.service_name} is already installed.`); + this._broadcast( + service.service_name, + 'dependency-installed', + `Dependency service ${dependency.service_name} is already installed.` + ) } } } // Start pulling the Docker image and wait for it to complete - const pullStream = await this.docker.pull(service.container_image); - this._broadcast(service.service_name, 'pulling', `Pulling Docker image ${service.container_image}...`); - await new Promise(res => this.docker.modem.followProgress(pullStream, res)); + const pullStream = await this.docker.pull(service.container_image) + this._broadcast( + service.service_name, + 'pulling', + `Pulling Docker image ${service.container_image}...` + ) + await new Promise((res) => this.docker.modem.followProgress(pullStream, res)) if (service.service_name === DockerService.KIWIX_SERVICE_NAME) { - await this._runPreinstallActions__KiwixServe(); - this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`); - } else if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { - await this._runPreinstallActions__OpenStreetMap(service.container_image, containerConfig); - this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for OpenStreetMap completed successfully.`); + await this._runPreinstallActions__KiwixServe() + this._broadcast( + service.service_name, + 'preinstall-complete', + `Pre-install actions for Kiwix Serve completed successfully.` + ) } - this._broadcast(service.service_name, 'creating', `Creating Docker container for service ${service.service_name}...`); + this._broadcast( + service.service_name, + 'creating', + `Creating Docker container for service ${service.service_name}...` + ) const container = await this.docker.createContainer({ Image: service.container_image, name: service.service_name, @@ -238,53 +258,69 @@ export class DockerService { ...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }), ...(containerConfig?.Env && { Env: containerConfig.Env }), ...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}), - }); + }) - this._broadcast(service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...`); - await container.start(); + this._broadcast( + service.service_name, + 'starting', + `Starting Docker container for service ${service.service_name}...` + ) + await container.start() - // Ensure OSM directories have correct permissions after install+start - if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { - await this._fixOSMPermissions(); - } + this._broadcast( + service.service_name, + 'finalizing', + `Finalizing installation of service ${service.service_name}...` + ) + service.installed = true + await service.save() - this._broadcast(service.service_name, 'finalizing', `Finalizing installation of service ${service.service_name}...`); - service.installed = true; - await service.save(); - - this._broadcast(service.service_name, 'completed', `Service ${service.service_name} installation completed successfully.`); + this._broadcast( + service.service_name, + 'completed', + `Service ${service.service_name} installation completed successfully.` + ) } catch (error) { - this._broadcast(service.service_name, 'error', `Error installing service ${service.service_name}: ${error.message}`); - throw new Error(`Failed to install service ${service.service_name}: ${error.message}`); + this._broadcast( + service.service_name, + 'error', + `Error installing service ${service.service_name}: ${error.message}` + ) + throw new Error(`Failed to install service ${service.service_name}: ${error.message}`) } } async _checkIfServiceContainerExists(serviceName: string): Promise { try { - const containers = await this.docker.listContainers({ all: true }); - return containers.some(container => container.Names.includes(`/${serviceName}`)); + const containers = await this.docker.listContainers({ all: true }) + return containers.some((container) => container.Names.includes(`/${serviceName}`)) } catch (error) { - logger.error(`Error checking if service container exists: ${error.message}`); - return false; + logger.error(`Error checking if service container exists: ${error.message}`) + return false } } - async _removeServiceContainer(serviceName: string): Promise<{ success: boolean; message: string }> { + async _removeServiceContainer( + serviceName: string + ): Promise<{ success: boolean; message: string }> { try { - const containers = await this.docker.listContainers({ all: true }); - const container = containers.find(c => c.Names.includes(`/${serviceName}`)); + const containers = await this.docker.listContainers({ all: true }) + const container = containers.find((c) => c.Names.includes(`/${serviceName}`)) if (!container) { - return { success: false, message: `Container for service ${serviceName} not found` }; + return { success: false, message: `Container for service ${serviceName} not found` } } - const dockerContainer = this.docker.getContainer(container.Id); - await dockerContainer.stop(); - await dockerContainer.remove(); + const dockerContainer = this.docker.getContainer(container.Id) + await dockerContainer.stop() + await dockerContainer.remove() - return { success: true, message: `Service ${serviceName} container removed successfully` }; + return { success: true, message: `Service ${serviceName} container removed successfully` } } catch (error) { - logger.error(`Error removing service container: ${error.message}`); - return { success: false, message: `Failed to remove service ${serviceName} container: ${error.message}` }; + logger.error(`Error removing service container: ${error.message}`) + return { + success: false, + message: `Failed to remove service ${serviceName} container: ${error.message}`, + } } } @@ -293,122 +329,38 @@ export class DockerService { * At least one .zim file must be available before we can start the kiwix container. * We'll download the lightweight mini Wikipedia Top 100 zim file for this purpose. **/ - const WIKIPEDIA_ZIM_URL = "https://github.com/Crosstalk-Solutions/project-nomad/blob/master/install/wikipedia_en_100_mini_2025-06.zim" - const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim'; + const WIKIPEDIA_ZIM_URL = + 'https://github.com/Crosstalk-Solutions/project-nomad/blob/master/install/wikipedia_en_100_mini_2025-06.zim' + const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim' - this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Running pre-install actions for Kiwix Serve...`); - this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`); + this._broadcast( + DockerService.KIWIX_SERVICE_NAME, + 'preinstall', + `Running pre-install actions for Kiwix Serve...` + ) + this._broadcast( + DockerService.KIWIX_SERVICE_NAME, + 'preinstall', + `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...` + ) const response = await axios.get(WIKIPEDIA_ZIM_URL, { responseType: 'stream', - }); - - const stream = response.data; - stream.on('error', (error: Error) => { - logger.error(`Error downloading Wikipedia ZIM file: ${error.message}`); - throw error; - }); - - const disk = drive.use('fs'); - await disk.putStream(PATH, stream); - - this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloaded Wikipedia ZIM file to ${PATH}`); - } - - /** - * Largely follows the install instructions here: https://github.com/Overv/openstreetmap-tile-server/blob/master/README.md - */ - private async _runPreinstallActions__OpenStreetMap(image: string, containerConfig: any): Promise { - const OSM_PBF_URL = 'https://download.geofabrik.de/north-america/us-pacific-latest.osm.pbf'; // Download US Pacific sub-region for initial import - - const IMPORT_FILE = 'region.osm.pbf'; - const IMPORT_FILE_PATH = `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/${IMPORT_FILE}`; // We only want to use the full abs path here because we need to pass it to the Docker container config - const IMPORT_BIND = `${IMPORT_FILE_PATH}:/data/${IMPORT_FILE}:rw`; - - const LOG_PATH = `/logs/${DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME}.log`; - const disk = drive.use('fs'); - - this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Running pre-install actions for OpenStreetMap Tile Server...`); - - // Ensure osm directory has proper perms for OSM container to write cached files to - this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', 'Ensuring OSM directory permissions are set correctly...'); - - // Ensure the /osm directories exist and have correct permissions - await this._fixOSMPermissions(); - - // If the initial import file already exists, delete it so we can ensure it is a good download - const fileExists = await disk.exists(IMPORT_FILE_PATH); - if (fileExists) { - await disk.delete(IMPORT_FILE_PATH); - } - - this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Downloading OpenStreetMap PBF file from ${OSM_PBF_URL}. This may take some time...`); - const response = await axios.get(OSM_PBF_URL, { - responseType: 'stream', - }); - await disk.putStream(`/osm/${IMPORT_FILE}`, response.data); - - // Do initial import of OSM data into the tile server DB - // We need to add the initial osm.pbf file as another volume bind so we can import it - const configWithImportBind = containerConfig.HostConfig || {}; - Object.assign(configWithImportBind, { - RestartPolicy: { Name: 'no' }, - Binds: [...(containerConfig.HostConfig?.Binds || []), IMPORT_BIND], - Memory: 4 * 1024 * 1024 * 1024, // 4GB - MemorySwap: -1 - }); - - this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'importing', `Processing initial import of OSM data. This may take some time...`); - await disk.put(LOG_PATH, 'Beginning OpenStreetMap data import...\n'); - - const container = await this.docker.createContainer({ - Image: image, - name: DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, - Cmd: ['import'], - HostConfig: configWithImportBind, - }); - - await container.start(); - - const logStream = await container.logs({ - stdout: true, - stderr: true, - follow: true, - timestamps: true }) - const readableLogStream: Readable = Readable.from(logStream); - await disk.putStream(LOG_PATH, readableLogStream); + const stream = response.data + stream.on('error', (error: Error) => { + logger.error(`Error downloading Wikipedia ZIM file: ${error.message}`) + throw error + }) - const data = await container.wait(); - logger.debug(`OpenStreetMap data import result: ${JSON.stringify(data)}`); + const disk = drive.use('fs') + await disk.putStream(PATH, stream) - const statusCode = data.StatusCode; - await container.remove(); - - // Run permission fix again in case the import changed perms - await this._fixOSMPermissions(); - - if (statusCode !== 0) { - throw new Error(`OpenStreetMap data import failed with status code ${statusCode}. Check the log file at ${LOG_PATH} for details.`); - } - } - - private async _fixOSMPermissions(): Promise { - try { - // Ensure directories exist - await fs.promises.mkdir(`/osm/db`, { recursive: true }); - await fs.promises.mkdir(`/osm/tiles`, { recursive: true }); - - // Must be able to read directories and read/write files inside - await chmodRecursive(`/osm/db`, 0o755, 0o755); - await chownRecursive(`/osm/db`, 1000, 1000); - - // Must be able to read directories and read/write files inside - await chmodRecursive(`/osm/tiles`, 0o755, 0o755); - await chownRecursive(`/osm/tiles`, 1000, 1000); - } catch (error) { - logger.error(`Error fixing OSM permissions: ${error.message}`); - } + this._broadcast( + DockerService.KIWIX_SERVICE_NAME, + 'preinstall', + `Downloaded Wikipedia ZIM file to ${PATH}` + ) } private _broadcast(service: string, status: string, message: string) { @@ -417,26 +369,26 @@ export class DockerService { timestamp: new Date().toISOString(), status, message, - }); - logger.info(`[DockerService] [${service}] ${status}: ${message}`); + }) + logger.info(`[DockerService] [${service}] ${status}: ${message}`) } private _parseContainerConfig(containerConfig: any): any { if (!containerConfig) { - return {}; + return {} } try { // Handle the case where containerConfig is returned as an object by DB instead of a string - let toParse = containerConfig; + let toParse = containerConfig if (typeof containerConfig === 'object') { - toParse = JSON.stringify(containerConfig); + toParse = JSON.stringify(containerConfig) } - return JSON.parse(toParse); + return JSON.parse(toParse) } catch (error) { - logger.error(`Failed to parse container configuration: ${error.message}`); - throw new Error(`Invalid container configuration: ${error.message}`); + logger.error(`Failed to parse container configuration: ${error.message}`) + throw new Error(`Invalid container configuration: ${error.message}`) } } -} \ No newline at end of file +}