github-mirror/project-nomad
1
0
Fork 0
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git synced 2026-04-04 15:56:16 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(DockerService): cleanup old OSM stuff

Browse Source
This commit is contained in:
Jake Turner 2025-12-05 15:44:41 -08:00
parent 9562e688b5
commit eed4b43353
No known key found for this signature in database
GPG Key ID: 694BC38EF2ED4844
1 changed files with 188 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

424
admin/app/services/docker_service.ts
View File

@ -1,71 +1,65 @@
import Service from "#models/service"; import Service from '#models/service'
import Docker from "dockerode"; import Docker from 'dockerode'
import drive from '@adonisjs/drive/services/main' import drive from '@adonisjs/drive/services/main'
import axios from 'axios'; import axios from 'axios'
import logger from '@adonisjs/core/services/logger' import logger from '@adonisjs/core/services/logger'
import { inject } from "@adonisjs/core"; import { inject } from '@adonisjs/core'
import { ServiceStatus } from "../../types/services.js"; import { ServiceStatus } from '../../types/services.js'
import transmit from "@adonisjs/transmit/services/main"; import transmit from '@adonisjs/transmit/services/main'
import { Readable } from "stream";
import { chmodRecursive, chownRecursive } from "../../util/files.js";
import fs from 'fs'
@inject() @inject()
export class DockerService { export class DockerService {
private docker: Docker; private docker: Docker
public static KIWIX_SERVICE_NAME = 'nomad_kiwix_serve'; public static KIWIX_SERVICE_NAME = 'nomad_kiwix_serve'
public static OPENSTREETMAP_SERVICE_NAME = 'nomad_openstreetmap'; public static OLLAMA_SERVICE_NAME = 'nomad_ollama'
public static OPENSTREETMAP_IMPORT_SERVICE_NAME = 'nomad_openstreetmap_import'; public static OPEN_WEBUI_SERVICE_NAME = 'nomad_open_webui'
public static OLLAMA_SERVICE_NAME = 'nomad_ollama'; public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef'
public static OPEN_WEBUI_SERVICE_NAME = 'nomad_open_webui'; public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes'
public static CYBERCHEF_SERVICE_NAME = 'nomad_cyberchef'; public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri'
public static FLATNOTES_SERVICE_NAME = 'nomad_flatnotes'; public static NOMAD_STORAGE_ABS_PATH = '/opt/project-nomad/storage'
public static KOLIBRI_SERVICE_NAME = 'nomad_kolibri';
public static NOMAD_STORAGE_ABS_PATH = '/opt/project-nomad/storage';
constructor() { constructor() {
this.docker = new Docker({ socketPath: '/var/run/docker.sock' }); this.docker = new Docker({ socketPath: '/var/run/docker.sock' })
} }
async affectContainer(serviceName: string, action: 'start' | 'stop' | 'restart'): Promise<{ success: boolean; message: string }> { async affectContainer(
serviceName: string,
action: 'start' | 'stop' | 'restart'
): Promise<{ success: boolean; message: string }> {
try { try {
const service = await Service.query().where('service_name', serviceName).first(); const service = await Service.query().where('service_name', serviceName).first()
if (!service || !service.installed) { if (!service || !service.installed) {
return { return {
success: false, success: false,
message: `Service ${serviceName} not found or not installed`, message: `Service ${serviceName} not found or not installed`,
}; }
} }
const containers = await this.docker.listContainers({ all: true }); const containers = await this.docker.listContainers({ all: true })
const container = containers.find(c => c.Names.includes(`/${serviceName}`)); const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
if (!container) { if (!container) {
return { return {
success: false, success: false,
message: `Container for service ${serviceName} not found`, message: `Container for service ${serviceName} not found`,
}; }
} }
const dockerContainer = this.docker.getContainer(container.Id); const dockerContainer = this.docker.getContainer(container.Id)
if (action === 'stop') { if (action === 'stop') {
await dockerContainer.stop(); await dockerContainer.stop()
return { return {
success: true, success: true,
message: `Service ${serviceName} stopped successfully`, message: `Service ${serviceName} stopped successfully`,
}; }
} }
if (action === 'restart') { if (action === 'restart') {
await dockerContainer.restart(); await dockerContainer.restart()
if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) {
await this._fixOSMPermissions();
}
return { return {
success: true, success: true,
message: `Service ${serviceName} restarted successfully`, message: `Service ${serviceName} restarted successfully`,
}; }
} }
if (action === 'start') { if (action === 'start') {
@ -73,19 +67,15 @@ export class DockerService {
return { return {
success: true, success: true,
message: `Service ${serviceName} is already running`, message: `Service ${serviceName} is already running`,
}; }
} }
await dockerContainer.start(); await dockerContainer.start()
if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) {
await this._fixOSMPermissions();
}
return { return {
success: true, success: true,
message: `Service ${serviceName} started successfully`, message: `Service ${serviceName} started successfully`,
}; }
} }
return { return {
@ -93,71 +83,74 @@ export class DockerService {
message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`, message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`,
} }
} catch (error) { } catch (error) {
logger.error(`Error starting service ${serviceName}: ${error.message}`); logger.error(`Error starting service ${serviceName}: ${error.message}`)
return { return {
success: false, success: false,
message: `Failed to start service ${serviceName}: ${error.message}`, message: `Failed to start service ${serviceName}: ${error.message}`,
}; }
} }
} }
async getServicesStatus(): Promise<{ async getServicesStatus(): Promise<
service_name: string; {
status: ServiceStatus; service_name: string
}[]> { status: ServiceStatus
}[]
> {
try { try {
const services = await Service.query().where('installed', true); const services = await Service.query().where('installed', true)
if (!services || services.length === 0) { if (!services || services.length === 0) {
return []; return []
} }
const containers = await this.docker.listContainers({ all: true }); const containers = await this.docker.listContainers({ all: true })
const containerMap = new Map<string, Docker.ContainerInfo>(); const containerMap = new Map<string, Docker.ContainerInfo>()
containers.forEach(container => { containers.forEach((container) => {
const name = container.Names[0].replace('/', ''); const name = container.Names[0].replace('/', '')
if (name.startsWith('nomad_')) { if (name.startsWith('nomad_')) {
containerMap.set(name, container); containerMap.set(name, container)
} }
}); })
const getStatus = (state: string): ServiceStatus => { const getStatus = (state: string): ServiceStatus => {
switch (state) { switch (state) {
case 'running': case 'running':
return 'running'; return 'running'
case 'exited': case 'exited':
case 'created': case 'created':
case 'paused': case 'paused':
return 'stopped'; return 'stopped'
default: default:
return 'unknown'; return 'unknown'
} }
}; }
return Array.from(containerMap.entries()).map(([name, container]) => ({ return Array.from(containerMap.entries()).map(([name, container]) => ({
service_name: name, service_name: name,
status: getStatus(container.State), status: getStatus(container.State),
})); }))
} catch (error) { } catch (error) {
console.error(`Error fetching services status: ${error.message}`); console.error(`Error fetching services status: ${error.message}`)
return []; return []
} }
} }
async createContainerPreflight(serviceName: string): Promise<{ success: boolean; message: string }> { async createContainerPreflight(
const service = await Service.query().where('service_name', serviceName).first(); serviceName: string
): Promise<{ success: boolean; message: string }> {
const service = await Service.query().where('service_name', serviceName).first()
if (!service) { if (!service) {
return { return {
success: false, success: false,
message: `Service ${serviceName} not found`, message: `Service ${serviceName} not found`,
}; }
} }
if (service.installed) { if (service.installed) {
return { return {
success: false, success: false,
message: `Service ${serviceName} is already installed`, message: `Service ${serviceName} is already installed`,
}; }
} }
// Check if a service wasn't marked as installed but has an existing container // Check if a service wasn't marked as installed but has an existing container
@ -173,8 +166,8 @@ export class DockerService {
// } // }
// } // }
const containerConfig = this._parseContainerConfig(service.container_config); const containerConfig = this._parseContainerConfig(service.container_config)
this._createContainer(service, containerConfig); // Don't await this method - we will use server-sent events to notify the client of progress this._createContainer(service, containerConfig) // Don't await this method - we will use server-sent events to notify the client of progress
return { return {
success: true, success: true,
@ -187,48 +180,75 @@ export class DockerService {
* NOTE: This method should not be called directly. Instead, use `createContainerPreflight` to check prerequisites first * NOTE: This method should not be called directly. Instead, use `createContainerPreflight` to check prerequisites first
* and return an HTTP response to the client, if needed. This method will then transmit server-sent events to the client * and return an HTTP response to the client, if needed. This method will then transmit server-sent events to the client
* to notify them of the progress. * to notify them of the progress.
* @param serviceName * @param serviceName
* @returns * @returns
*/ */
async _createContainer(service: Service & { dependencies?: Service[] }, containerConfig: any): Promise<void> { async _createContainer(
service: Service & { dependencies?: Service[] },
containerConfig: any
): Promise<void> {
try { try {
this._broadcast(service.service_name, 'initializing', ''); this._broadcast(service.service_name, 'initializing', '')
let dependencies = []; let dependencies = []
if (service.depends_on) { if (service.depends_on) {
const dependency = await Service.query().where('service_name', service.depends_on).first(); const dependency = await Service.query().where('service_name', service.depends_on).first()
if (dependency) { if (dependency) {
dependencies.push(dependency); dependencies.push(dependency)
} }
} }
// First, check if the service has any dependencies that need to be installed first // First, check if the service has any dependencies that need to be installed first
if (dependencies && dependencies.length > 0) { if (dependencies && dependencies.length > 0) {
this._broadcast(service.service_name, 'checking-dependencies', `Checking dependencies for service ${service.service_name}...`); this._broadcast(
service.service_name,
'checking-dependencies',
`Checking dependencies for service ${service.service_name}...`
)
for (const dependency of dependencies) { for (const dependency of dependencies) {
if (!dependency.installed) { if (!dependency.installed) {
this._broadcast(service.service_name, 'dependency-not-installed', `Dependency service ${dependency.service_name} is not installed. Installing it first...`); this._broadcast(
await this._createContainer(dependency, this._parseContainerConfig(dependency.container_config)); service.service_name,
'dependency-not-installed',
`Dependency service ${dependency.service_name} is not installed. Installing it first...`
)
await this._createContainer(
dependency,
this._parseContainerConfig(dependency.container_config)
)
} else { } else {
this._broadcast(service.service_name, 'dependency-installed', `Dependency service ${dependency.service_name} is already installed.`); this._broadcast(
service.service_name,
'dependency-installed',
`Dependency service ${dependency.service_name} is already installed.`
)
} }
} }
} }
// Start pulling the Docker image and wait for it to complete // Start pulling the Docker image and wait for it to complete
const pullStream = await this.docker.pull(service.container_image); const pullStream = await this.docker.pull(service.container_image)
this._broadcast(service.service_name, 'pulling', `Pulling Docker image ${service.container_image}...`); this._broadcast(
await new Promise(res => this.docker.modem.followProgress(pullStream, res)); service.service_name,
'pulling',
`Pulling Docker image ${service.container_image}...`
)
await new Promise((res) => this.docker.modem.followProgress(pullStream, res))
if (service.service_name === DockerService.KIWIX_SERVICE_NAME) { if (service.service_name === DockerService.KIWIX_SERVICE_NAME) {
await this._runPreinstallActions__KiwixServe(); await this._runPreinstallActions__KiwixServe()
this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`); this._broadcast(
} else if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { service.service_name,
await this._runPreinstallActions__OpenStreetMap(service.container_image, containerConfig); 'preinstall-complete',
this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for OpenStreetMap completed successfully.`); `Pre-install actions for Kiwix Serve completed successfully.`
)
} }
this._broadcast(service.service_name, 'creating', `Creating Docker container for service ${service.service_name}...`); this._broadcast(
service.service_name,
'creating',
`Creating Docker container for service ${service.service_name}...`
)
const container = await this.docker.createContainer({ const container = await this.docker.createContainer({
Image: service.container_image, Image: service.container_image,
name: service.service_name, name: service.service_name,
@ -238,53 +258,69 @@ export class DockerService {
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }), ...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
...(containerConfig?.Env && { Env: containerConfig.Env }), ...(containerConfig?.Env && { Env: containerConfig.Env }),
...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}), ...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}),
}); })
this._broadcast(service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...`); this._broadcast(
await container.start(); service.service_name,
'starting',
`Starting Docker container for service ${service.service_name}...`
)
await container.start()
// Ensure OSM directories have correct permissions after install+start this._broadcast(
if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) { service.service_name,
await this._fixOSMPermissions(); 'finalizing',
} `Finalizing installation of service ${service.service_name}...`
)
service.installed = true
await service.save()
this._broadcast(service.service_name, 'finalizing', `Finalizing installation of service ${service.service_name}...`); this._broadcast(
service.installed = true; service.service_name,
await service.save(); 'completed',
`Service ${service.service_name} installation completed successfully.`
this._broadcast(service.service_name, 'completed', `Service ${service.service_name} installation completed successfully.`); )
} catch (error) { } catch (error) {
this._broadcast(service.service_name, 'error', `Error installing service ${service.service_name}: ${error.message}`); this._broadcast(
throw new Error(`Failed to install service ${service.service_name}: ${error.message}`); service.service_name,
'error',
`Error installing service ${service.service_name}: ${error.message}`
)
throw new Error(`Failed to install service ${service.service_name}: ${error.message}`)
} }
} }
async _checkIfServiceContainerExists(serviceName: string): Promise<boolean> { async _checkIfServiceContainerExists(serviceName: string): Promise<boolean> {
try { try {
const containers = await this.docker.listContainers({ all: true }); const containers = await this.docker.listContainers({ all: true })
return containers.some(container => container.Names.includes(`/${serviceName}`)); return containers.some((container) => container.Names.includes(`/${serviceName}`))
} catch (error) { } catch (error) {
logger.error(`Error checking if service container exists: ${error.message}`); logger.error(`Error checking if service container exists: ${error.message}`)
return false; return false
} }
} }
async _removeServiceContainer(serviceName: string): Promise<{ success: boolean; message: string }> { async _removeServiceContainer(
serviceName: string
): Promise<{ success: boolean; message: string }> {
try { try {
const containers = await this.docker.listContainers({ all: true }); const containers = await this.docker.listContainers({ all: true })
const container = containers.find(c => c.Names.includes(`/${serviceName}`)); const container = containers.find((c) => c.Names.includes(`/${serviceName}`))
if (!container) { if (!container) {
return { success: false, message: `Container for service ${serviceName} not found` }; return { success: false, message: `Container for service ${serviceName} not found` }
} }
const dockerContainer = this.docker.getContainer(container.Id); const dockerContainer = this.docker.getContainer(container.Id)
await dockerContainer.stop(); await dockerContainer.stop()
await dockerContainer.remove(); await dockerContainer.remove()
return { success: true, message: `Service ${serviceName} container removed successfully` }; return { success: true, message: `Service ${serviceName} container removed successfully` }
} catch (error) { } catch (error) {
logger.error(`Error removing service container: ${error.message}`); logger.error(`Error removing service container: ${error.message}`)
return { success: false, message: `Failed to remove service ${serviceName} container: ${error.message}` }; return {
success: false,
message: `Failed to remove service ${serviceName} container: ${error.message}`,
}
} }
} }
@ -293,122 +329,38 @@ export class DockerService {
* At least one .zim file must be available before we can start the kiwix container. * At least one .zim file must be available before we can start the kiwix container.
* We'll download the lightweight mini Wikipedia Top 100 zim file for this purpose. * We'll download the lightweight mini Wikipedia Top 100 zim file for this purpose.
**/ **/
const WIKIPEDIA_ZIM_URL = "https://github.com/Crosstalk-Solutions/project-nomad/blob/master/install/wikipedia_en_100_mini_2025-06.zim" const WIKIPEDIA_ZIM_URL =
const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim'; 'https://github.com/Crosstalk-Solutions/project-nomad/blob/master/install/wikipedia_en_100_mini_2025-06.zim'
const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim'
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Running pre-install actions for Kiwix Serve...`); this._broadcast(
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`); DockerService.KIWIX_SERVICE_NAME,
'preinstall',
`Running pre-install actions for Kiwix Serve...`
)
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall',
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
)
const response = await axios.get(WIKIPEDIA_ZIM_URL, { const response = await axios.get(WIKIPEDIA_ZIM_URL, {
responseType: 'stream', responseType: 'stream',
});
const stream = response.data;
stream.on('error', (error: Error) => {
logger.error(`Error downloading Wikipedia ZIM file: ${error.message}`);
throw error;
});
const disk = drive.use('fs');
await disk.putStream(PATH, stream);
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloaded Wikipedia ZIM file to ${PATH}`);
}
/**
* Largely follows the install instructions here: https://github.com/Overv/openstreetmap-tile-server/blob/master/README.md
*/
private async _runPreinstallActions__OpenStreetMap(image: string, containerConfig: any): Promise<void> {
const OSM_PBF_URL = 'https://download.geofabrik.de/north-america/us-pacific-latest.osm.pbf'; // Download US Pacific sub-region for initial import
const IMPORT_FILE = 'region.osm.pbf';
const IMPORT_FILE_PATH = `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/${IMPORT_FILE}`; // We only want to use the full abs path here because we need to pass it to the Docker container config
const IMPORT_BIND = `${IMPORT_FILE_PATH}:/data/${IMPORT_FILE}:rw`;
const LOG_PATH = `/logs/${DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME}.log`;
const disk = drive.use('fs');
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Running pre-install actions for OpenStreetMap Tile Server...`);
// Ensure osm directory has proper perms for OSM container to write cached files to
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', 'Ensuring OSM directory permissions are set correctly...');
// Ensure the /osm directories exist and have correct permissions
await this._fixOSMPermissions();
// If the initial import file already exists, delete it so we can ensure it is a good download
const fileExists = await disk.exists(IMPORT_FILE_PATH);
if (fileExists) {
await disk.delete(IMPORT_FILE_PATH);
}
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Downloading OpenStreetMap PBF file from ${OSM_PBF_URL}. This may take some time...`);
const response = await axios.get(OSM_PBF_URL, {
responseType: 'stream',
});
await disk.putStream(`/osm/${IMPORT_FILE}`, response.data);
// Do initial import of OSM data into the tile server DB
// We need to add the initial osm.pbf file as another volume bind so we can import it
const configWithImportBind = containerConfig.HostConfig || {};
Object.assign(configWithImportBind, {
RestartPolicy: { Name: 'no' },
Binds: [...(containerConfig.HostConfig?.Binds || []), IMPORT_BIND],
Memory: 4 * 1024 * 1024 * 1024, // 4GB
MemorySwap: -1
});
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'importing', `Processing initial import of OSM data. This may take some time...`);
await disk.put(LOG_PATH, 'Beginning OpenStreetMap data import...\n');
const container = await this.docker.createContainer({
Image: image,
name: DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME,
Cmd: ['import'],
HostConfig: configWithImportBind,
});
await container.start();
const logStream = await container.logs({
stdout: true,
stderr: true,
follow: true,
timestamps: true
}) })
const readableLogStream: Readable = Readable.from(logStream); const stream = response.data
await disk.putStream(LOG_PATH, readableLogStream); stream.on('error', (error: Error) => {
logger.error(`Error downloading Wikipedia ZIM file: ${error.message}`)
throw error
})
const data = await container.wait(); const disk = drive.use('fs')
logger.debug(`OpenStreetMap data import result: ${JSON.stringify(data)}`); await disk.putStream(PATH, stream)
const statusCode = data.StatusCode; this._broadcast(
await container.remove(); DockerService.KIWIX_SERVICE_NAME,
'preinstall',
// Run permission fix again in case the import changed perms `Downloaded Wikipedia ZIM file to ${PATH}`
await this._fixOSMPermissions(); )
if (statusCode !== 0) {
throw new Error(`OpenStreetMap data import failed with status code ${statusCode}. Check the log file at ${LOG_PATH} for details.`);
}
}
private async _fixOSMPermissions(): Promise<void> {
try {
// Ensure directories exist
await fs.promises.mkdir(`/osm/db`, { recursive: true });
await fs.promises.mkdir(`/osm/tiles`, { recursive: true });
// Must be able to read directories and read/write files inside
await chmodRecursive(`/osm/db`, 0o755, 0o755);
await chownRecursive(`/osm/db`, 1000, 1000);
// Must be able to read directories and read/write files inside
await chmodRecursive(`/osm/tiles`, 0o755, 0o755);
await chownRecursive(`/osm/tiles`, 1000, 1000);
} catch (error) {
logger.error(`Error fixing OSM permissions: ${error.message}`);
}
} }
private _broadcast(service: string, status: string, message: string) { private _broadcast(service: string, status: string, message: string) {
@ -417,26 +369,26 @@ export class DockerService {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status, status,
message, message,
}); })
logger.info(`[DockerService] [${service}] ${status}: ${message}`); logger.info(`[DockerService] [${service}] ${status}: ${message}`)
} }
private _parseContainerConfig(containerConfig: any): any { private _parseContainerConfig(containerConfig: any): any {
if (!containerConfig) { if (!containerConfig) {
return {}; return {}
} }
try { try {
// Handle the case where containerConfig is returned as an object by DB instead of a string // Handle the case where containerConfig is returned as an object by DB instead of a string
let toParse = containerConfig; let toParse = containerConfig
if (typeof containerConfig === 'object') { if (typeof containerConfig === 'object') {
toParse = JSON.stringify(containerConfig); toParse = JSON.stringify(containerConfig)
} }
return JSON.parse(toParse); return JSON.parse(toParse)
} catch (error) { } catch (error) {
logger.error(`Failed to parse container configuration: ${error.message}`); logger.error(`Failed to parse container configuration: ${error.message}`)
throw new Error(`Invalid container configuration: ${error.message}`); throw new Error(`Invalid container configuration: ${error.message}`)
} }
} }
} }