github-mirror/project-nomad
1
0
Fork 0
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git synced 2026-04-03 23:36:17 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(Kiwix): migrate to Kiwix library mode for improved stability (#622)

Browse Source
This commit is contained in:
Jake Turner 2026-04-01 23:09:00 -07:00
parent d5a6b319b4
commit 78a9c43c0d
9 changed files with 555 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
admin/adonisrc.ts
View File

@ -53,7 +53,8 @@ export default defineConfig({
() => import('@adonisjs/lucid/database_provider'),
() => import('@adonisjs/inertia/inertia_provider'),
() => import('@adonisjs/transmit/transmit_provider'),
() => import('#providers/map_static_provider')
() => import('#providers/map_static_provider'),
() => import('#providers/kiwix_migration_provider'),
],
/*

166
admin/app/services/docker_service.ts
View File

@ -6,18 +6,20 @@ import transmit from '@adonisjs/transmit/services/main'
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
import { join } from 'path'
import { ZIM_STORAGE_PATH } from '../utils/fs.js'
import { KiwixLibraryService } from './kiwix_library_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { exec } from 'child_process'
import { promisify } from 'util'
// import { readdir } from 'fs/promises'
import KVStore from '#models/kv_store'
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
@inject()
export class DockerService {
public docker: Docker
private activeInstallations: Set<string> = new Set()
public static NOMAD_NETWORK = 'project-nomad_default'
public static NOMAD_NETWORK = 'project-nomad_default'
constructor() {
// Support both Linux (production) and Windows (development with Docker Desktop)
@ -63,6 +65,15 @@ export class DockerService {
}
if (action === 'restart') {
if (serviceName === SERVICE_NAMES.KIWIX) {
const isLegacy = await this.isKiwixOnLegacyConfig()
if (isLegacy) {
logger.info('[DockerService] Kiwix on legacy glob config — running migration instead of restart.')
await this.migrateKiwixToLibraryMode()
return { success: true, message: 'Kiwix migrated to library mode successfully.' }
}
}
await dockerContainer.restart()
return {
@ -91,7 +102,7 @@ export class DockerService {
success: false,
message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`,
}
} catch (error) {
} catch (error: any) {
logger.error(`Error starting service ${serviceName}: ${error.message}`)
return {
success: false,
@ -123,7 +134,7 @@ export class DockerService {
service_name: name,
status: container.State,
}))
} catch (error) {
} catch (error: any) {
logger.error(`Error fetching services status: ${error.message}`)
return []
}
@ -312,7 +323,7 @@ export class DockerService {
`No existing container found, proceeding with installation...`
)
}
} catch (error) {
} catch (error: any) {
logger.warn(`Error during container cleanup: ${error.message}`)
this._broadcast(serviceName, 'cleanup-warning', `Warning during cleanup: ${error.message}`)
}
@ -331,7 +342,7 @@ export class DockerService {
const volume = this.docker.getVolume(vol.Name)
await volume.remove({ force: true })
this._broadcast(serviceName, 'volume-removed', `Removed volume: ${vol.Name}`)
} catch (error) {
} catch (error: any) {
logger.warn(`Failed to remove volume ${vol.Name}: ${error.message}`)
}
}
@ -339,7 +350,7 @@ export class DockerService {
if (serviceVolumes.length === 0) {
this._broadcast(serviceName, 'no-volumes', `No volumes found to clear`)
}
} catch (error) {
} catch (error: any) {
logger.warn(`Error during volume cleanup: ${error.message}`)
this._broadcast(
serviceName,
@ -367,7 +378,7 @@ export class DockerService {
success: true,
message: `Service ${serviceName} force reinstall initiated successfully. You can receive updates via server-sent events.`,
}
} catch (error) {
} catch (error: any) {
logger.error(`Force reinstall failed for ${serviceName}: ${error.message}`)
await this._cleanupFailedInstallation(serviceName)
return {
@ -583,7 +594,7 @@ export class DockerService {
'completed',
`Service ${service.service_name} installation completed successfully.`
)
} catch (error) {
} catch (error: any) {
this._broadcast(
service.service_name,
'error',
@ -599,7 +610,7 @@ export class DockerService {
try {
const containers = await this.docker.listContainers({ all: true })
return containers.some((container) => container.Names.includes(`/${serviceName}`))
} catch (error) {
} catch (error: any) {
logger.error(`Error checking if service container exists: ${error.message}`)
return false
}
@ -619,7 +630,7 @@ export class DockerService {
await dockerContainer.remove({ force: true })
return { success: true, message: `Service ${serviceName} container removed successfully` }
} catch (error) {
} catch (error: any) {
logger.error(`Error removing service container: ${error.message}`)
return {
success: false,
@ -667,7 +678,12 @@ export class DockerService {
'preinstall',
`Downloaded Wikipedia ZIM file to ${filepath}`
)
} catch (error) {
// Generate the initial kiwix library XML before the container is created
const kiwixLibraryService = new KiwixLibraryService()
await kiwixLibraryService.rebuildFromDisk()
this._broadcast(SERVICE_NAMES.KIWIX, 'preinstall', 'Generated kiwix library XML.')
} catch (error: any) {
this._broadcast(
SERVICE_NAMES.KIWIX,
'preinstall-error',
@ -690,13 +706,121 @@ export class DockerService {
await this._removeServiceContainer(serviceName)
logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
} catch (error) {
} catch (error: any) {
logger.error(
`[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`
)
}
}
/**
* Checks whether the running kiwix container is using the legacy glob-pattern command
* (`*.zim --address=all`) rather than the library-file command. Used to detect containers
* that need to be migrated to library mode.
*/
async isKiwixOnLegacyConfig(): Promise<boolean> {
try {
const containers = await this.docker.listContainers({ all: true })
const info = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
if (!info) return false
const inspected = await this.docker.getContainer(info.Id).inspect()
const cmd: string[] = inspected.Config?.Cmd ?? []
return cmd.some((arg) => arg.includes('*.zim'))
} catch (err: any) {
logger.warn(`[DockerService] Could not inspect kiwix container: ${err.message}`)
return false
}
}
/**
* Migrates the kiwix container from legacy glob mode (`*.zim`) to library mode
* (`--library /data/kiwix-library.xml --monitorLibrary`).
*
* This is a non-destructive recreation: ZIM files and volumes are preserved.
* The container is stopped, removed, and recreated with the correct library-mode command.
* This function is authoritative: it writes the correct command to the DB itself rather than
* trusting the DB to have been pre-updated by a separate migration.
*/
async migrateKiwixToLibraryMode(): Promise<void> {
if (this.activeInstallations.has(SERVICE_NAMES.KIWIX)) {
logger.warn('[DockerService] Kiwix migration already in progress, skipping duplicate call.')
return
}
this.activeInstallations.add(SERVICE_NAMES.KIWIX)
try {
// Step 1: Build/update the XML from current disk state
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Migrating kiwix to library mode...')
const kiwixLibraryService = new KiwixLibraryService()
await kiwixLibraryService.rebuildFromDisk()
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Built kiwix library XML from existing ZIM files.')
// Step 2: Stop and remove old container (leave ZIM volumes intact)
const containers = await this.docker.listContainers({ all: true })
const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
if (containerInfo) {
const oldContainer = this.docker.getContainer(containerInfo.Id)
if (containerInfo.State === 'running') {
await oldContainer.stop({ t: 10 }).catch((e: any) =>
logger.warn(`[DockerService] Kiwix stop warning during migration: ${e.message}`)
)
}
await oldContainer.remove({ force: true }).catch((e: any) =>
logger.warn(`[DockerService] Kiwix remove warning during migration: ${e.message}`)
)
}
// Step 3: Read the service record and authoritatively set the correct command.
// Do NOT rely on prior DB state — we write container_command here so the record
// stays consistent regardless of whether the DB migration ran.
const service = await Service.query().where('service_name', SERVICE_NAMES.KIWIX).first()
if (!service) {
throw new Error('Kiwix service record not found in DB during migration')
}
service.container_command = KIWIX_LIBRARY_CMD
service.installed = false
service.installation_status = 'installing'
await service.save()
const containerConfig = this._parseContainerConfig(service.container_config)
// Step 4: Recreate container directly (skipping _createContainer to avoid re-downloading
// the bootstrap ZIM — ZIM files already exist on disk)
this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Recreating kiwix container with library mode config...')
const newContainer = await this.docker.createContainer({
Image: service.container_image,
name: service.service_name,
HostConfig: containerConfig?.HostConfig ?? {},
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
Cmd: KIWIX_LIBRARY_CMD.split(' '),
...(process.env.NODE_ENV === 'production' && {
NetworkingConfig: {
EndpointsConfig: {
[DockerService.NOMAD_NETWORK]: {},
},
},
}),
})
await newContainer.start()
service.installed = true
service.installation_status = 'idle'
await service.save()
this.activeInstallations.delete(SERVICE_NAMES.KIWIX)
this._broadcast(SERVICE_NAMES.KIWIX, 'migrated', 'Kiwix successfully migrated to library mode.')
logger.info('[DockerService] Kiwix migration to library mode complete.')
} catch (error: any) {
logger.error(`[DockerService] Kiwix migration failed: ${error.message}`)
await this._cleanupFailedInstallation(SERVICE_NAMES.KIWIX)
throw error
}
}
/**
* Detect GPU type and toolkit availability.
* Primary: Check Docker runtimes via docker.info() (works from inside containers).
@ -713,7 +837,7 @@ export class DockerService {
await this._persistGPUType('nvidia')
return { type: 'nvidia' }
}
} catch (error) {
} catch (error: any) {
logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`)
}
@ -730,7 +854,7 @@ export class DockerService {
logger.warn('[DockerService] NVIDIA GPU detected via lspci but NVIDIA Container Toolkit is not installed')
return { type: 'none', toolkitMissing: true }
}
} catch (error) {
} catch (error: any) {
// lspci not available (likely inside Docker container), continue
}
@ -745,7 +869,7 @@ export class DockerService {
await this._persistGPUType('amd')
return { type: 'amd' }
}
} catch (error) {
} catch (error: any) {
// lspci not available, continue
}
@ -764,7 +888,7 @@ export class DockerService {
logger.info('[DockerService] No GPU detected')
return { type: 'none' }
} catch (error) {
} catch (error: any) {
logger.warn(`[DockerService] Error detecting GPU type: ${error.message}`)
return { type: 'none' }
}
@ -774,7 +898,7 @@ export class DockerService {
try {
await KVStore.setValue('gpu.type', type)
logger.info(`[DockerService] Persisted GPU type '${type}' to KV store`)
} catch (error) {
} catch (error: any) {
logger.warn(`[DockerService] Failed to persist GPU type: ${error.message}`)
}
}
@ -969,7 +1093,7 @@ export class DockerService {
let newContainer: any
try {
newContainer = await this.docker.createContainer(newContainerConfig)
} catch (createError) {
} catch (createError: any) {
// Rollback: rename old container back
this._broadcast(serviceName, 'update-rollback', `Failed to create new container: ${createError.message}. Rolling back...`)
const rollbackContainer = this.docker.getContainer((await this.docker.listContainers({ all: true })).find((c) => c.Names.includes(`/${oldName}`))!.Id)
@ -1042,7 +1166,7 @@ export class DockerService {
message: `Update failed: new container did not stay running. Rolled back to previous version.`,
}
}
} catch (error) {
} catch (error: any) {
this.activeInstallations.delete(serviceName)
this._broadcast(
serviceName,
@ -1077,7 +1201,7 @@ export class DockerService {
}
return JSON.parse(toParse)
} catch (error) {
} catch (error: any) {
logger.error(`Failed to parse container configuration: ${error.message}`)
throw new Error(`Invalid container configuration: ${error.message}`)
}
@ -1094,7 +1218,7 @@ export class DockerService {
// Check if any image has a RepoTag that matches the requested image
return images.some((image) => image.RepoTags && image.RepoTags.includes(imageName))
} catch (error) {
} catch (error: any) {
logger.warn(`Error checking if image exists: ${error.message}`)
// If run into an error, assume the image does not exist
return false

285
admin/app/services/kiwix_library_service.ts Normal file
View File

@ -0,0 +1,285 @@
import { XMLBuilder, XMLParser } from 'fast-xml-parser'
import { readFile, writeFile, rename, readdir } from 'fs/promises'
import { join } from 'path'
import { Archive } from '@openzim/libzim'
import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists } from '../utils/fs.js'
import logger from '@adonisjs/core/services/logger'
import { randomUUID } from 'node:crypto'
const CONTAINER_DATA_PATH = '/data'
const XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>\n'
interface KiwixBook {
id: string
path: string
title: string
description?: string
language?: string
creator?: string
publisher?: string
name?: string
flavour?: string
tags?: string
faviconMimeType?: string
favicon?: string
date?: string
articleCount?: number
mediaCount?: number
size?: number
}
export class KiwixLibraryService {
getLibraryFilePath(): string {
return join(process.cwd(), KIWIX_LIBRARY_XML_PATH)
}
containerLibraryPath(): string {
return '/data/kiwix-library.xml'
}
private _filenameToTitle(filename: string): string {
const withoutExt = filename.endsWith('.zim') ? filename.slice(0, -4) : filename
const parts = withoutExt.split('_')
// Drop last segment if it looks like a date (YYYY-MM)
const lastPart = parts[parts.length - 1]
const isDate = /^\d{4}-\d{2}$/.test(lastPart)
const titleParts = isDate && parts.length > 1 ? parts.slice(0, -1) : parts
return titleParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(' ')
}
/**
* Reads all kiwix-manage-compatible metadata from a ZIM file, including the internal UUID,
* rich text fields, and the base64-encoded favicon. Kiwix-serve uses the UUID for OPDS
* catalog entries and illustration URLs (/catalog/v2/illustration/{uuid}).
*
* Returns null on any error so callers can fall back gracefully.
*/
private _readZimMetadata(zimFilePath: string): Partial<KiwixBook> | null {
try {
const archive = new Archive(zimFilePath)
const getMeta = (key: string): string | undefined => {
try {
return archive.getMetadata(key) || undefined
} catch {
return undefined
}
}
let favicon: string | undefined
let faviconMimeType: string | undefined
try {
if (archive.illustrationSizes.size > 0) {
const size = archive.illustrationSizes.has(48)
? 48
: ([...archive.illustrationSizes][0] as number)
const item = archive.getIllustrationItem(size)
favicon = item.data.data.toString('base64')
faviconMimeType = item.mimetype || undefined
}
} catch {
// ZIM has no illustration — that's fine
}
const rawFilesize =
typeof archive.filesize === 'bigint' ? Number(archive.filesize) : archive.filesize
return {
id: archive.uuid || undefined,
title: getMeta('Title'),
description: getMeta('Description'),
language: getMeta('Language'),
creator: getMeta('Creator'),
publisher: getMeta('Publisher'),
name: getMeta('Name'),
flavour: getMeta('Flavour'),
tags: getMeta('Tags'),
date: getMeta('Date'),
articleCount: archive.articleCount,
mediaCount: archive.mediaCount,
size: Math.floor(rawFilesize / 1024),
favicon,
faviconMimeType,
}
} catch {
return null
}
}
private _buildXml(books: KiwixBook[]): string {
const builder = new XMLBuilder({
ignoreAttributes: false,
attributeNamePrefix: '@_',
format: true,
suppressEmptyNode: false,
})
const obj: Record<string, any> = {
library: {
'@_version': '20110515',
...(books.length > 0 && {
book: books.map((b) => ({
'@_id': b.id,
'@_path': b.path,
'@_title': b.title,
...(b.description !== undefined && { '@_description': b.description }),
...(b.language !== undefined && { '@_language': b.language }),
...(b.creator !== undefined && { '@_creator': b.creator }),
...(b.publisher !== undefined && { '@_publisher': b.publisher }),
...(b.name !== undefined && { '@_name': b.name }),
...(b.flavour !== undefined && { '@_flavour': b.flavour }),
...(b.tags !== undefined && { '@_tags': b.tags }),
...(b.faviconMimeType !== undefined && { '@_faviconMimeType': b.faviconMimeType }),
...(b.favicon !== undefined && { '@_favicon': b.favicon }),
...(b.date !== undefined && { '@_date': b.date }),
...(b.articleCount !== undefined && { '@_articleCount': b.articleCount }),
...(b.mediaCount !== undefined && { '@_mediaCount': b.mediaCount }),
...(b.size !== undefined && { '@_size': b.size }),
})),
}),
},
}
return XML_DECLARATION + builder.build(obj)
}
private async _atomicWrite(content: string): Promise<void> {
const filePath = this.getLibraryFilePath()
const tmpPath = `${filePath}.tmp.${randomUUID()}`
await writeFile(tmpPath, content, 'utf-8')
await rename(tmpPath, filePath)
}
private _parseExistingBooks(xmlContent: string): KiwixBook[] {
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
isArray: (name) => name === 'book',
})
const parsed = parser.parse(xmlContent)
const books: any[] = parsed?.library?.book ?? []
return books
.map((b) => ({
id: b['@_id'] ?? '',
path: b['@_path'] ?? '',
title: b['@_title'] ?? '',
description: b['@_description'],
language: b['@_language'],
creator: b['@_creator'],
publisher: b['@_publisher'],
name: b['@_name'],
flavour: b['@_flavour'],
tags: b['@_tags'],
faviconMimeType: b['@_faviconMimeType'],
favicon: b['@_favicon'],
date: b['@_date'],
articleCount:
b['@_articleCount'] !== undefined ? Number(b['@_articleCount']) : undefined,
mediaCount: b['@_mediaCount'] !== undefined ? Number(b['@_mediaCount']) : undefined,
size: b['@_size'] !== undefined ? Number(b['@_size']) : undefined,
}))
.filter((b) => b.id && b.path)
}
async rebuildFromDisk(opts?: { excludeFilenames?: string[] }): Promise<void> {
const dirPath = join(process.cwd(), ZIM_STORAGE_PATH)
await ensureDirectoryExists(dirPath)
let entries: string[] = []
try {
entries = await readdir(dirPath)
} catch {
entries = []
}
const excludeSet = new Set(opts?.excludeFilenames ?? [])
const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name))
const books: KiwixBook[] = zimFiles.map((filename) => {
const meta = this._readZimMetadata(join(dirPath, filename))
const containerPath = `${CONTAINER_DATA_PATH}/${filename}`
return {
...meta,
// Override fields that must be derived locally, not from ZIM metadata
id: meta?.id ?? filename.slice(0, -4),
path: containerPath,
title: meta?.title ?? this._filenameToTitle(filename),
}
})
const xml = this._buildXml(books)
await this._atomicWrite(xml)
logger.info(`[KiwixLibraryService] Rebuilt library XML with ${books.length} book(s).`)
}
async addBook(filename: string): Promise<void> {
const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
const filePath = this.getLibraryFilePath()
let existingBooks: KiwixBook[] = []
try {
const content = await readFile(filePath, 'utf-8')
existingBooks = this._parseExistingBooks(content)
} catch (err: any) {
if (err.code === 'ENOENT') {
// XML doesn't exist yet — rebuild from disk; the completed download is already there
await this.rebuildFromDisk()
return
}
throw err
}
if (existingBooks.some((b) => b.path === containerPath)) {
logger.info(`[KiwixLibraryService] ${zimFilename} already in library, skipping.`)
return
}
const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename)
const meta = this._readZimMetadata(fullPath)
existingBooks.push({
...meta,
id: meta?.id ?? zimFilename.slice(0, -4),
path: containerPath,
title: meta?.title ?? this._filenameToTitle(zimFilename),
})
const xml = this._buildXml(existingBooks)
await this._atomicWrite(xml)
logger.info(`[KiwixLibraryService] Added ${zimFilename} to library XML.`)
}
async removeBook(filename: string): Promise<void> {
const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
const filePath = this.getLibraryFilePath()
let existingBooks: KiwixBook[] = []
try {
const content = await readFile(filePath, 'utf-8')
existingBooks = this._parseExistingBooks(content)
} catch (err: any) {
if (err.code === 'ENOENT') {
logger.warn(`[KiwixLibraryService] Library XML not found, nothing to remove.`)
return
}
throw err
}
const filtered = existingBooks.filter((b) => b.path !== containerPath)
if (filtered.length === existingBooks.length) {
logger.info(`[KiwixLibraryService] ${zimFilename} not found in library, nothing to remove.`)
return
}
const xml = this._buildXml(filtered)
await this._atomicWrite(xml)
logger.info(`[KiwixLibraryService] Removed ${zimFilename} from library XML.`)
}
}

39
admin/app/services/zim_service.ts
View File

@ -25,6 +25,7 @@ import InstalledResource from '#models/installed_resource'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { CollectionManifestService } from './collection_manifest_service.js'
import { KiwixLibraryService } from './kiwix_library_service.js'
import type { CategoryWithStatus } from '../../types/collections.js'
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
@ -260,6 +261,17 @@ export class ZimService {
await this.onWikipediaDownloadComplete(url, true)
}
}
// Update the kiwix library XML after all downloaded ZIM files are in place.
// This covers all ZIM types including Wikipedia. Rebuilding once from disk
// avoids repeated XML parse/write cycles and reduces the chance of write races
// when multiple download jobs complete concurrently.
const kiwixLibraryService = new KiwixLibraryService()
try {
await kiwixLibraryService.rebuildFromDisk()
} catch (err) {
logger.error('[ZimService] Failed to rebuild kiwix library from disk:', err)
}
if (restart) {
// Check if there are any remaining ZIM download jobs before restarting
@ -289,13 +301,20 @@ export class ZimService {
if (hasRemainingZimJobs) {
logger.info('[ZimService] Skipping container restart - more ZIM downloads pending')
} else {
// Restart KIWIX container to pick up new ZIM file
logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container')
await this.dockerService
.affectContainer(SERVICE_NAMES.KIWIX, 'restart')
.catch((error) => {
logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
})
// If kiwix is already running in library mode, --monitorLibrary will pick up
// the XML change automatically — no restart needed.
const isLegacy = await this.dockerService.isKiwixOnLegacyConfig()
if (!isLegacy) {
logger.info('[ZimService] Kiwix is in library mode — XML updated, no container restart needed.')
} else {
// Legacy config: restart (affectContainer will trigger migration instead)
logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container')
await this.dockerService
.affectContainer(SERVICE_NAMES.KIWIX, 'restart')
.catch((error) => {
logger.error(`[ZimService] Failed to restart KIWIX container:`, error)
})
}
}
}
@ -353,6 +372,12 @@ export class ZimService {
await deleteFileIfExists(fullPath)
// Remove from kiwix library XML so --monitorLibrary stops serving the deleted file
const kiwixLibraryService = new KiwixLibraryService()
await kiwixLibraryService.removeBook(fileName).catch((err) => {
logger.error(`[ZimService] Failed to remove ${fileName} from kiwix library:`, err)
})
// Clean up InstalledResource entry
const parsed = CollectionManifestService.parseZimFilename(fileName)
if (parsed) {

9
admin/app/utils/fs.ts
View File

@ -5,6 +5,7 @@ import { createReadStream } from 'fs'
import { LSBlockDevice, NomadDiskInfoRaw } from '../../types/system.js'
export const ZIM_STORAGE_PATH = '/storage/zim'
export const KIWIX_LIBRARY_XML_PATH = '/storage/zim/kiwix-library.xml'
export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
const entries = await readdir(path, { withFileTypes: true })
@ -49,7 +50,7 @@ export async function listDirectoryContentsRecursive(path: string): Promise<File
export async function ensureDirectoryExists(path: string): Promise<void> {
try {
await stat(path)
} catch (error) {
} catch (error: any) {
if (error.code === 'ENOENT') {
await mkdir(path, { recursive: true })
}
@ -73,7 +74,7 @@ export async function getFile(
return createReadStream(path)
}
return await readFile(path)
} catch (error) {
} catch (error: any) {
if (error.code === 'ENOENT') {
return null
}
@ -90,7 +91,7 @@ export async function getFileStatsIfExists(
size: stats.size,
modifiedTime: stats.mtime,
}
} catch (error) {
} catch (error: any) {
if (error.code === 'ENOENT') {
return null
}
@ -101,7 +102,7 @@ export async function getFileStatsIfExists(
export async function deleteFileIfExists(path: string): Promise<void> {
try {
await unlink(path)
} catch (error) {
} catch (error: any) {
if (error.code !== 'ENOENT') {
throw error
}

2
admin/constants/kiwix.ts Normal file
View File

@ -0,0 +1,2 @@
export const KIWIX_LIBRARY_CMD = '--library /data/kiwix-library.xml --monitorLibrary --address=all'

29
admin/database/migrations/1771100000001_migrate_kiwix_to_library_mode.ts Normal file
View File

@ -0,0 +1,29 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'services'
async up() {
this.defer(async (db) => {
await db
.from(this.tableName)
.where('service_name', 'nomad_kiwix_server')
.whereRaw('`container_command` LIKE ?', ['%*.zim%'])
.update({
container_command: '--library /data/kiwix-library.xml --monitorLibrary --address=all',
})
})
}
async down() {
this.defer(async (db) => {
await db
.from(this.tableName)
.where('service_name', 'nomad_kiwix_server')
.where('container_command', '--library /data/kiwix-library.xml --monitorLibrary --address=all')
.update({
container_command: '*.zim --address=all',
})
})
}
}

3
admin/database/seeders/service_seeder.ts
View File

@ -3,6 +3,7 @@ import { BaseSeeder } from '@adonisjs/lucid/seeders'
import { ModelAttributes } from '@adonisjs/lucid/types/model'
import env from '#start/env'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
export default class ServiceSeeder extends BaseSeeder {
// Use environment variable with fallback to production default
@ -24,7 +25,7 @@ export default class ServiceSeeder extends BaseSeeder {
icon: 'IconBooks',
container_image: 'ghcr.io/kiwix/kiwix-serve:3.8.1',
source_repo: 'https://github.com/kiwix/kiwix-tools',
container_command: '*.zim --address=all',
container_command: KIWIX_LIBRARY_CMD,
container_config: JSON.stringify({
HostConfig: {
RestartPolicy: { Name: 'unless-stopped' },

53
admin/providers/kiwix_migration_provider.ts Normal file
View File

@ -0,0 +1,53 @@
import logger from '@adonisjs/core/services/logger'
import type { ApplicationService } from '@adonisjs/core/types'
/**
* Checks whether the installed kiwix container is still using the legacy glob-pattern
* command (`*.zim --address=all`) and, if so, migrates it to library mode
* (`--library /data/kiwix-library.xml --monitorLibrary --address=all`) automatically.
*
* This provider runs once on every admin startup. After migration the check is a no-op
* (inspects the container and finds the new command).
*/
export default class KiwixMigrationProvider {
constructor(protected app: ApplicationService) {}
async boot() {
// Only run in the web (HTTP server) environment — skip for ace commands and tests
if (this.app.getEnvironment() !== 'web') return
// Defer past synchronous boot so DB connections and all providers are fully ready
setImmediate(async () => {
try {
const Service = (await import('#models/service')).default
const { SERVICE_NAMES } = await import('../constants/service_names.js')
const { DockerService } = await import('#services/docker_service')
const kiwixService = await Service.query()
.where('service_name', SERVICE_NAMES.KIWIX)
.first()
if (!kiwixService?.installed) {
logger.info('[KiwixMigrationProvider] Kiwix not installed — skipping migration check.')
return
}
const dockerService = new DockerService()
const isLegacy = await dockerService.isKiwixOnLegacyConfig()
if (!isLegacy) {
logger.info('[KiwixMigrationProvider] Kiwix is already in library mode — no migration needed.')
return
}
logger.info('[KiwixMigrationProvider] Kiwix on legacy config — running automatic migration to library mode.')
await dockerService.migrateKiwixToLibraryMode()
logger.info('[KiwixMigrationProvider] Startup migration complete.')
} catch (err: any) {
logger.error(`[KiwixMigrationProvider] Startup migration failed: ${err.message}`)
// Non-fatal: the next affectContainer('restart') call will retry via the
// intercept in DockerService.affectContainer().
}
})
}
}