diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 6cab20d..37046d2 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -53,7 +53,8 @@ export default defineConfig({ () => import('@adonisjs/lucid/database_provider'), () => import('@adonisjs/inertia/inertia_provider'), () => import('@adonisjs/transmit/transmit_provider'), - () => import('#providers/map_static_provider') + () => import('#providers/map_static_provider'), + () => import('#providers/kiwix_migration_provider'), ], /* diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 3c44332..ef7f3d1 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -6,18 +6,20 @@ import transmit from '@adonisjs/transmit/services/main' import { doResumableDownloadWithRetry } from '../utils/downloads.js' import { join } from 'path' import { ZIM_STORAGE_PATH } from '../utils/fs.js' +import { KiwixLibraryService } from './kiwix_library_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { exec } from 'child_process' import { promisify } from 'util' // import { readdir } from 'fs/promises' import KVStore from '#models/kv_store' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js' @inject() export class DockerService { public docker: Docker private activeInstallations: Set = new Set() - public static NOMAD_NETWORK = 'project-nomad_default' + public static NOMAD_NETWORK = 'project-nomad_default' constructor() { // Support both Linux (production) and Windows (development with Docker Desktop) @@ -63,6 +65,15 @@ export class DockerService { } if (action === 'restart') { + if (serviceName === SERVICE_NAMES.KIWIX) { + const isLegacy = await this.isKiwixOnLegacyConfig() + if (isLegacy) { + logger.info('[DockerService] Kiwix on legacy glob config — running migration instead of restart.') + await this.migrateKiwixToLibraryMode() + return { success: true, message: 'Kiwix migrated to library mode successfully.' } + } + } + await dockerContainer.restart() return { @@ -91,7 +102,7 @@ export class DockerService { success: false, message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`, } - } catch (error) { + } catch (error: any) { logger.error(`Error starting service ${serviceName}: ${error.message}`) return { success: false, @@ -123,7 +134,7 @@ export class DockerService { service_name: name, status: container.State, })) - } catch (error) { + } catch (error: any) { logger.error(`Error fetching services status: ${error.message}`) return [] } @@ -312,7 +323,7 @@ export class DockerService { `No existing container found, proceeding with installation...` ) } - } catch (error) { + } catch (error: any) { logger.warn(`Error during container cleanup: ${error.message}`) this._broadcast(serviceName, 'cleanup-warning', `Warning during cleanup: ${error.message}`) } @@ -331,7 +342,7 @@ export class DockerService { const volume = this.docker.getVolume(vol.Name) await volume.remove({ force: true }) this._broadcast(serviceName, 'volume-removed', `Removed volume: ${vol.Name}`) - } catch (error) { + } catch (error: any) { logger.warn(`Failed to remove volume ${vol.Name}: ${error.message}`) } } @@ -339,7 +350,7 @@ export class DockerService { if (serviceVolumes.length === 0) { this._broadcast(serviceName, 'no-volumes', `No volumes found to clear`) } - } catch (error) { + } catch (error: any) { logger.warn(`Error during volume cleanup: ${error.message}`) this._broadcast( serviceName, @@ -367,7 +378,7 @@ export class DockerService { success: true, message: `Service ${serviceName} force reinstall initiated successfully. You can receive updates via server-sent events.`, } - } catch (error) { + } catch (error: any) { logger.error(`Force reinstall failed for ${serviceName}: ${error.message}`) await this._cleanupFailedInstallation(serviceName) return { @@ -583,7 +594,7 @@ export class DockerService { 'completed', `Service ${service.service_name} installation completed successfully.` ) - } catch (error) { + } catch (error: any) { this._broadcast( service.service_name, 'error', @@ -599,7 +610,7 @@ export class DockerService { try { const containers = await this.docker.listContainers({ all: true }) return containers.some((container) => container.Names.includes(`/${serviceName}`)) - } catch (error) { + } catch (error: any) { logger.error(`Error checking if service container exists: ${error.message}`) return false } @@ -619,7 +630,7 @@ export class DockerService { await dockerContainer.remove({ force: true }) return { success: true, message: `Service ${serviceName} container removed successfully` } - } catch (error) { + } catch (error: any) { logger.error(`Error removing service container: ${error.message}`) return { success: false, @@ -667,7 +678,12 @@ export class DockerService { 'preinstall', `Downloaded Wikipedia ZIM file to ${filepath}` ) - } catch (error) { + + // Generate the initial kiwix library XML before the container is created + const kiwixLibraryService = new KiwixLibraryService() + await kiwixLibraryService.rebuildFromDisk() + this._broadcast(SERVICE_NAMES.KIWIX, 'preinstall', 'Generated kiwix library XML.') + } catch (error: any) { this._broadcast( SERVICE_NAMES.KIWIX, 'preinstall-error', @@ -690,13 +706,121 @@ export class DockerService { await this._removeServiceContainer(serviceName) logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`) - } catch (error) { + } catch (error: any) { logger.error( `[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}` ) } } + /** + * Checks whether the running kiwix container is using the legacy glob-pattern command + * (`*.zim --address=all`) rather than the library-file command. Used to detect containers + * that need to be migrated to library mode. + */ + async isKiwixOnLegacyConfig(): Promise { + try { + const containers = await this.docker.listContainers({ all: true }) + const info = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`)) + if (!info) return false + + const inspected = await this.docker.getContainer(info.Id).inspect() + const cmd: string[] = inspected.Config?.Cmd ?? [] + return cmd.some((arg) => arg.includes('*.zim')) + } catch (err: any) { + logger.warn(`[DockerService] Could not inspect kiwix container: ${err.message}`) + return false + } + } + + /** + * Migrates the kiwix container from legacy glob mode (`*.zim`) to library mode + * (`--library /data/kiwix-library.xml --monitorLibrary`). + * + * This is a non-destructive recreation: ZIM files and volumes are preserved. + * The container is stopped, removed, and recreated with the correct library-mode command. + * This function is authoritative: it writes the correct command to the DB itself rather than + * trusting the DB to have been pre-updated by a separate migration. + */ + async migrateKiwixToLibraryMode(): Promise { + if (this.activeInstallations.has(SERVICE_NAMES.KIWIX)) { + logger.warn('[DockerService] Kiwix migration already in progress, skipping duplicate call.') + return + } + + this.activeInstallations.add(SERVICE_NAMES.KIWIX) + + try { + // Step 1: Build/update the XML from current disk state + this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Migrating kiwix to library mode...') + const kiwixLibraryService = new KiwixLibraryService() + await kiwixLibraryService.rebuildFromDisk() + this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Built kiwix library XML from existing ZIM files.') + + // Step 2: Stop and remove old container (leave ZIM volumes intact) + const containers = await this.docker.listContainers({ all: true }) + const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`)) + if (containerInfo) { + const oldContainer = this.docker.getContainer(containerInfo.Id) + if (containerInfo.State === 'running') { + await oldContainer.stop({ t: 10 }).catch((e: any) => + logger.warn(`[DockerService] Kiwix stop warning during migration: ${e.message}`) + ) + } + await oldContainer.remove({ force: true }).catch((e: any) => + logger.warn(`[DockerService] Kiwix remove warning during migration: ${e.message}`) + ) + } + + // Step 3: Read the service record and authoritatively set the correct command. + // Do NOT rely on prior DB state — we write container_command here so the record + // stays consistent regardless of whether the DB migration ran. + const service = await Service.query().where('service_name', SERVICE_NAMES.KIWIX).first() + if (!service) { + throw new Error('Kiwix service record not found in DB during migration') + } + + service.container_command = KIWIX_LIBRARY_CMD + service.installed = false + service.installation_status = 'installing' + await service.save() + + const containerConfig = this._parseContainerConfig(service.container_config) + + // Step 4: Recreate container directly (skipping _createContainer to avoid re-downloading + // the bootstrap ZIM — ZIM files already exist on disk) + this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Recreating kiwix container with library mode config...') + const newContainer = await this.docker.createContainer({ + Image: service.container_image, + name: service.service_name, + HostConfig: containerConfig?.HostConfig ?? {}, + ...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }), + Cmd: KIWIX_LIBRARY_CMD.split(' '), + ...(process.env.NODE_ENV === 'production' && { + NetworkingConfig: { + EndpointsConfig: { + [DockerService.NOMAD_NETWORK]: {}, + }, + }, + }), + }) + + await newContainer.start() + + service.installed = true + service.installation_status = 'idle' + await service.save() + this.activeInstallations.delete(SERVICE_NAMES.KIWIX) + + this._broadcast(SERVICE_NAMES.KIWIX, 'migrated', 'Kiwix successfully migrated to library mode.') + logger.info('[DockerService] Kiwix migration to library mode complete.') + } catch (error: any) { + logger.error(`[DockerService] Kiwix migration failed: ${error.message}`) + await this._cleanupFailedInstallation(SERVICE_NAMES.KIWIX) + throw error + } + } + /** * Detect GPU type and toolkit availability. * Primary: Check Docker runtimes via docker.info() (works from inside containers). @@ -713,7 +837,7 @@ export class DockerService { await this._persistGPUType('nvidia') return { type: 'nvidia' } } - } catch (error) { + } catch (error: any) { logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`) } @@ -730,7 +854,7 @@ export class DockerService { logger.warn('[DockerService] NVIDIA GPU detected via lspci but NVIDIA Container Toolkit is not installed') return { type: 'none', toolkitMissing: true } } - } catch (error) { + } catch (error: any) { // lspci not available (likely inside Docker container), continue } @@ -745,7 +869,7 @@ export class DockerService { await this._persistGPUType('amd') return { type: 'amd' } } - } catch (error) { + } catch (error: any) { // lspci not available, continue } @@ -764,7 +888,7 @@ export class DockerService { logger.info('[DockerService] No GPU detected') return { type: 'none' } - } catch (error) { + } catch (error: any) { logger.warn(`[DockerService] Error detecting GPU type: ${error.message}`) return { type: 'none' } } @@ -774,7 +898,7 @@ export class DockerService { try { await KVStore.setValue('gpu.type', type) logger.info(`[DockerService] Persisted GPU type '${type}' to KV store`) - } catch (error) { + } catch (error: any) { logger.warn(`[DockerService] Failed to persist GPU type: ${error.message}`) } } @@ -969,7 +1093,7 @@ export class DockerService { let newContainer: any try { newContainer = await this.docker.createContainer(newContainerConfig) - } catch (createError) { + } catch (createError: any) { // Rollback: rename old container back this._broadcast(serviceName, 'update-rollback', `Failed to create new container: ${createError.message}. Rolling back...`) const rollbackContainer = this.docker.getContainer((await this.docker.listContainers({ all: true })).find((c) => c.Names.includes(`/${oldName}`))!.Id) @@ -1042,7 +1166,7 @@ export class DockerService { message: `Update failed: new container did not stay running. Rolled back to previous version.`, } } - } catch (error) { + } catch (error: any) { this.activeInstallations.delete(serviceName) this._broadcast( serviceName, @@ -1077,7 +1201,7 @@ export class DockerService { } return JSON.parse(toParse) - } catch (error) { + } catch (error: any) { logger.error(`Failed to parse container configuration: ${error.message}`) throw new Error(`Invalid container configuration: ${error.message}`) } @@ -1094,7 +1218,7 @@ export class DockerService { // Check if any image has a RepoTag that matches the requested image return images.some((image) => image.RepoTags && image.RepoTags.includes(imageName)) - } catch (error) { + } catch (error: any) { logger.warn(`Error checking if image exists: ${error.message}`) // If run into an error, assume the image does not exist return false diff --git a/admin/app/services/kiwix_library_service.ts b/admin/app/services/kiwix_library_service.ts new file mode 100644 index 0000000..28e6576 --- /dev/null +++ b/admin/app/services/kiwix_library_service.ts @@ -0,0 +1,285 @@ +import { XMLBuilder, XMLParser } from 'fast-xml-parser' +import { readFile, writeFile, rename, readdir } from 'fs/promises' +import { join } from 'path' +import { Archive } from '@openzim/libzim' +import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists } from '../utils/fs.js' +import logger from '@adonisjs/core/services/logger' +import { randomUUID } from 'node:crypto' + +const CONTAINER_DATA_PATH = '/data' +const XML_DECLARATION = '\n' + +interface KiwixBook { + id: string + path: string + title: string + description?: string + language?: string + creator?: string + publisher?: string + name?: string + flavour?: string + tags?: string + faviconMimeType?: string + favicon?: string + date?: string + articleCount?: number + mediaCount?: number + size?: number +} + +export class KiwixLibraryService { + getLibraryFilePath(): string { + return join(process.cwd(), KIWIX_LIBRARY_XML_PATH) + } + + containerLibraryPath(): string { + return '/data/kiwix-library.xml' + } + + private _filenameToTitle(filename: string): string { + const withoutExt = filename.endsWith('.zim') ? filename.slice(0, -4) : filename + const parts = withoutExt.split('_') + // Drop last segment if it looks like a date (YYYY-MM) + const lastPart = parts[parts.length - 1] + const isDate = /^\d{4}-\d{2}$/.test(lastPart) + const titleParts = isDate && parts.length > 1 ? parts.slice(0, -1) : parts + return titleParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(' ') + } + + /** + * Reads all kiwix-manage-compatible metadata from a ZIM file, including the internal UUID, + * rich text fields, and the base64-encoded favicon. Kiwix-serve uses the UUID for OPDS + * catalog entries and illustration URLs (/catalog/v2/illustration/{uuid}). + * + * Returns null on any error so callers can fall back gracefully. + */ + private _readZimMetadata(zimFilePath: string): Partial | null { + try { + const archive = new Archive(zimFilePath) + + const getMeta = (key: string): string | undefined => { + try { + return archive.getMetadata(key) || undefined + } catch { + return undefined + } + } + + let favicon: string | undefined + let faviconMimeType: string | undefined + try { + if (archive.illustrationSizes.size > 0) { + const size = archive.illustrationSizes.has(48) + ? 48 + : ([...archive.illustrationSizes][0] as number) + const item = archive.getIllustrationItem(size) + favicon = item.data.data.toString('base64') + faviconMimeType = item.mimetype || undefined + } + } catch { + // ZIM has no illustration — that's fine + } + + const rawFilesize = + typeof archive.filesize === 'bigint' ? Number(archive.filesize) : archive.filesize + + return { + id: archive.uuid || undefined, + title: getMeta('Title'), + description: getMeta('Description'), + language: getMeta('Language'), + creator: getMeta('Creator'), + publisher: getMeta('Publisher'), + name: getMeta('Name'), + flavour: getMeta('Flavour'), + tags: getMeta('Tags'), + date: getMeta('Date'), + articleCount: archive.articleCount, + mediaCount: archive.mediaCount, + size: Math.floor(rawFilesize / 1024), + favicon, + faviconMimeType, + } + } catch { + return null + } + } + + private _buildXml(books: KiwixBook[]): string { + const builder = new XMLBuilder({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + format: true, + suppressEmptyNode: false, + }) + + const obj: Record = { + library: { + '@_version': '20110515', + ...(books.length > 0 && { + book: books.map((b) => ({ + '@_id': b.id, + '@_path': b.path, + '@_title': b.title, + ...(b.description !== undefined && { '@_description': b.description }), + ...(b.language !== undefined && { '@_language': b.language }), + ...(b.creator !== undefined && { '@_creator': b.creator }), + ...(b.publisher !== undefined && { '@_publisher': b.publisher }), + ...(b.name !== undefined && { '@_name': b.name }), + ...(b.flavour !== undefined && { '@_flavour': b.flavour }), + ...(b.tags !== undefined && { '@_tags': b.tags }), + ...(b.faviconMimeType !== undefined && { '@_faviconMimeType': b.faviconMimeType }), + ...(b.favicon !== undefined && { '@_favicon': b.favicon }), + ...(b.date !== undefined && { '@_date': b.date }), + ...(b.articleCount !== undefined && { '@_articleCount': b.articleCount }), + ...(b.mediaCount !== undefined && { '@_mediaCount': b.mediaCount }), + ...(b.size !== undefined && { '@_size': b.size }), + })), + }), + }, + } + + return XML_DECLARATION + builder.build(obj) + } + + private async _atomicWrite(content: string): Promise { + const filePath = this.getLibraryFilePath() + const tmpPath = `${filePath}.tmp.${randomUUID()}` + await writeFile(tmpPath, content, 'utf-8') + await rename(tmpPath, filePath) + } + + private _parseExistingBooks(xmlContent: string): KiwixBook[] { + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + isArray: (name) => name === 'book', + }) + + const parsed = parser.parse(xmlContent) + const books: any[] = parsed?.library?.book ?? [] + + return books + .map((b) => ({ + id: b['@_id'] ?? '', + path: b['@_path'] ?? '', + title: b['@_title'] ?? '', + description: b['@_description'], + language: b['@_language'], + creator: b['@_creator'], + publisher: b['@_publisher'], + name: b['@_name'], + flavour: b['@_flavour'], + tags: b['@_tags'], + faviconMimeType: b['@_faviconMimeType'], + favicon: b['@_favicon'], + date: b['@_date'], + articleCount: + b['@_articleCount'] !== undefined ? Number(b['@_articleCount']) : undefined, + mediaCount: b['@_mediaCount'] !== undefined ? Number(b['@_mediaCount']) : undefined, + size: b['@_size'] !== undefined ? Number(b['@_size']) : undefined, + })) + .filter((b) => b.id && b.path) + } + + async rebuildFromDisk(opts?: { excludeFilenames?: string[] }): Promise { + const dirPath = join(process.cwd(), ZIM_STORAGE_PATH) + await ensureDirectoryExists(dirPath) + + let entries: string[] = [] + try { + entries = await readdir(dirPath) + } catch { + entries = [] + } + + const excludeSet = new Set(opts?.excludeFilenames ?? []) + const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name)) + + const books: KiwixBook[] = zimFiles.map((filename) => { + const meta = this._readZimMetadata(join(dirPath, filename)) + const containerPath = `${CONTAINER_DATA_PATH}/${filename}` + return { + ...meta, + // Override fields that must be derived locally, not from ZIM metadata + id: meta?.id ?? filename.slice(0, -4), + path: containerPath, + title: meta?.title ?? this._filenameToTitle(filename), + } + }) + + const xml = this._buildXml(books) + await this._atomicWrite(xml) + logger.info(`[KiwixLibraryService] Rebuilt library XML with ${books.length} book(s).`) + } + + async addBook(filename: string): Promise { + const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim` + const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}` + + const filePath = this.getLibraryFilePath() + let existingBooks: KiwixBook[] = [] + + try { + const content = await readFile(filePath, 'utf-8') + existingBooks = this._parseExistingBooks(content) + } catch (err: any) { + if (err.code === 'ENOENT') { + // XML doesn't exist yet — rebuild from disk; the completed download is already there + await this.rebuildFromDisk() + return + } + throw err + } + + if (existingBooks.some((b) => b.path === containerPath)) { + logger.info(`[KiwixLibraryService] ${zimFilename} already in library, skipping.`) + return + } + + const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename) + const meta = this._readZimMetadata(fullPath) + + existingBooks.push({ + ...meta, + id: meta?.id ?? zimFilename.slice(0, -4), + path: containerPath, + title: meta?.title ?? this._filenameToTitle(zimFilename), + }) + + const xml = this._buildXml(existingBooks) + await this._atomicWrite(xml) + logger.info(`[KiwixLibraryService] Added ${zimFilename} to library XML.`) + } + + async removeBook(filename: string): Promise { + const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim` + const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}` + + const filePath = this.getLibraryFilePath() + let existingBooks: KiwixBook[] = [] + + try { + const content = await readFile(filePath, 'utf-8') + existingBooks = this._parseExistingBooks(content) + } catch (err: any) { + if (err.code === 'ENOENT') { + logger.warn(`[KiwixLibraryService] Library XML not found, nothing to remove.`) + return + } + throw err + } + + const filtered = existingBooks.filter((b) => b.path !== containerPath) + + if (filtered.length === existingBooks.length) { + logger.info(`[KiwixLibraryService] ${zimFilename} not found in library, nothing to remove.`) + return + } + + const xml = this._buildXml(filtered) + await this._atomicWrite(xml) + logger.info(`[KiwixLibraryService] Removed ${zimFilename} from library XML.`) + } +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index bc587aa..bee4309 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -25,6 +25,7 @@ import InstalledResource from '#models/installed_resource' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' +import { KiwixLibraryService } from './kiwix_library_service.js' import type { CategoryWithStatus } from '../../types/collections.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] @@ -260,6 +261,17 @@ export class ZimService { await this.onWikipediaDownloadComplete(url, true) } } + + // Update the kiwix library XML after all downloaded ZIM files are in place. + // This covers all ZIM types including Wikipedia. Rebuilding once from disk + // avoids repeated XML parse/write cycles and reduces the chance of write races + // when multiple download jobs complete concurrently. + const kiwixLibraryService = new KiwixLibraryService() + try { + await kiwixLibraryService.rebuildFromDisk() + } catch (err) { + logger.error('[ZimService] Failed to rebuild kiwix library from disk:', err) + } if (restart) { // Check if there are any remaining ZIM download jobs before restarting @@ -289,13 +301,20 @@ export class ZimService { if (hasRemainingZimJobs) { logger.info('[ZimService] Skipping container restart - more ZIM downloads pending') } else { - // Restart KIWIX container to pick up new ZIM file - logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container') - await this.dockerService - .affectContainer(SERVICE_NAMES.KIWIX, 'restart') - .catch((error) => { - logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. - }) + // If kiwix is already running in library mode, --monitorLibrary will pick up + // the XML change automatically — no restart needed. + const isLegacy = await this.dockerService.isKiwixOnLegacyConfig() + if (!isLegacy) { + logger.info('[ZimService] Kiwix is in library mode — XML updated, no container restart needed.') + } else { + // Legacy config: restart (affectContainer will trigger migration instead) + logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container') + await this.dockerService + .affectContainer(SERVICE_NAMES.KIWIX, 'restart') + .catch((error) => { + logger.error(`[ZimService] Failed to restart KIWIX container:`, error) + }) + } } } @@ -353,6 +372,12 @@ export class ZimService { await deleteFileIfExists(fullPath) + // Remove from kiwix library XML so --monitorLibrary stops serving the deleted file + const kiwixLibraryService = new KiwixLibraryService() + await kiwixLibraryService.removeBook(fileName).catch((err) => { + logger.error(`[ZimService] Failed to remove ${fileName} from kiwix library:`, err) + }) + // Clean up InstalledResource entry const parsed = CollectionManifestService.parseZimFilename(fileName) if (parsed) { diff --git a/admin/app/utils/fs.ts b/admin/app/utils/fs.ts index 8c459b5..c3ee398 100644 --- a/admin/app/utils/fs.ts +++ b/admin/app/utils/fs.ts @@ -5,6 +5,7 @@ import { createReadStream } from 'fs' import { LSBlockDevice, NomadDiskInfoRaw } from '../../types/system.js' export const ZIM_STORAGE_PATH = '/storage/zim' +export const KIWIX_LIBRARY_XML_PATH = '/storage/zim/kiwix-library.xml' export async function listDirectoryContents(path: string): Promise { const entries = await readdir(path, { withFileTypes: true }) @@ -49,7 +50,7 @@ export async function listDirectoryContentsRecursive(path: string): Promise { try { await stat(path) - } catch (error) { + } catch (error: any) { if (error.code === 'ENOENT') { await mkdir(path, { recursive: true }) } @@ -73,7 +74,7 @@ export async function getFile( return createReadStream(path) } return await readFile(path) - } catch (error) { + } catch (error: any) { if (error.code === 'ENOENT') { return null } @@ -90,7 +91,7 @@ export async function getFileStatsIfExists( size: stats.size, modifiedTime: stats.mtime, } - } catch (error) { + } catch (error: any) { if (error.code === 'ENOENT') { return null } @@ -101,7 +102,7 @@ export async function getFileStatsIfExists( export async function deleteFileIfExists(path: string): Promise { try { await unlink(path) - } catch (error) { + } catch (error: any) { if (error.code !== 'ENOENT') { throw error } diff --git a/admin/constants/kiwix.ts b/admin/constants/kiwix.ts new file mode 100644 index 0000000..bc2504c --- /dev/null +++ b/admin/constants/kiwix.ts @@ -0,0 +1,2 @@ + +export const KIWIX_LIBRARY_CMD = '--library /data/kiwix-library.xml --monitorLibrary --address=all' \ No newline at end of file diff --git a/admin/database/migrations/1771100000001_migrate_kiwix_to_library_mode.ts b/admin/database/migrations/1771100000001_migrate_kiwix_to_library_mode.ts new file mode 100644 index 0000000..909870d --- /dev/null +++ b/admin/database/migrations/1771100000001_migrate_kiwix_to_library_mode.ts @@ -0,0 +1,29 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'services' + + async up() { + this.defer(async (db) => { + await db + .from(this.tableName) + .where('service_name', 'nomad_kiwix_server') + .whereRaw('`container_command` LIKE ?', ['%*.zim%']) + .update({ + container_command: '--library /data/kiwix-library.xml --monitorLibrary --address=all', + }) + }) + } + + async down() { + this.defer(async (db) => { + await db + .from(this.tableName) + .where('service_name', 'nomad_kiwix_server') + .where('container_command', '--library /data/kiwix-library.xml --monitorLibrary --address=all') + .update({ + container_command: '*.zim --address=all', + }) + }) + } +} diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index dd93771..4bf1087 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -3,6 +3,7 @@ import { BaseSeeder } from '@adonisjs/lucid/seeders' import { ModelAttributes } from '@adonisjs/lucid/types/model' import env from '#start/env' import { SERVICE_NAMES } from '../../constants/service_names.js' +import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js' export default class ServiceSeeder extends BaseSeeder { // Use environment variable with fallback to production default @@ -24,7 +25,7 @@ export default class ServiceSeeder extends BaseSeeder { icon: 'IconBooks', container_image: 'ghcr.io/kiwix/kiwix-serve:3.8.1', source_repo: 'https://github.com/kiwix/kiwix-tools', - container_command: '*.zim --address=all', + container_command: KIWIX_LIBRARY_CMD, container_config: JSON.stringify({ HostConfig: { RestartPolicy: { Name: 'unless-stopped' }, diff --git a/admin/providers/kiwix_migration_provider.ts b/admin/providers/kiwix_migration_provider.ts new file mode 100644 index 0000000..5e010b3 --- /dev/null +++ b/admin/providers/kiwix_migration_provider.ts @@ -0,0 +1,53 @@ +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' + +/** + * Checks whether the installed kiwix container is still using the legacy glob-pattern + * command (`*.zim --address=all`) and, if so, migrates it to library mode + * (`--library /data/kiwix-library.xml --monitorLibrary --address=all`) automatically. + * + * This provider runs once on every admin startup. After migration the check is a no-op + * (inspects the container and finds the new command). + */ +export default class KiwixMigrationProvider { + constructor(protected app: ApplicationService) {} + + async boot() { + // Only run in the web (HTTP server) environment — skip for ace commands and tests + if (this.app.getEnvironment() !== 'web') return + + // Defer past synchronous boot so DB connections and all providers are fully ready + setImmediate(async () => { + try { + const Service = (await import('#models/service')).default + const { SERVICE_NAMES } = await import('../constants/service_names.js') + const { DockerService } = await import('#services/docker_service') + + const kiwixService = await Service.query() + .where('service_name', SERVICE_NAMES.KIWIX) + .first() + + if (!kiwixService?.installed) { + logger.info('[KiwixMigrationProvider] Kiwix not installed — skipping migration check.') + return + } + + const dockerService = new DockerService() + const isLegacy = await dockerService.isKiwixOnLegacyConfig() + + if (!isLegacy) { + logger.info('[KiwixMigrationProvider] Kiwix is already in library mode — no migration needed.') + return + } + + logger.info('[KiwixMigrationProvider] Kiwix on legacy config — running automatic migration to library mode.') + await dockerService.migrateKiwixToLibraryMode() + logger.info('[KiwixMigrationProvider] Startup migration complete.') + } catch (err: any) { + logger.error(`[KiwixMigrationProvider] Startup migration failed: ${err.message}`) + // Non-fatal: the next affectContainer('restart') call will retry via the + // intercept in DockerService.affectContainer(). + } + }) + } +}