feat(Kiwix): migrate to Kiwix library mode for improved stability (#622)

admin/adonisrc.ts

@@ -53,7 +53,8 @@ export default defineConfig({
     () => import('@adonisjs/lucid/database_provider'),
     () => import('@adonisjs/inertia/inertia_provider'),
     () => import('@adonisjs/transmit/transmit_provider'),
-    () => import('#providers/map_static_provider')
+    () => import('#providers/map_static_provider'),
+    () => import('#providers/kiwix_migration_provider'),
   ],
 
   /*

admin/app/services/docker_service.ts

@@ -6,12 +6,14 @@ import transmit from '@adonisjs/transmit/services/main'
 import { doResumableDownloadWithRetry } from '../utils/downloads.js'
 import { join } from 'path'
 import { ZIM_STORAGE_PATH } from '../utils/fs.js'
+import { KiwixLibraryService } from './kiwix_library_service.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { exec } from 'child_process'
 import { promisify } from 'util'
 // import { readdir } from 'fs/promises'
 import KVStore from '#models/kv_store'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
 
 @inject()
 export class DockerService {
@@ -63,6 +65,15 @@ export class DockerService {
       }
 
       if (action === 'restart') {
+        if (serviceName === SERVICE_NAMES.KIWIX) {
+          const isLegacy = await this.isKiwixOnLegacyConfig()
+          if (isLegacy) {
+            logger.info('[DockerService] Kiwix on legacy glob config — running migration instead of restart.')
+            await this.migrateKiwixToLibraryMode()
+            return { success: true, message: 'Kiwix migrated to library mode successfully.' }
+          }
+        }
+
         await dockerContainer.restart()
 
         return {
@@ -91,7 +102,7 @@ export class DockerService {
         success: false,
         message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`,
       }
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Error starting service ${serviceName}: ${error.message}`)
       return {
         success: false,
@@ -123,7 +134,7 @@ export class DockerService {
         service_name: name,
         status: container.State,
       }))
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Error fetching services status: ${error.message}`)
       return []
     }
@@ -312,7 +323,7 @@ export class DockerService {
             `No existing container found, proceeding with installation...`
           )
         }
-      } catch (error) {
+      } catch (error: any) {
         logger.warn(`Error during container cleanup: ${error.message}`)
         this._broadcast(serviceName, 'cleanup-warning', `Warning during cleanup: ${error.message}`)
       }
@@ -331,7 +342,7 @@ export class DockerService {
             const volume = this.docker.getVolume(vol.Name)
             await volume.remove({ force: true })
             this._broadcast(serviceName, 'volume-removed', `Removed volume: ${vol.Name}`)
-          } catch (error) {
+          } catch (error: any) {
             logger.warn(`Failed to remove volume ${vol.Name}: ${error.message}`)
           }
         }
@@ -339,7 +350,7 @@ export class DockerService {
         if (serviceVolumes.length === 0) {
           this._broadcast(serviceName, 'no-volumes', `No volumes found to clear`)
         }
-      } catch (error) {
+      } catch (error: any) {
         logger.warn(`Error during volume cleanup: ${error.message}`)
         this._broadcast(
           serviceName,
@@ -367,7 +378,7 @@ export class DockerService {
         success: true,
         message: `Service ${serviceName} force reinstall initiated successfully. You can receive updates via server-sent events.`,
       }
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Force reinstall failed for ${serviceName}: ${error.message}`)
       await this._cleanupFailedInstallation(serviceName)
       return {
@@ -583,7 +594,7 @@ export class DockerService {
         'completed',
         `Service ${service.service_name} installation completed successfully.`
       )
-    } catch (error) {
+    } catch (error: any) {
       this._broadcast(
         service.service_name,
         'error',
@@ -599,7 +610,7 @@ export class DockerService {
     try {
       const containers = await this.docker.listContainers({ all: true })
       return containers.some((container) => container.Names.includes(`/${serviceName}`))
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Error checking if service container exists: ${error.message}`)
       return false
     }
@@ -619,7 +630,7 @@ export class DockerService {
       await dockerContainer.remove({ force: true })
 
       return { success: true, message: `Service ${serviceName} container removed successfully` }
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Error removing service container: ${error.message}`)
       return {
         success: false,
@@ -667,7 +678,12 @@ export class DockerService {
         'preinstall',
         `Downloaded Wikipedia ZIM file to ${filepath}`
       )
-    } catch (error) {
+
+      // Generate the initial kiwix library XML before the container is created
+      const kiwixLibraryService = new KiwixLibraryService()
+      await kiwixLibraryService.rebuildFromDisk()
+      this._broadcast(SERVICE_NAMES.KIWIX, 'preinstall', 'Generated kiwix library XML.')
+    } catch (error: any) {
       this._broadcast(
         SERVICE_NAMES.KIWIX,
         'preinstall-error',
@@ -690,13 +706,121 @@ export class DockerService {
       await this._removeServiceContainer(serviceName)
 
       logger.info(`[DockerService] Cleaned up failed installation for ${serviceName}`)
-    } catch (error) {
+    } catch (error: any) {
       logger.error(
         `[DockerService] Failed to cleanup installation for ${serviceName}: ${error.message}`
       )
     }
   }
 
+  /**
+   * Checks whether the running kiwix container is using the legacy glob-pattern command
+   * (`*.zim --address=all`) rather than the library-file command. Used to detect containers
+   * that need to be migrated to library mode.
+   */
+  async isKiwixOnLegacyConfig(): Promise<boolean> {
+    try {
+      const containers = await this.docker.listContainers({ all: true })
+      const info = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
+      if (!info) return false
+
+      const inspected = await this.docker.getContainer(info.Id).inspect()
+      const cmd: string[] = inspected.Config?.Cmd ?? []
+      return cmd.some((arg) => arg.includes('*.zim'))
+    } catch (err: any) {
+      logger.warn(`[DockerService] Could not inspect kiwix container: ${err.message}`)
+      return false
+    }
+  }
+
+  /**
+   * Migrates the kiwix container from legacy glob mode (`*.zim`) to library mode
+   * (`--library /data/kiwix-library.xml --monitorLibrary`).
+   *
+   * This is a non-destructive recreation: ZIM files and volumes are preserved.
+   * The container is stopped, removed, and recreated with the correct library-mode command.
+   * This function is authoritative: it writes the correct command to the DB itself rather than
+   * trusting the DB to have been pre-updated by a separate migration.
+   */
+  async migrateKiwixToLibraryMode(): Promise<void> {
+    if (this.activeInstallations.has(SERVICE_NAMES.KIWIX)) {
+      logger.warn('[DockerService] Kiwix migration already in progress, skipping duplicate call.')
+      return
+    }
+
+    this.activeInstallations.add(SERVICE_NAMES.KIWIX)
+
+    try {
+      // Step 1: Build/update the XML from current disk state
+      this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Migrating kiwix to library mode...')
+      const kiwixLibraryService = new KiwixLibraryService()
+      await kiwixLibraryService.rebuildFromDisk()
+      this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Built kiwix library XML from existing ZIM files.')
+
+      // Step 2: Stop and remove old container (leave ZIM volumes intact)
+      const containers = await this.docker.listContainers({ all: true })
+      const containerInfo = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.KIWIX}`))
+      if (containerInfo) {
+        const oldContainer = this.docker.getContainer(containerInfo.Id)
+        if (containerInfo.State === 'running') {
+          await oldContainer.stop({ t: 10 }).catch((e: any) =>
+            logger.warn(`[DockerService] Kiwix stop warning during migration: ${e.message}`)
+          )
+        }
+        await oldContainer.remove({ force: true }).catch((e: any) =>
+          logger.warn(`[DockerService] Kiwix remove warning during migration: ${e.message}`)
+        )
+      }
+
+      // Step 3: Read the service record and authoritatively set the correct command.
+      // Do NOT rely on prior DB state — we write container_command here so the record
+      // stays consistent regardless of whether the DB migration ran.
+      const service = await Service.query().where('service_name', SERVICE_NAMES.KIWIX).first()
+      if (!service) {
+        throw new Error('Kiwix service record not found in DB during migration')
+      }
+
+      service.container_command = KIWIX_LIBRARY_CMD
+      service.installed = false
+      service.installation_status = 'installing'
+      await service.save()
+
+      const containerConfig = this._parseContainerConfig(service.container_config)
+
+      // Step 4: Recreate container directly (skipping _createContainer to avoid re-downloading
+      // the bootstrap ZIM — ZIM files already exist on disk)
+      this._broadcast(SERVICE_NAMES.KIWIX, 'migrating', 'Recreating kiwix container with library mode config...')
+      const newContainer = await this.docker.createContainer({
+        Image: service.container_image,
+        name: service.service_name,
+        HostConfig: containerConfig?.HostConfig ?? {},
+        ...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
+        Cmd: KIWIX_LIBRARY_CMD.split(' '),
+        ...(process.env.NODE_ENV === 'production' && {
+          NetworkingConfig: {
+            EndpointsConfig: {
+              [DockerService.NOMAD_NETWORK]: {},
+            },
+          },
+        }),
+      })
+
+      await newContainer.start()
+
+      service.installed = true
+      service.installation_status = 'idle'
+      await service.save()
+      this.activeInstallations.delete(SERVICE_NAMES.KIWIX)
+
+      this._broadcast(SERVICE_NAMES.KIWIX, 'migrated', 'Kiwix successfully migrated to library mode.')
+      logger.info('[DockerService] Kiwix migration to library mode complete.')
+    } catch (error: any) {
+      logger.error(`[DockerService] Kiwix migration failed: ${error.message}`)
+      await this._cleanupFailedInstallation(SERVICE_NAMES.KIWIX)
+      throw error
+    }
+  }
+
   /**
    * Detect GPU type and toolkit availability.
    * Primary: Check Docker runtimes via docker.info() (works from inside containers).
@@ -713,7 +837,7 @@ export class DockerService {
           await this._persistGPUType('nvidia')
           return { type: 'nvidia' }
         }
-      } catch (error) {
+      } catch (error: any) {
         logger.warn(`[DockerService] Could not query Docker info for GPU runtimes: ${error.message}`)
       }
 
@@ -730,7 +854,7 @@ export class DockerService {
           logger.warn('[DockerService] NVIDIA GPU detected via lspci but NVIDIA Container Toolkit is not installed')
           return { type: 'none', toolkitMissing: true }
         }
-      } catch (error) {
+      } catch (error: any) {
         // lspci not available (likely inside Docker container), continue
       }
 
@@ -745,7 +869,7 @@ export class DockerService {
           await this._persistGPUType('amd')
           return { type: 'amd' }
         }
-      } catch (error) {
+      } catch (error: any) {
         // lspci not available, continue
       }
 
@@ -764,7 +888,7 @@ export class DockerService {
 
       logger.info('[DockerService] No GPU detected')
       return { type: 'none' }
-    } catch (error) {
+    } catch (error: any) {
       logger.warn(`[DockerService] Error detecting GPU type: ${error.message}`)
       return { type: 'none' }
     }
@@ -774,7 +898,7 @@ export class DockerService {
     try {
       await KVStore.setValue('gpu.type', type)
       logger.info(`[DockerService] Persisted GPU type '${type}' to KV store`)
-    } catch (error) {
+    } catch (error: any) {
       logger.warn(`[DockerService] Failed to persist GPU type: ${error.message}`)
     }
   }
@@ -969,7 +1093,7 @@ export class DockerService {
       let newContainer: any
       try {
         newContainer = await this.docker.createContainer(newContainerConfig)
-      } catch (createError) {
+      } catch (createError: any) {
         // Rollback: rename old container back
         this._broadcast(serviceName, 'update-rollback', `Failed to create new container: ${createError.message}. Rolling back...`)
         const rollbackContainer = this.docker.getContainer((await this.docker.listContainers({ all: true })).find((c) => c.Names.includes(`/${oldName}`))!.Id)
@@ -1042,7 +1166,7 @@ export class DockerService {
           message: `Update failed: new container did not stay running. Rolled back to previous version.`,
         }
       }
-    } catch (error) {
+    } catch (error: any) {
       this.activeInstallations.delete(serviceName)
       this._broadcast(
         serviceName,
@@ -1077,7 +1201,7 @@ export class DockerService {
       }
 
       return JSON.parse(toParse)
-    } catch (error) {
+    } catch (error: any) {
       logger.error(`Failed to parse container configuration: ${error.message}`)
       throw new Error(`Invalid container configuration: ${error.message}`)
     }
@@ -1094,7 +1218,7 @@ export class DockerService {
 
       // Check if any image has a RepoTag that matches the requested image
       return images.some((image) => image.RepoTags && image.RepoTags.includes(imageName))
-    } catch (error) {
+    } catch (error: any) {
       logger.warn(`Error checking if image exists: ${error.message}`)
       // If run into an error, assume the image does not exist
       return false

admin/app/services/kiwix_library_service.ts

@@ -0,0 +1,285 @@
+import { XMLBuilder, XMLParser } from 'fast-xml-parser'
+import { readFile, writeFile, rename, readdir } from 'fs/promises'
+import { join } from 'path'
+import { Archive } from '@openzim/libzim'
+import { KIWIX_LIBRARY_XML_PATH, ZIM_STORAGE_PATH, ensureDirectoryExists } from '../utils/fs.js'
+import logger from '@adonisjs/core/services/logger'
+import { randomUUID } from 'node:crypto'
+
+const CONTAINER_DATA_PATH = '/data'
+const XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>\n'
+
+interface KiwixBook {
+  id: string
+  path: string
+  title: string
+  description?: string
+  language?: string
+  creator?: string
+  publisher?: string
+  name?: string
+  flavour?: string
+  tags?: string
+  faviconMimeType?: string
+  favicon?: string
+  date?: string
+  articleCount?: number
+  mediaCount?: number
+  size?: number
+}
+
+export class KiwixLibraryService {
+  getLibraryFilePath(): string {
+    return join(process.cwd(), KIWIX_LIBRARY_XML_PATH)
+  }
+
+  containerLibraryPath(): string {
+    return '/data/kiwix-library.xml'
+  }
+
+  private _filenameToTitle(filename: string): string {
+    const withoutExt = filename.endsWith('.zim') ? filename.slice(0, -4) : filename
+    const parts = withoutExt.split('_')
+    // Drop last segment if it looks like a date (YYYY-MM)
+    const lastPart = parts[parts.length - 1]
+    const isDate = /^\d{4}-\d{2}$/.test(lastPart)
+    const titleParts = isDate && parts.length > 1 ? parts.slice(0, -1) : parts
+    return titleParts.map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(' ')
+  }
+
+  /**
+   * Reads all kiwix-manage-compatible metadata from a ZIM file, including the internal UUID,
+   * rich text fields, and the base64-encoded favicon. Kiwix-serve uses the UUID for OPDS
+   * catalog entries and illustration URLs (/catalog/v2/illustration/{uuid}).
+   *
+   * Returns null on any error so callers can fall back gracefully.
+   */
+  private _readZimMetadata(zimFilePath: string): Partial<KiwixBook> | null {
+    try {
+      const archive = new Archive(zimFilePath)
+
+      const getMeta = (key: string): string | undefined => {
+        try {
+          return archive.getMetadata(key) || undefined
+        } catch {
+          return undefined
+        }
+      }
+
+      let favicon: string | undefined
+      let faviconMimeType: string | undefined
+      try {
+        if (archive.illustrationSizes.size > 0) {
+          const size = archive.illustrationSizes.has(48)
+            ? 48
+            : ([...archive.illustrationSizes][0] as number)
+          const item = archive.getIllustrationItem(size)
+          favicon = item.data.data.toString('base64')
+          faviconMimeType = item.mimetype || undefined
+        }
+      } catch {
+        // ZIM has no illustration — that's fine
+      }
+
+      const rawFilesize =
+        typeof archive.filesize === 'bigint' ? Number(archive.filesize) : archive.filesize
+
+      return {
+        id: archive.uuid || undefined,
+        title: getMeta('Title'),
+        description: getMeta('Description'),
+        language: getMeta('Language'),
+        creator: getMeta('Creator'),
+        publisher: getMeta('Publisher'),
+        name: getMeta('Name'),
+        flavour: getMeta('Flavour'),
+        tags: getMeta('Tags'),
+        date: getMeta('Date'),
+        articleCount: archive.articleCount,
+        mediaCount: archive.mediaCount,
+        size: Math.floor(rawFilesize / 1024),
+        favicon,
+        faviconMimeType,
+      }
+    } catch {
+      return null
+    }
+  }
+
+  private _buildXml(books: KiwixBook[]): string {
+    const builder = new XMLBuilder({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      format: true,
+      suppressEmptyNode: false,
+    })
+
+    const obj: Record<string, any> = {
+      library: {
+        '@_version': '20110515',
+        ...(books.length > 0 && {
+          book: books.map((b) => ({
+            '@_id': b.id,
+            '@_path': b.path,
+            '@_title': b.title,
+            ...(b.description !== undefined && { '@_description': b.description }),
+            ...(b.language !== undefined && { '@_language': b.language }),
+            ...(b.creator !== undefined && { '@_creator': b.creator }),
+            ...(b.publisher !== undefined && { '@_publisher': b.publisher }),
+            ...(b.name !== undefined && { '@_name': b.name }),
+            ...(b.flavour !== undefined && { '@_flavour': b.flavour }),
+            ...(b.tags !== undefined && { '@_tags': b.tags }),
+            ...(b.faviconMimeType !== undefined && { '@_faviconMimeType': b.faviconMimeType }),
+            ...(b.favicon !== undefined && { '@_favicon': b.favicon }),
+            ...(b.date !== undefined && { '@_date': b.date }),
+            ...(b.articleCount !== undefined && { '@_articleCount': b.articleCount }),
+            ...(b.mediaCount !== undefined && { '@_mediaCount': b.mediaCount }),
+            ...(b.size !== undefined && { '@_size': b.size }),
+          })),
+        }),
+      },
+    }
+
+    return XML_DECLARATION + builder.build(obj)
+  }
+
+  private async _atomicWrite(content: string): Promise<void> {
+    const filePath = this.getLibraryFilePath()
+    const tmpPath = `${filePath}.tmp.${randomUUID()}`
+    await writeFile(tmpPath, content, 'utf-8')
+    await rename(tmpPath, filePath)
+  }
+
+  private _parseExistingBooks(xmlContent: string): KiwixBook[] {
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      isArray: (name) => name === 'book',
+    })
+
+    const parsed = parser.parse(xmlContent)
+    const books: any[] = parsed?.library?.book ?? []
+
+    return books
+      .map((b) => ({
+        id: b['@_id'] ?? '',
+        path: b['@_path'] ?? '',
+        title: b['@_title'] ?? '',
+        description: b['@_description'],
+        language: b['@_language'],
+        creator: b['@_creator'],
+        publisher: b['@_publisher'],
+        name: b['@_name'],
+        flavour: b['@_flavour'],
+        tags: b['@_tags'],
+        faviconMimeType: b['@_faviconMimeType'],
+        favicon: b['@_favicon'],
+        date: b['@_date'],
+        articleCount:
+          b['@_articleCount'] !== undefined ? Number(b['@_articleCount']) : undefined,
+        mediaCount: b['@_mediaCount'] !== undefined ? Number(b['@_mediaCount']) : undefined,
+        size: b['@_size'] !== undefined ? Number(b['@_size']) : undefined,
+      }))
+      .filter((b) => b.id && b.path)
+  }
+
+  async rebuildFromDisk(opts?: { excludeFilenames?: string[] }): Promise<void> {
+    const dirPath = join(process.cwd(), ZIM_STORAGE_PATH)
+    await ensureDirectoryExists(dirPath)
+
+    let entries: string[] = []
+    try {
+      entries = await readdir(dirPath)
+    } catch {
+      entries = []
+    }
+
+    const excludeSet = new Set(opts?.excludeFilenames ?? [])
+    const zimFiles = entries.filter((name) => name.endsWith('.zim') && !excludeSet.has(name))
+
+    const books: KiwixBook[] = zimFiles.map((filename) => {
+      const meta = this._readZimMetadata(join(dirPath, filename))
+      const containerPath = `${CONTAINER_DATA_PATH}/${filename}`
+      return {
+        ...meta,
+        // Override fields that must be derived locally, not from ZIM metadata
+        id: meta?.id ?? filename.slice(0, -4),
+        path: containerPath,
+        title: meta?.title ?? this._filenameToTitle(filename),
+      }
+    })
+
+    const xml = this._buildXml(books)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Rebuilt library XML with ${books.length} book(s).`)
+  }
+
+  async addBook(filename: string): Promise<void> {
+    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
+    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
+
+    const filePath = this.getLibraryFilePath()
+    let existingBooks: KiwixBook[] = []
+
+    try {
+      const content = await readFile(filePath, 'utf-8')
+      existingBooks = this._parseExistingBooks(content)
+    } catch (err: any) {
+      if (err.code === 'ENOENT') {
+        // XML doesn't exist yet — rebuild from disk; the completed download is already there
+        await this.rebuildFromDisk()
+        return
+      }
+      throw err
+    }
+
+    if (existingBooks.some((b) => b.path === containerPath)) {
+      logger.info(`[KiwixLibraryService] ${zimFilename} already in library, skipping.`)
+      return
+    }
+
+    const fullPath = join(process.cwd(), ZIM_STORAGE_PATH, zimFilename)
+    const meta = this._readZimMetadata(fullPath)
+
+    existingBooks.push({
+      ...meta,
+      id: meta?.id ?? zimFilename.slice(0, -4),
+      path: containerPath,
+      title: meta?.title ?? this._filenameToTitle(zimFilename),
+    })
+
+    const xml = this._buildXml(existingBooks)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Added ${zimFilename} to library XML.`)
+  }
+
+  async removeBook(filename: string): Promise<void> {
+    const zimFilename = filename.endsWith('.zim') ? filename : `${filename}.zim`
+    const containerPath = `${CONTAINER_DATA_PATH}/${zimFilename}`
+
+    const filePath = this.getLibraryFilePath()
+    let existingBooks: KiwixBook[] = []
+
+    try {
+      const content = await readFile(filePath, 'utf-8')
+      existingBooks = this._parseExistingBooks(content)
+    } catch (err: any) {
+      if (err.code === 'ENOENT') {
+        logger.warn(`[KiwixLibraryService] Library XML not found, nothing to remove.`)
+        return
+      }
+      throw err
+    }
+
+    const filtered = existingBooks.filter((b) => b.path !== containerPath)
+
+    if (filtered.length === existingBooks.length) {
+      logger.info(`[KiwixLibraryService] ${zimFilename} not found in library, nothing to remove.`)
+      return
+    }
+
+    const xml = this._buildXml(filtered)
+    await this._atomicWrite(xml)
+    logger.info(`[KiwixLibraryService] Removed ${zimFilename} from library XML.`)
+  }
+}

admin/app/services/zim_service.ts

@@ -25,6 +25,7 @@ import InstalledResource from '#models/installed_resource'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { CollectionManifestService } from './collection_manifest_service.js'
+import { KiwixLibraryService } from './kiwix_library_service.js'
 import type { CategoryWithStatus } from '../../types/collections.js'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
@@ -261,6 +262,17 @@ export class ZimService {
       }
     }
     
+    // Update the kiwix library XML after all downloaded ZIM files are in place.
+    // This covers all ZIM types including Wikipedia. Rebuilding once from disk
+    // avoids repeated XML parse/write cycles and reduces the chance of write races
+    // when multiple download jobs complete concurrently.
+    const kiwixLibraryService = new KiwixLibraryService()
+    try {
+      await kiwixLibraryService.rebuildFromDisk()
+    } catch (err) {
+      logger.error('[ZimService] Failed to rebuild kiwix library from disk:', err)
+    }
+
     if (restart) {
       // Check if there are any remaining ZIM download jobs before restarting
       const { QueueService } = await import('./queue_service.js')
@@ -289,15 +301,22 @@ export class ZimService {
       if (hasRemainingZimJobs) {
         logger.info('[ZimService] Skipping container restart - more ZIM downloads pending')
       } else {
-        // Restart KIWIX container to pick up new ZIM file
+        // If kiwix is already running in library mode, --monitorLibrary will pick up
+        // the XML change automatically — no restart needed.
+        const isLegacy = await this.dockerService.isKiwixOnLegacyConfig()
+        if (!isLegacy) {
+          logger.info('[ZimService] Kiwix is in library mode — XML updated, no container restart needed.')
+        } else {
+          // Legacy config: restart (affectContainer will trigger migration instead)
           logger.info('[ZimService] No more ZIM downloads pending - restarting KIWIX container')
           await this.dockerService
             .affectContainer(SERVICE_NAMES.KIWIX, 'restart')
             .catch((error) => {
-            logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
+              logger.error(`[ZimService] Failed to restart KIWIX container:`, error)
             })
         }
       }
+    }
 
     // Create InstalledResource entries for downloaded files
     for (const url of urls) {
@@ -353,6 +372,12 @@ export class ZimService {
 
     await deleteFileIfExists(fullPath)
 
+    // Remove from kiwix library XML so --monitorLibrary stops serving the deleted file
+    const kiwixLibraryService = new KiwixLibraryService()
+    await kiwixLibraryService.removeBook(fileName).catch((err) => {
+      logger.error(`[ZimService] Failed to remove ${fileName} from kiwix library:`, err)
+    })
+
     // Clean up InstalledResource entry
     const parsed = CollectionManifestService.parseZimFilename(fileName)
     if (parsed) {

admin/app/utils/fs.ts

@@ -5,6 +5,7 @@ import { createReadStream } from 'fs'
 import { LSBlockDevice, NomadDiskInfoRaw } from '../../types/system.js'
 
 export const ZIM_STORAGE_PATH = '/storage/zim'
+export const KIWIX_LIBRARY_XML_PATH = '/storage/zim/kiwix-library.xml'
 
 export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
   const entries = await readdir(path, { withFileTypes: true })
@@ -49,7 +50,7 @@ export async function listDirectoryContentsRecursive(path: string): Promise<File
 export async function ensureDirectoryExists(path: string): Promise<void> {
   try {
     await stat(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       await mkdir(path, { recursive: true })
     }
@@ -73,7 +74,7 @@ export async function getFile(
       return createReadStream(path)
     }
     return await readFile(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       return null
     }
@@ -90,7 +91,7 @@ export async function getFileStatsIfExists(
       size: stats.size,
       modifiedTime: stats.mtime,
     }
-  } catch (error) {
+  } catch (error: any) {
     if (error.code === 'ENOENT') {
       return null
     }
@@ -101,7 +102,7 @@ export async function getFileStatsIfExists(
 export async function deleteFileIfExists(path: string): Promise<void> {
   try {
     await unlink(path)
-  } catch (error) {
+  } catch (error: any) {
     if (error.code !== 'ENOENT') {
       throw error
     }

admin/constants/kiwix.ts

@@ -0,0 +1,2 @@
+
+export const KIWIX_LIBRARY_CMD = '--library /data/kiwix-library.xml --monitorLibrary --address=all'

admin/database/migrations/1771100000001_migrate_kiwix_to_library_mode.ts

@@ -0,0 +1,29 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'services'
+
+  async up() {
+    this.defer(async (db) => {
+      await db
+        .from(this.tableName)
+        .where('service_name', 'nomad_kiwix_server')
+        .whereRaw('`container_command` LIKE ?', ['%*.zim%'])
+        .update({
+          container_command: '--library /data/kiwix-library.xml --monitorLibrary --address=all',
+        })
+    })
+  }
+
+  async down() {
+    this.defer(async (db) => {
+      await db
+        .from(this.tableName)
+        .where('service_name', 'nomad_kiwix_server')
+        .where('container_command', '--library /data/kiwix-library.xml --monitorLibrary --address=all')
+        .update({
+          container_command: '*.zim --address=all',
+        })
+    })
+  }
+}

admin/database/seeders/service_seeder.ts

@@ -3,6 +3,7 @@ import { BaseSeeder } from '@adonisjs/lucid/seeders'
 import { ModelAttributes } from '@adonisjs/lucid/types/model'
 import env from '#start/env'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
+import { KIWIX_LIBRARY_CMD } from '../../constants/kiwix.js'
 
 export default class ServiceSeeder extends BaseSeeder {
   // Use environment variable with fallback to production default
@@ -24,7 +25,7 @@ export default class ServiceSeeder extends BaseSeeder {
       icon: 'IconBooks',
       container_image: 'ghcr.io/kiwix/kiwix-serve:3.8.1',
       source_repo: 'https://github.com/kiwix/kiwix-tools',
-      container_command: '*.zim --address=all',
+      container_command: KIWIX_LIBRARY_CMD,
       container_config: JSON.stringify({
         HostConfig: {
           RestartPolicy: { Name: 'unless-stopped' },

admin/providers/kiwix_migration_provider.ts

@@ -0,0 +1,53 @@
+import logger from '@adonisjs/core/services/logger'
+import type { ApplicationService } from '@adonisjs/core/types'
+
+/**
+ * Checks whether the installed kiwix container is still using the legacy glob-pattern
+ * command (`*.zim --address=all`) and, if so, migrates it to library mode
+ * (`--library /data/kiwix-library.xml --monitorLibrary --address=all`) automatically.
+ *
+ * This provider runs once on every admin startup. After migration the check is a no-op
+ * (inspects the container and finds the new command).
+ */
+export default class KiwixMigrationProvider {
+  constructor(protected app: ApplicationService) {}
+
+  async boot() {
+    // Only run in the web (HTTP server) environment — skip for ace commands and tests
+    if (this.app.getEnvironment() !== 'web') return
+
+    // Defer past synchronous boot so DB connections and all providers are fully ready
+    setImmediate(async () => {
+      try {
+        const Service = (await import('#models/service')).default
+        const { SERVICE_NAMES } = await import('../constants/service_names.js')
+        const { DockerService } = await import('#services/docker_service')
+
+        const kiwixService = await Service.query()
+          .where('service_name', SERVICE_NAMES.KIWIX)
+          .first()
+
+        if (!kiwixService?.installed) {
+          logger.info('[KiwixMigrationProvider] Kiwix not installed — skipping migration check.')
+          return
+        }
+
+        const dockerService = new DockerService()
+        const isLegacy = await dockerService.isKiwixOnLegacyConfig()
+
+        if (!isLegacy) {
+          logger.info('[KiwixMigrationProvider] Kiwix is already in library mode — no migration needed.')
+          return
+        }
+
+        logger.info('[KiwixMigrationProvider] Kiwix on legacy config — running automatic migration to library mode.')
+        await dockerService.migrateKiwixToLibraryMode()
+        logger.info('[KiwixMigrationProvider] Startup migration complete.')
+      } catch (err: any) {
+        logger.error(`[KiwixMigrationProvider] Startup migration failed: ${err.message}`)
+        // Non-fatal: the next affectContainer('restart') call will retry via the
+        // intercept in DockerService.affectContainer().
+      }
+    })
+  }
+}