feat: background job overhaul with bullmq

admin/.env.example

@@ -4,7 +4,6 @@ LOG_LEVEL=info
 APP_KEY=some_random_key
 NODE_ENV=development
 SESSION_DRIVER=cookie
-DRIVE_DISK=fs
 DB_HOST=localhost
 DB_PORT=3306
 DB_USER=root

admin/adonisrc.ts

@@ -52,8 +52,8 @@ export default defineConfig({
     () => import('@adonisjs/cors/cors_provider'),
     () => import('@adonisjs/lucid/database_provider'),
     () => import('@adonisjs/inertia/inertia_provider'),
-    () => import('@adonisjs/drive/drive_provider'),
+    () => import('@adonisjs/transmit/transmit_provider'),
-    () => import('@adonisjs/transmit/transmit_provider')
+    () => import('#providers/map_static_provider')
   ],
 
   /*

admin/app/controllers/downloads_controller.ts

@@ -0,0 +1,18 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import { DownloadService } from '#services/download_service'
+import { downloadJobsByFiletypeSchema } from '#validators/download'
+import { inject } from '@adonisjs/core'
+
+@inject()
+export default class DownloadsController {
+  constructor(private downloadService: DownloadService) {}
+
+  async index() {
+    return this.downloadService.listDownloadJobs()
+  }
+
+  async filetype({ request }: HttpContext) {
+    const payload = await request.validateUsing(downloadJobsByFiletypeSchema)
+    return this.downloadService.listDownloadJobs(payload.params.filetype)
+  }
+}

admin/app/controllers/maps_controller.ts

@@ -1,6 +1,6 @@
 import { MapService } from '#services/map_service'
 import {
-  filenameValidator,
+  filenameParamValidator,
   remoteDownloadValidator,
   remoteDownloadValidatorOptional,
 } from '#validators/common'
@@ -53,14 +53,14 @@ export default class MapsController {
   }
 
   async delete({ request, response }: HttpContext) {
-    const payload = await request.validateUsing(filenameValidator)
+    const payload = await request.validateUsing(filenameParamValidator)
 
     try {
-      await this.mapService.delete(payload.filename)
+      await this.mapService.delete(payload.params.filename)
     } catch (error) {
       if (error.message === 'not_found') {
         return response.status(404).send({
-          message: `Map file with key ${payload.filename} not found`,
+          message: `Map file with key ${payload.params.filename} not found`,
         })
       }
       throw error // Re-throw any other errors and let the global error handler catch

admin/app/controllers/zim_controller.ts

@@ -1,7 +1,7 @@
 import { ZimService } from '#services/zim_service'
 import {
   downloadCollectionValidator,
-  filenameValidator,
+  filenameParamValidator,
   remoteDownloadValidator,
 } from '#validators/common'
 import { listRemoteZimValidator } from '#validators/zim'
@@ -24,11 +24,12 @@ export default class ZimController {
 
   async downloadRemote({ request }: HttpContext) {
     const payload = await request.validateUsing(remoteDownloadValidator)
-    const filename = await this.zimService.downloadRemote(payload.url)
+    const { filename, jobId } = await this.zimService.downloadRemote(payload.url)
 
     return {
       message: 'Download started successfully',
       filename,
+      jobId,
       url: payload.url,
     }
   }
@@ -44,10 +45,6 @@ export default class ZimController {
     }
   }
 
-  async listActiveDownloads({}: HttpContext) {
-    return this.zimService.listActiveDownloads()
-  }
-
   async listCuratedCollections({}: HttpContext) {
     return this.zimService.listCuratedCollections()
   }
@@ -58,14 +55,14 @@ export default class ZimController {
   }
 
   async delete({ request, response }: HttpContext) {
-    const payload = await request.validateUsing(filenameValidator)
+    const payload = await request.validateUsing(filenameParamValidator)
 
     try {
-      await this.zimService.delete(payload.filename)
+      await this.zimService.delete(payload.params.filename)
     } catch (error) {
       if (error.message === 'not_found') {
         return response.status(404).send({
-          message: `ZIM file with key ${payload.filename} not found`,
+          message: `ZIM file with key ${payload.params.filename} not found`,
         })
       }
       throw error // Re-throw any other errors and let the global error handler catch

admin/app/jobs/run_download_job.ts

@@ -0,0 +1,107 @@
+import { Job } from 'bullmq'
+import { RunDownloadJobParams } from '../../types/downloads.js'
+import { QueueService } from '#services/queue_service'
+import { doResumableDownload } from '../utils/downloads.js'
+import { createHash } from 'crypto'
+import { DockerService } from '#services/docker_service'
+import { ZimService } from '#services/zim_service'
+
+export class RunDownloadJob {
+  static get queue() {
+    return 'downloads'
+  }
+
+  static get key() {
+    return 'run-download'
+  }
+
+  static getJobId(url: string): string {
+    return createHash('sha256').update(url).digest('hex').slice(0, 16)
+  }
+
+  async handle(job: Job) {
+    const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } =
+      job.data as RunDownloadJobParams
+
+    //   console.log("Simulating delay for job for URL:", url)
+    // await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
+    // console.log("Starting download for URL:", url)
+
+    // // simulate progress updates for demonstration
+    // for (let progress = 0; progress <= 100; progress += 10) {
+    //   await new Promise((resolve) => setTimeout(resolve, 20000)) // Simulate time taken for each progress step
+    //   job.updateProgress(progress)
+    //   console.log(`Job progress for URL ${url}: ${progress}%`)
+    // }
+
+    await doResumableDownload({
+      url,
+      filepath,
+      timeout,
+      allowedMimeTypes,
+      forceNew,
+      onProgress(progress) {
+        const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
+        job.updateProgress(Math.floor(progressPercent))
+      },
+      async onComplete(url) {
+        if (filetype === 'zim') {
+          try {
+            const dockerService = new DockerService()
+            const zimService = new ZimService(dockerService)
+            await zimService.downloadRemoteSuccessCallback([url], true)
+          } catch (error) {
+            console.error(
+              `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
+              error
+            )
+          }
+        }
+        job.updateProgress(100)
+      },
+    })
+
+    return {
+      url,
+      filepath,
+    }
+  }
+
+  static async getByUrl(url: string): Promise<Job | undefined> {
+    const queueService = new QueueService()
+    const queue = queueService.getQueue(this.queue)
+    const jobId = this.getJobId(url)
+    return await queue.getJob(jobId)
+  }
+
+  static async dispatch(params: RunDownloadJobParams) {
+    const queueService = new QueueService()
+    const queue = queueService.getQueue(this.queue)
+    const jobId = this.getJobId(params.url)
+
+    try {
+      const job = await queue.add(this.key, params, {
+        jobId,
+        attempts: 3,
+        backoff: { type: 'exponential', delay: 2000 },
+        removeOnComplete: true,
+      })
+
+      return {
+        job,
+        created: true,
+        message: `Dispatched download job for URL ${params.url}`,
+      }
+    } catch (error) {
+      if (error.message.includes('job already exists')) {
+        const existing = await queue.getJob(jobId)
+        return {
+          job: existing,
+          created: false,
+          message: `Job already exists for URL ${params.url}`,
+        }
+      }
+      throw error
+    }
+  }
+}

admin/app/middleware/maps_static_middleware.ts

@@ -0,0 +1,20 @@
+import type { HttpContext } from '@adonisjs/core/http'
+import type { NextFn } from '@adonisjs/core/types/http'
+import StaticMiddleware from '@adonisjs/static/static_middleware'
+import { AssetsConfig } from '@adonisjs/static/types'
+
+/**
+ * See #providers/map_static_provider.ts for explanation
+ * of why this middleware exists.
+ */
+export default class MapsStaticMiddleware {
+  constructor(
+    private path: string,
+    private config: AssetsConfig
+  ) {}
+
+  async handle(ctx: HttpContext, next: NextFn) {
+    const staticMiddleware = new StaticMiddleware(this.path, this.config)
+    return staticMiddleware.handle(ctx, next)
+  }
+}

admin/app/services/docker_service.ts

@@ -4,7 +4,7 @@ import logger from '@adonisjs/core/services/logger'
 import { inject } from '@adonisjs/core'
 import { ServiceStatus } from '../../types/services.js'
 import transmit from '@adonisjs/transmit/services/main'
-import { doSimpleDownload } from '../utils/downloads.js'
+import { doResumableDownloadWithRetry } from '../utils/downloads.js'
 import path from 'path'
 
 @inject()
@@ -347,10 +347,15 @@ export class DockerService {
     )
 
     try {
-      await doSimpleDownload({
+      await doResumableDownloadWithRetry({
         url: WIKIPEDIA_ZIM_URL,
         filepath,
         timeout: 60000,
+        allowedMimeTypes: [
+          'application/x-zim',
+          'application/x-openzim',
+          'application/octet-stream',
+        ],
       })
 
       this._broadcast(

admin/app/services/docs_service.ts

@@ -1,73 +1,65 @@
-import drive from '@adonisjs/drive/services/main';
+import Markdoc from '@markdoc/markdoc'
-import Markdoc from '@markdoc/markdoc';
+import { streamToString } from '../../util/docs.js'
-import { streamToString } from '../../util/docs.js';
+import { getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
+import path from 'path'
 
 export class DocsService {
+  private docsPath = path.join(process.cwd(), 'docs')
   async getDocs() {
-    const disk = drive.use('docs');
+    const contents = await listDirectoryContentsRecursive(this.docsPath)
-    if (!disk) {
+    const files: Array<{ title: string; slug: string }> = []
-      throw new Error('Docs disk not configured');
-    }
 
-    const contents = await disk.listAll('/');
+    for (const item of contents) {
-    const files: Array<{ title: string; slug: string }> = [];
+      if (item.type === 'file' && item.name.endsWith('.md')) {
-
+        const cleaned = this.prettify(item.name)
-    for (const item of contents.objects) {
-      if (item.isFile && item.name.endsWith('.md')) {
-        const cleaned = this.prettify(item.name);
         files.push({
           title: cleaned,
-          slug: item.name.replace(/\.md$/, '')
+          slug: item.name.replace(/\.md$/, ''),
-        });
+        })
       }
     }
 
-    return files.sort((a, b) => a.title.localeCompare(b.title));
+    return files.sort((a, b) => a.title.localeCompare(b.title))
   }
 
   parse(content: string) {
-    const ast = Markdoc.parse(content);
+    const ast = Markdoc.parse(content)
-    const config = this.getConfig();
+    const config = this.getConfig()
-    const errors = Markdoc.validate(ast, config);
+    const errors = Markdoc.validate(ast, config)
 
     if (errors.length > 0) {
-      throw new Error(`Markdoc validation errors: ${errors.map(e => e.error).join(', ')}`);
+      throw new Error(`Markdoc validation errors: ${errors.map((e) => e.error).join(', ')}`)
     }
 
-    return Markdoc.transform(ast, config);
+    return Markdoc.transform(ast, config)
   }
 
   async parseFile(_filename: string) {
-    const disk = drive.use('docs');
-    if (!disk) {
-      throw new Error('Docs disk not configured');
-    }
-
     if (!_filename) {
-      throw new Error('Filename is required');
+      throw new Error('Filename is required')
     }
 
-    const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`;
+    const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`
 
-    const fileExists = await disk.exists(filename);
+    const fileExists = await getFileStatsIfExists(path.join(this.docsPath, filename))
     if (!fileExists) {
-      throw new Error(`File not found: ${filename}`);
+      throw new Error(`File not found: ${filename}`)
     }
 
-    const fileStream = await disk.getStream(filename);
+    const fileStream = await getFile(path.join(this.docsPath, filename), 'stream')
     if (!fileStream) {
-      throw new Error(`Failed to read file stream: ${filename}`);
+      throw new Error(`Failed to read file stream: ${filename}`)
     }
-    const content = await streamToString(fileStream);
+    const content = await streamToString(fileStream)
-    return this.parse(content);
+    return this.parse(content)
   }
 
   private prettify(filename: string) {
     // Remove hyphens, underscores, and file extension
-    const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ');
+    const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ')
     // Convert to Title Case
-    const titleCased = cleaned.replace(/\b\w/g, char => char.toUpperCase());
+    const titleCased = cleaned.replace(/\b\w/g, (char) => char.toUpperCase())
-    return titleCased.charAt(0).toUpperCase() + titleCased.slice(1);
+    return titleCased.charAt(0).toUpperCase() + titleCased.slice(1)
   }
 
   private getConfig() {
@@ -79,12 +71,12 @@ export class DocsService {
             type: {
               type: String,
               default: 'info',
-              matches: ['info', 'warning', 'error', 'success']
+              matches: ['info', 'warning', 'error', 'success'],
             },
             title: {
-              type: String
+              type: String,
-            }
+            },
-          }
+          },
         },
       },
       nodes: {
@@ -92,10 +84,10 @@ export class DocsService {
           render: 'Heading',
           attributes: {
             level: { type: Number, required: true },
-            id: { type: String }
+            id: { type: String },
-          }
+          },
-        }
+        },
-      }
+      },
     }
   }
-}
+}

admin/app/services/download_service.ts

@@ -0,0 +1,25 @@
+import { inject } from '@adonisjs/core'
+import { QueueService } from './queue_service.js'
+import { RunDownloadJob } from '#jobs/run_download_job'
+import { DownloadJobWithProgress } from '../../types/downloads.js'
+import { normalize } from 'path'
+
+@inject()
+export class DownloadService {
+  constructor(private queueService: QueueService) {}
+
+  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
+    const queue = this.queueService.getQueue(RunDownloadJob.queue)
+    const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
+
+    return jobs
+      .map((job) => ({
+        jobId: job.id!.toString(),
+        url: job.data.url,
+        progress: parseInt(job.progress.toString(), 10),
+        filepath: normalize(job.data.filepath),
+        filetype: job.data.filetype,
+      }))
+      .filter((job) => !filetype || job.filetype === filetype)
+  }
+}

admin/app/services/map_service.ts

@@ -1,6 +1,6 @@
 import { BaseStylesFile, MapLayer } from '../../types/maps.js'
 import { FileEntry } from '../../types/files.js'
-import { doBackgroundDownload, doResumableDownloadWithRetry } from '../utils/downloads.js'
+import { doResumableDownloadWithRetry } from '../utils/downloads.js'
 import { extract } from 'tar'
 import env from '#start/env'
 import {
@@ -14,7 +14,8 @@ import {
 import { join } from 'path'
 import urlJoin from 'url-join'
 import axios from 'axios'
-import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
+import { RunDownloadJob } from '#jobs/run_download_job'
+import logger from '@adonisjs/core/services/logger'
 
 const BASE_ASSETS_MIME_TYPES = [
   'application/gzip',
@@ -32,7 +33,6 @@ export class MapService {
   private readonly basemapsAssetsDir = 'basemaps-assets'
   private readonly baseAssetsTarFile = 'base-assets.tar.gz'
   private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
-  private activeDownloads = new Map<string, AbortController>()
 
   async listRegions() {
     const files = (await this.listAllMapStorageItems()).filter(
@@ -80,13 +80,13 @@ export class MapService {
     return true
   }
 
-  async downloadRemote(url: string): Promise<string> {
+  async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> {
     const parsed = new URL(url)
     if (!parsed.pathname.endsWith('.pmtiles')) {
       throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
     }
 
-    const existing = this.activeDownloads.get(url)
+    const existing = await RunDownloadJob.getByUrl(url)
     if (existing) {
       throw new Error(`Download already in progress for URL ${url}`)
     }
@@ -98,18 +98,26 @@ export class MapService {
 
     const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
 
-    // Don't await the download, run it in the background
+    // Dispatch background job
-    doBackgroundDownload({
+    const result = await RunDownloadJob.dispatch({
       url,
       filepath,
       timeout: 30000,
       allowedMimeTypes: PMTILES_MIME_TYPES,
       forceNew: true,
-      channel: BROADCAST_CHANNELS.MAP,
+      filetype: 'pmtiles',
-      activeDownloads: this.activeDownloads,
     })
 
-    return filename
+    if (!result.job) {
+      throw new Error('Failed to dispatch download job')
+    }
+
+    logger.info(`[MapService] Dispatched download job ${result.job.id} for URL ${url}`)
+
+    return {
+      filename,
+      jobId: result.job?.id,
+    }
   }
 
   async downloadRemotePreflight(

admin/app/services/queue_service.ts

@@ -0,0 +1,22 @@
+import { Queue } from 'bullmq'
+import queueConfig from '#config/queue'
+
+export class QueueService {
+  private queues: Map<string, Queue> = new Map()
+
+  getQueue(name: string): Queue {
+    if (!this.queues.has(name)) {
+      const queue = new Queue(name, {
+        connection: queueConfig.connection,
+      })
+      this.queues.set(name, queue)
+    }
+    return this.queues.get(name)!
+  }
+
+  async close() {
+    for (const queue of this.queues.values()) {
+      await queue.close()
+    }
+  }
+}

admin/app/services/zim_service.ts

@@ -6,11 +6,9 @@ import {
 import axios from 'axios'
 import { XMLParser } from 'fast-xml-parser'
 import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
-import transmit from '@adonisjs/transmit/services/main'
 import logger from '@adonisjs/core/services/logger'
 import { DockerService } from './docker_service.js'
 import { inject } from '@adonisjs/core'
-import { doBackgroundDownload } from '../utils/downloads.js'
 import {
   deleteFileIfExists,
   ensureDirectoryExists,
@@ -23,7 +21,7 @@ import vine from '@vinejs/vine'
 import { curatedCollectionsFileSchema } from '#validators/curated_collections'
 import CuratedCollection from '#models/curated_collection'
 import CuratedCollectionResource from '#models/curated_collection_resource'
-import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
+import { RunDownloadJob } from '#jobs/run_download_job'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
 const COLLECTIONS_URL =
@@ -32,7 +30,6 @@ const COLLECTIONS_URL =
 @inject()
 export class ZimService {
   private zimStoragePath = '/storage/zim'
-  private activeDownloads = new Map<string, AbortController>()
 
   constructor(private dockerService: DockerService) {}
 
@@ -140,15 +137,15 @@ export class ZimService {
     }
   }
 
-  async downloadRemote(url: string): Promise<string> {
+  async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> {
     const parsed = new URL(url)
     if (!parsed.pathname.endsWith('.zim')) {
       throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`)
     }
 
-    const existing = this.activeDownloads.get(url)
+    const existing = await RunDownloadJob.getByUrl(url)
     if (existing) {
-      throw new Error(`Download already in progress for URL ${url}`)
+      throw new Error('A download for this URL is already in progress')
     }
 
     // Extract the filename from the URL
@@ -159,19 +156,26 @@ export class ZimService {
 
     const filepath = join(process.cwd(), this.zimStoragePath, filename)
 
-    // Don't await the download, run it in the background
+    // Dispatch a background download job
-    doBackgroundDownload({
+    const result = await RunDownloadJob.dispatch({
       url,
       filepath,
-      channel: BROADCAST_CHANNELS.ZIM,
-      activeDownloads: this.activeDownloads,
-      allowedMimeTypes: ZIM_MIME_TYPES,
       timeout: 30000,
+      allowedMimeTypes: ZIM_MIME_TYPES,
       forceNew: true,
-      onComplete: (url) => this._downloadRemoteSuccessCallback([url]),
+      filetype: 'zim',
     })
 
-    return filename
+    if (!result || !result.job) {
+      throw new Error('Failed to dispatch download job')
+    }
+
+    logger.info(`[ZimService] Dispatched background download job for ZIM file: ${filename}`)
+
+    return {
+      filename,
+      jobId: result.job.id,
+    }
   }
 
   async downloadCollection(slug: string): Promise<string[] | null> {
@@ -188,49 +192,43 @@ export class ZimService {
     const downloadUrls = resources.map((res) => res.url)
     const downloadFilenames: string[] = []
 
-    for (const [idx, url] of downloadUrls.entries()) {
+    for (const url of downloadUrls) {
-      const existing = this.activeDownloads.get(url)
+      const existing = await RunDownloadJob.getByUrl(url)
       if (existing) {
-        logger.warn(`Download already in progress for URL ${url}, skipping.`)
+        logger.warn(`[ZimService] Download already in progress for URL ${url}, skipping.`)
         continue
       }
 
       // Extract the filename from the URL
       const filename = url.split('/').pop()
       if (!filename) {
-        logger.warn(`Could not determine filename from URL ${url}, skipping.`)
+        logger.warn(`[ZimService] Could not determine filename from URL ${url}, skipping.`)
         continue
       }
 
-      const filepath = join(process.cwd(), this.zimStoragePath, filename)
       downloadFilenames.push(filename)
+      const filepath = join(process.cwd(), this.zimStoragePath, filename)
 
-      const isLastDownload = idx === downloadUrls.length - 1
+      await RunDownloadJob.dispatch({
-
-      // Don't await the download, run it in the background
-      doBackgroundDownload({
         url,
         filepath,
-        channel: BROADCAST_CHANNELS.ZIM,
-        activeDownloads: this.activeDownloads,
-        allowedMimeTypes: ZIM_MIME_TYPES,
         timeout: 30000,
+        allowedMimeTypes: ZIM_MIME_TYPES,
         forceNew: true,
-        onComplete: (url) =>
+        filetype: 'zim',
-          this._downloadRemoteSuccessCallback([url], isLastDownload),
       })
     }
 
     return downloadFilenames.length > 0 ? downloadFilenames : null
   }
 
-  async _downloadRemoteSuccessCallback(urls: string[], restart = true) {
+  async downloadRemoteSuccessCallback(urls: string[], restart = true) {
     // Restart KIWIX container to pick up new ZIM file
     if (restart) {
       await this.dockerService
         .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
         .catch((error) => {
-          logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
+          logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
         })
     }
 
@@ -242,21 +240,6 @@ export class ZimService {
     }
   }
 
-  listActiveDownloads(): string[] {
-    return Array.from(this.activeDownloads.keys())
-  }
-
-  cancelDownload(url: string): boolean {
-    const entry = this.activeDownloads.get(url)
-    if (entry) {
-      entry.abort()
-      this.activeDownloads.delete(url)
-      transmit.broadcast(BROADCAST_CHANNELS.ZIM, { url, status: 'cancelled' })
-      return true
-    }
-    return false
-  }
-
   async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
     const collections = await CuratedCollection.query().preload('resources')
     return collections.map((collection) => ({
@@ -282,17 +265,17 @@ export class ZimService {
             type: 'zim',
           }
         )
-        logger.info(`Upserted curated collection: ${collection.slug}`)
+        logger.info(`[ZimService] Upserted curated collection: ${collection.slug}`)
 
         await collectionResult.related('resources').createMany(collection.resources)
         logger.info(
-          `Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
+          `[ZimService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
         )
       }
 
       return true
     } catch (error) {
-      logger.error('Failed to download latest Kiwix collections:', error)
+      logger.error(`[ZimService] Failed to download latest Kiwix collections:`, error)
       return false
     }
   }

admin/app/utils/downloads.ts

@@ -1,7 +1,5 @@
 import {
-  DoBackgroundDownloadParams,
   DoResumableDownloadParams,
-  DoResumableDownloadProgress,
   DoResumableDownloadWithRetryParams,
   DoSimpleDownloadParams,
 } from '../../types/downloads.js'
@@ -9,9 +7,6 @@ import axios, { AxiosResponse } from 'axios'
 import { Transform } from 'stream'
 import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
 import { createWriteStream } from 'fs'
-import { formatSpeed } from './misc.js'
-import { DownloadProgress } from '../../types/files.js'
-import transmit from '@adonisjs/transmit/services/main'
 import logger from '@adonisjs/core/services/logger'
 import path from 'path'
 
@@ -86,6 +81,7 @@ export async function doResumableDownload({
   timeout = 30000,
   signal,
   onProgress,
+  onComplete,
   forceNew = false,
   allowedMimeTypes,
 }: DoResumableDownloadParams): Promise<string> {
@@ -200,7 +196,7 @@ export async function doResumableDownload({
       cleanup(new Error('Download aborted'))
     })
 
-    writeStream.on('finish', () => {
+    writeStream.on('finish', async () => {
       if (onProgress) {
         onProgress({
           downloadedBytes,
@@ -210,6 +206,9 @@ export async function doResumableDownload({
           url,
         })
       }
+      if (onComplete) {
+        await onComplete(url, filepath)
+      }
       resolve(filepath)
     })
 
@@ -276,82 +275,6 @@ export async function doResumableDownloadWithRetry({
   throw lastError || new Error('Unknown error during download')
 }
 
-export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise<void> {
-  const { url, filepath, channel, activeDownloads, onComplete, ...restParams } = params
-
-  try {
-    const dirname = path.dirname(filepath)
-    await ensureDirectoryExists(dirname)
-
-    const abortController = new AbortController()
-    activeDownloads.set(url, abortController)
-
-    await doResumableDownloadWithRetry({
-      url,
-      filepath,
-      signal: abortController.signal,
-      ...restParams,
-      onProgress: (progressData) => {
-        sendProgressBroadcast(channel, progressData)
-      },
-    })
-
-    sendCompletedBroadcast(channel, url, filepath)
-
-    if (onComplete) {
-      await onComplete(url, filepath)
-    }
-  } catch (error) {
-    logger.error(`Background download failed for ${url}: ${error.message}`)
-    sendErrorBroadcast(channel, url, error.message)
-  } finally {
-    activeDownloads.delete(url)
-  }
-}
-
-export function sendProgressBroadcast(
-  channel: string,
-  progressData: DoResumableDownloadProgress,
-  status = 'in_progress'
-) {
-  const { downloadedBytes, totalBytes, lastProgressTime, lastDownloadedBytes, url } = progressData
-  const now = Date.now()
-  const timeDiff = (now - lastProgressTime) / 1000
-  const bytesDiff = downloadedBytes - lastDownloadedBytes
-  const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0
-  const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0
-  const speed = formatSpeed(rawSpeed)
-
-  const progress: DownloadProgress = {
-    downloaded_bytes: downloadedBytes,
-    total_bytes: totalBytes,
-    percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
-    speed,
-    time_remaining: timeRemaining,
-  }
-
-  transmit.broadcast(channel, { url, progress, status })
-}
-
-export function sendCompletedBroadcast(channel: string, url: string, path: string) {
-  transmit.broadcast(channel, {
-    url,
-    path,
-    status: 'completed',
-    progress: {
-      downloaded_bytes: 0,
-      total_bytes: 0,
-      percentage: 100,
-      speed: '0 B/s',
-      time_remaining: 0,
-    },
-  })
-}
-
-export function sendErrorBroadcast(channel: string, url: string, errorMessage: string) {
-  transmit.broadcast(channel, { url, error: errorMessage, status: 'failed' })
-}
-
 async function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }

admin/app/utils/fs.ts

@@ -1,9 +1,7 @@
 import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises'
 import { join } from 'path'
-import { DriveDisks } from '@adonisjs/drive/types'
-import driveConfig from '#config/drive'
-import app from '@adonisjs/core/services/app'
 import { FileEntry } from '../../types/files.js'
+import { createReadStream } from 'fs'
 
 export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
   const entries = await readdir(path, { withFileTypes: true })
@@ -56,14 +54,22 @@ export async function ensureDirectoryExists(path: string): Promise<void> {
 }
 
 export async function getFile(path: string, returnType: 'buffer'): Promise<Buffer | null>
+export async function getFile(
+  path: string,
+  returnType: 'stream'
+): Promise<NodeJS.ReadableStream | null>
 export async function getFile(path: string, returnType: 'string'): Promise<string | null>
-export async function getFile(path: string, returnType: 'buffer' | 'string' = 'buffer'): Promise<Buffer | string | null> {
+export async function getFile(
+  path: string,
+  returnType: 'buffer' | 'string' | 'stream' = 'buffer'
+): Promise<Buffer | string | NodeJS.ReadableStream | null> {
   try {
-    if (returnType === 'buffer') {
+    if (returnType === 'string') {
-      return await readFile(path)
-    } else {
       return await readFile(path, 'utf-8')
+    } else if (returnType === 'stream') {
+      return createReadStream(path)
     }
+    return await readFile(path)
   } catch (error) {
     if (error.code === 'ENOENT') {
       return null
@@ -98,18 +104,3 @@ export async function deleteFileIfExists(path: string): Promise<void> {
     }
   }
 }
-
-export async function getFullDrivePath(diskName: keyof DriveDisks): Promise<string> {
-  const config = await driveConfig.resolver(app)
-  const serviceConfig = config.config.services[diskName]
-  const resolved = serviceConfig()
-  if (!resolved) {
-    throw new Error(`Disk ${diskName} not configured`)
-  }
-
-  let path = resolved.options.location
-  if (path instanceof URL) {
-    return path.pathname
-  }
-  return path
-}

admin/app/validators/common.ts

@@ -23,9 +23,11 @@ export const remoteDownloadValidatorOptional = vine.compile(
   })
 )
 
-export const filenameValidator = vine.compile(
+export const filenameParamValidator = vine.compile(
   vine.object({
-    filename: vine.string().trim().minLength(1).maxLength(4096),
+    params: vine.object({
+      filename: vine.string().trim().minLength(1).maxLength(4096),
+    }),
   })
 )
 

admin/app/validators/download.ts

@@ -0,0 +1,9 @@
+import vine from '@vinejs/vine'
+
+export const downloadJobsByFiletypeSchema = vine.compile(
+  vine.object({
+    params: vine.object({
+      filetype: vine.string(),
+    }),
+  })
+)

admin/commands/queue/work.ts

@@ -0,0 +1,67 @@
+import { BaseCommand, flags } from '@adonisjs/core/ace'
+import type { CommandOptions } from '@adonisjs/core/types/ace'
+import { Worker } from 'bullmq'
+import queueConfig from '#config/queue'
+
+export default class QueueWork extends BaseCommand {
+  static commandName = 'queue:work'
+  static description = 'Start processing jobs from the queue'
+
+  @flags.string({ description: 'Queue name to process', required: true })
+  declare queue: string
+
+  static options: CommandOptions = {
+    startApp: true,
+    staysAlive: true,
+  }
+
+  async run() {
+    const queueName = this.queue || 'default'
+
+    const jobHandlers = await this.loadJobHandlers()
+
+    const worker = new Worker(
+      queueName,
+      async (job) => {
+        this.logger.info(`Processing job: ${job.id} of type: ${job.name}`)
+        const jobHandler = jobHandlers.get(job.name)
+        if (!jobHandler) {
+          throw new Error(`No handler found for job: ${job.name}`)
+        }
+
+        return await jobHandler.handle(job)
+      },
+      {
+        connection: queueConfig.connection,
+        concurrency: 3,
+        autorun: true,
+      }
+    )
+
+    worker.on('failed', (job, err) => {
+      this.logger.error(`Job failed: ${job?.id}, Error: ${err.message}`)
+    })
+
+    worker.on('completed', (job) => {
+      this.logger.info(`Job completed: ${job.id}`)
+    })
+
+    this.logger.info(`Worker started for queue: ${queueName}`)
+
+    process.on('SIGTERM', async () => {
+      this.logger.info('SIGTERM received. Shutting down worker...')
+      await worker.close()
+      this.logger.info('Worker shut down gracefully.')
+      process.exit(0)
+    })
+  }
+
+  private async loadJobHandlers() {
+    const handlers = new Map<string, any>()
+
+    const { RunDownloadJob } = await import('#jobs/run_download_job')
+    handlers.set(RunDownloadJob.key, new RunDownloadJob())
+
+    return handlers
+  }
+}

admin/config/drive.ts

@@ -1,31 +0,0 @@
-import env from '#start/env'
-import app from '@adonisjs/core/services/app'
-import { defineConfig, services } from '@adonisjs/drive'
-
-const driveConfig = defineConfig({
-  default: env.get('DRIVE_DISK'),
-
-  /**
-   * The services object can be used to configure multiple file system
-   * services each using the same or a different driver.
-   */
-  services: {
-    fs: services.fs({
-      location: app.makePath('storage'),
-      serveFiles: true,
-      routeBasePath: '/storage',
-      visibility: 'public',
-    }),
-    docs: services.fs({
-      location: app.makePath('docs'),
-      serveFiles: false, // Don't serve files directly - we handle this via routes/Inertia
-      visibility: 'public',
-    }),
-  },
-})
-
-export default driveConfig
-
-declare module '@adonisjs/drive/types' {
-  export interface DriveDisks extends InferDriveDisks<typeof driveConfig> { }
-}

admin/config/queue.ts

@@ -0,0 +1,10 @@
+import env from '#start/env'
+
+const queueConfig = {
+  connection: {
+    host: env.get('REDIS_HOST'),
+    port: env.get('REDIS_PORT') ?? 6379,
+  },
+}
+
+export default queueConfig

admin/config/static.ts

@@ -12,6 +12,7 @@ const staticServerConfig = defineConfig({
   etag: true,
   lastModified: true,
   dotFiles: 'ignore',
+  acceptRanges: true,
 })
 
 export default staticServerConfig

admin/inertia/components/HorizontalBarChart.tsx

@@ -8,11 +8,23 @@ interface HorizontalBarChartProps {
     used: string
     type?: string
   }>
-  maxValue?: number
+  statuses?: Array<{
+    label: string
+    min_threshold: number
+    color_class: string
+  }>
+  progressiveBarColor?: boolean
 }
 
-export default function HorizontalBarChart({ items, maxValue = 100 }: HorizontalBarChartProps) {
+export default function HorizontalBarChart({
+  items,
+  statuses,
+  progressiveBarColor = false,
+}: HorizontalBarChartProps) {
+  const sortedStatus = statuses?.sort((a, b) => b.min_threshold - a.min_threshold) || []
+
   const getBarColor = (value: number) => {
+    if (!progressiveBarColor) return 'bg-desert-green'
     if (value >= 90) return 'bg-desert-red'
     if (value >= 75) return 'bg-desert-orange'
     if (value >= 50) return 'bg-desert-tan'
@@ -26,6 +38,26 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
     return 'shadow-desert-olive/50'
   }
 
+  const getStatusLabel = (value: number) => {
+    if (sortedStatus.length === 0) return ''
+    for (const status of sortedStatus) {
+      if (value >= status.min_threshold) {
+        return status.label
+      }
+    }
+    return ''
+  }
+
+  const getStatusColor = (value: number) => {
+    if (sortedStatus.length === 0) return ''
+    for (const status of sortedStatus) {
+      if (value >= status.min_threshold) {
+        return status.color_class
+      }
+    }
+    return ''
+  }
+
   return (
     <div className="space-y-6">
       {items.map((item, index) => (
@@ -56,28 +88,7 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
                   width: `${item.value}%`,
                   animationDelay: `${index * 100}ms`,
                 }}
-              >
+              ></div>
-                {/* Animated shine effect */}
-                {/* <div
-                  className="absolute inset-0 bg-gradient-to-r from-transparent via-white to-transparent opacity-20 animate-shimmer"
-                  style={{
-                    animation: 'shimmer 3s infinite',
-                    animationDelay: `${index * 0.5}s`,
-                  }}
-                /> */}
-                {/* <div
-                  className="absolute inset-0 opacity-10"
-                  style={{
-                    backgroundImage: `repeating-linear-gradient(
-                      90deg,
-                      transparent,
-                      transparent 10px,
-                      rgba(255, 255, 255, 0.1) 10px,
-                      rgba(255, 255, 255, 0.1) 11px
-                    )`,
-                  }}
-                /> */}
-              </div>
             </div>
             <div
               className={classNames(
@@ -90,25 +101,17 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
               {Math.round(item.value)}%
             </div>
           </div>
-          <div className="flex items-center gap-2">
+          {getStatusLabel(item.value) && (
-            <div
+            <div className="flex items-center gap-2">
-              className={classNames(
+              <div
-                'w-2 h-2 rounded-full animate-pulse',
+                className={classNames(
-                item.value >= 90
+                  'w-2 h-2 rounded-full animate-pulse',
-                  ? 'bg-desert-red'
+                  getStatusColor(item.value)
-                  : item.value >= 75
+                )}
-                    ? 'bg-desert-orange'
+              />
-                    : 'bg-desert-olive'
+              <span className="text-xs text-desert-stone">{getStatusLabel(item.value)}</span>
-              )}
+            </div>
-            />
+          )}
-            <span className="text-xs text-desert-stone">
-              {item.value >= 90
-                ? 'Critical - Disk Almost Full'
-                : item.value >= 75
-                  ? 'Warning - Usage High'
-                  : 'Normal'}
-            </span>
-          </div>
         </div>
       ))}
     </div>

admin/inertia/hooks/useDownloads.ts

@@ -0,0 +1,31 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query'
+import { useMemo } from 'react'
+import api from '~/lib/api'
+
+export type useDownloadsProps = {
+  filetype?: string
+  enabled?: boolean
+}
+
+const useDownloads = (props: useDownloadsProps) => {
+  const queryClient = useQueryClient()
+
+  const queryKey = useMemo(() => {
+    return props.filetype ? ['download-jobs', props.filetype] : ['download-jobs']
+  }, [props.filetype])
+
+  const queryData = useQuery({
+    queryKey: queryKey,
+    queryFn: () => api.listDownloadJobs(props.filetype),
+    refetchInterval: 2000, // Refetch every 2 seconds to get updated progress
+    enabled: props.enabled ?? true,
+  })
+
+  const invalidate = () => {
+    queryClient.invalidateQueries({ queryKey: queryKey })
+  }
+
+  return { ...queryData, invalidate }
+}
+
+export default useDownloads

admin/inertia/lib/api.ts

@@ -3,7 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi
 import { ServiceSlim } from '../../types/services'
 import { FileEntry } from '../../types/files'
 import { SystemInformationResponse } from '../../types/system'
-import { CuratedCollectionWithStatus } from '../../types/downloads'
+import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads'
 
 class API {
   private client: AxiosInstance
@@ -142,16 +142,6 @@ class API {
     }
   }
 
-  async listActiveZimDownloads(): Promise<string[]> {
-    try {
-      const response = await this.client.get<string[]>('/zim/active-downloads')
-      return response.data
-    } catch (error) {
-      console.error('Error listing active ZIM downloads:', error)
-      throw error
-    }
-  }
-
   async downloadRemoteZimFile(url: string): Promise<{
     message: string
     filename: string
@@ -209,6 +199,17 @@ class API {
       throw error
     }
   }
+
+  async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
+    try {
+      const endpoint = filetype ? `/downloads/jobs/${filetype}` : '/downloads/jobs'
+      const response = await this.client.get<DownloadJobWithProgress[]>(endpoint)
+      return response.data
+    } catch (error) {
+      console.error('Error listing download jobs:', error)
+      throw error
+    }
+  }
 }
 
 export default new API()

admin/inertia/pages/settings/system.tsx

@@ -3,7 +3,7 @@ import SettingsLayout from '~/layouts/SettingsLayout'
 import { SystemInformationResponse } from '../../../types/system'
 import { formatBytes } from '~/lib/util'
 import CircularGauge from '~/components/systeminfo/CircularGauge'
-import HorizontalBarChart from '~/components/systeminfo/HorizontalBarChart'
+import HorizontalBarChart from '~/components/HorizontalBarChart'
 import InfoCard from '~/components/systeminfo/InfoCard'
 import {
   CpuChipIcon,
@@ -195,7 +195,27 @@ export default function SettingsPage(props: {
 
             <div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow">
               {diskData && diskData.length > 0 ? (
-                <HorizontalBarChart items={diskData} />
+                <HorizontalBarChart
+                  items={diskData}
+                  progressiveBarColor={true}
+                  statuses={[
+                    {
+                      label: 'Normal',
+                      min_threshold: 0,
+                      color_class: 'bg-desert-olive',
+                    },
+                    {
+                      label: 'Warning - Usage High',
+                      min_threshold: 75,
+                      color_class: 'bg-desert-orange',
+                    },
+                    {
+                      label: 'Critical - Disk Almost Full',
+                      min_threshold: 90,
+                      color_class: 'bg-desert-red',
+                    },
+                  ]}
+                />
               ) : (
                 <div className="text-center text-desert-stone-dark py-8">
                   No storage devices detected

admin/inertia/pages/settings/zim/remote-explorer.tsx

@@ -16,8 +16,6 @@ import { formatBytes } from '~/lib/util'
 import StyledButton from '~/components/StyledButton'
 import { useModals } from '~/context/ModalContext'
 import StyledModal from '~/components/StyledModal'
-import { useTransmit } from 'react-adonis-transmit'
-import ProgressBar from '~/components/ProgressBar'
 import { useNotifications } from '~/context/NotificationContext'
 import useInternetStatus from '~/hooks/useInternetStatus'
 import Alert from '~/components/Alert'
@@ -28,7 +26,8 @@ import useDebounce from '~/hooks/useDebounce'
 import CuratedCollectionCard from '~/components/CuratedCollectionCard'
 import StyledSectionHeader from '~/components/StyledSectionHeader'
 import { CuratedCollectionWithStatus } from '../../../../types/downloads'
-import { BROADCAST_CHANNELS } from '../../../../util/broadcast_channels'
+import useDownloads from '~/hooks/useDownloads'
+import HorizontalBarChart from '~/components/HorizontalBarChart'
 
 const CURATED_COLLECTIONS_KEY = 'curated-zim-collections'
 
@@ -36,7 +35,6 @@ export default function ZimRemoteExplorer() {
   const queryClient = useQueryClient()
   const tableParentRef = useRef<HTMLDivElement>(null)
 
-  const { subscribe } = useTransmit()
   const { openModal, closeAllModals } = useModals()
   const { addNotification } = useNotifications()
   const { isOnline } = useInternetStatus()
@@ -45,44 +43,22 @@ export default function ZimRemoteExplorer() {
 
   const [query, setQuery] = useState('')
   const [queryUI, setQueryUI] = useState('')
-  const [activeDownloads, setActiveDownloads] = useState<
-    Map<string, { status: string; progress: number; speed: string }>
-  >(new Map<string, { status: string; progress: number; speed: string }>())
 
   const debouncedSetQuery = debounce((val: string) => {
     setQuery(val)
   }, 400)
 
-  useEffect(() => {
-    const unsubscribe = subscribe(BROADCAST_CHANNELS.ZIM, (data: any) => {
-      if (data.url && data.progress?.percentage) {
-        setActiveDownloads((prev) =>
-          new Map(prev).set(data.url, {
-            status: data.status,
-            progress: data.progress.percentage || 0,
-            speed: data.progress.speed || '0 KB/s',
-          })
-        )
-        if (data.status === 'completed') {
-          addNotification({
-            message: `The download for ${data.url} has completed successfully.`,
-            type: 'success',
-          })
-        }
-      }
-    })
-
-    return () => {
-      unsubscribe()
-    }
-  }, [])
-
   const { data: curatedCollections } = useQuery({
     queryKey: [CURATED_COLLECTIONS_KEY],
     queryFn: () => api.listCuratedZimCollections(),
     refetchOnWindowFocus: false,
   })
 
+  const { data: downloads, invalidate: invalidateDownloads } = useDownloads({
+    filetype: 'zim',
+    enabled: true,
+  })
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery<ListRemoteZimFilesResponse>({
       queryKey: ['remote-zim-files', query],
@@ -103,7 +79,17 @@ export default function ZimRemoteExplorer() {
       placeholderData: keepPreviousData,
     })
 
-  const flatData = useMemo(() => data?.pages.flatMap((page) => page.items) || [], [data])
+  const flatData = useMemo(() => {
+    const mapped = data?.pages.flatMap((page) => page.items) || []
+    // remove items that are currently downloading
+    return mapped.filter((item) => {
+      const isDownloading = downloads?.some((download) => {
+        const filename = item.download_url.split('/').pop()
+        return filename && download.filepath.endsWith(filename)
+      })
+      return !isDownloading
+    })
+  }, [data, downloads])
   const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data])
 
   const fetchOnBottomReached = useCallback(
@@ -176,6 +162,7 @@ export default function ZimRemoteExplorer() {
   async function downloadFile(record: RemoteZimFileEntry) {
     try {
       await api.downloadRemoteZimFile(record.download_url)
+      invalidateDownloads()
     } catch (error) {
       console.error('Error downloading file:', error)
     }
@@ -184,19 +171,12 @@ export default function ZimRemoteExplorer() {
   async function downloadCollection(record: CuratedCollectionWithStatus) {
     try {
       await api.downloadZimCollection(record.slug)
+      invalidateDownloads()
     } catch (error) {
       console.error('Error downloading collection:', error)
     }
   }
 
-  const EntryProgressBar = useCallback(
-    ({ url }: { url: string }) => {
-      const entry = activeDownloads.get(url)
-      return <ProgressBar progress={entry?.progress || 0} speed={entry?.speed} />
-    },
-    [activeDownloads]
-  )
-
   const fetchLatestCollections = useMutation({
     mutationFn: () => api.fetchLatestZimCollections(),
     onSuccess: () => {
@@ -208,6 +188,17 @@ export default function ZimRemoteExplorer() {
     },
   })
 
+  const extractFileName = (path: string) => {
+    if (!path) return ''
+    if (path.includes('/')) {
+      return path.substring(path.lastIndexOf('/') + 1)
+    }
+    if (path.includes('\\')) {
+      return path.substring(path.lastIndexOf('\\') + 1)
+    }
+    return path
+  }
+
   return (
     <SettingsLayout>
       <Head title="ZIM Remote Explorer | Project N.O.M.A.D." />
@@ -307,20 +298,16 @@ export default function ZimRemoteExplorer() {
               {
                 accessor: 'actions',
                 render(record) {
-                  const isDownloading = activeDownloads.has(record.download_url)
                   return (
                     <div className="flex space-x-2">
-                      {!isDownloading && (
+                      <StyledButton
-                        <StyledButton
+                        icon={'ArrowDownTrayIcon'}
-                          icon={'ArrowDownTrayIcon'}
+                        onClick={() => {
-                          onClick={() => {
+                          confirmDownload(record)
-                            confirmDownload(record)
+                        }}
-                          }}
+                      >
-                        >
+                        Download
-                          Download
+                      </StyledButton>
-                        </StyledButton>
-                      )}
-                      {isDownloading && <EntryProgressBar url={record.download_url} />}
                     </div>
                   )
                 },
@@ -337,6 +324,28 @@ export default function ZimRemoteExplorer() {
             compact
             rowLines
           />
+          <StyledSectionHeader title="Active Downloads" className="mt-12 mb-4" />
+          <div className="space-y-4">
+            {downloads && downloads.length > 0 ? (
+              downloads.map((download) => (
+                <div className="bg-desert-white rounded-lg p-4 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow">
+                  <HorizontalBarChart
+                    items={[
+                      {
+                        label: extractFileName(download.filepath) || download.url,
+                        value: download.progress,
+                        total: '100%',
+                        used: `${download.progress}%`,
+                        type: download.filetype,
+                      },
+                    ]}
+                  />
+                </div>
+              ))
+            ) : (
+              <p className="text-gray-500">No active downloads</p>
+            )}
+          </div>
         </main>
       </div>
     </SettingsLayout>

admin/package-lock.json

@@ -12,7 +12,6 @@
         "@adonisjs/auth": "^9.4.0",
         "@adonisjs/core": "^6.18.0",
         "@adonisjs/cors": "^2.2.1",
-        "@adonisjs/drive": "^3.4.1",
         "@adonisjs/inertia": "^3.1.1",
         "@adonisjs/lucid": "^21.6.1",
         "@adonisjs/session": "^7.5.1",
@@ -36,6 +35,7 @@
         "autoprefixer": "^10.4.21",
         "axios": "^1.13.1",
         "better-sqlite3": "^12.1.1",
+        "bullmq": "^5.65.1",
         "dockerode": "^4.0.7",
         "edge.js": "^6.2.1",
         "fast-xml-parser": "^5.2.5",
@@ -339,35 +339,6 @@
         "@adonisjs/core": "^6.2.0"
       }
     },
-    "node_modules/@adonisjs/drive": {
-      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/@adonisjs/drive/-/drive-3.4.1.tgz",
-      "integrity": "sha512-oDYY4wJ7wDMlO4E+dZPYBu+T3Av7Mj+JL8+J33qgyxtiJylnZgoZDuRfFjZZix/bFNNuWX2sLwTMnyiDcK+YsA==",
-      "license": "MIT",
-      "dependencies": {
-        "flydrive": "^1.1.0"
-      },
-      "engines": {
-        "node": ">=20.6.0"
-      },
-      "peerDependencies": {
-        "@adonisjs/core": "^6.2.0",
-        "@aws-sdk/client-s3": "^3.577.0",
-        "@aws-sdk/s3-request-presigner": "^3.577.0",
-        "@google-cloud/storage": "^7.10.2"
-      },
-      "peerDependenciesMeta": {
-        "@aws-sdk/client-s3": {
-          "optional": true
-        },
-        "@aws-sdk/s3-request-presigner": {
-          "optional": true
-        },
-        "@google-cloud/storage": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/@adonisjs/encryption": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/@adonisjs/encryption/-/encryption-6.0.2.tgz",
@@ -2083,6 +2054,7 @@
       "version": "0.4.3",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz",
       "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==",
+      "dev": true,
       "license": "Apache-2.0",
       "engines": {
         "node": ">=18.18"
@@ -2116,6 +2088,12 @@
         "react": "^16.9.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
       }
     },
+    "node_modules/@ioredis/commands": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",
+      "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==",
+      "license": "MIT"
+    },
     "node_modules/@isaacs/fs-minipass": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
@@ -2461,6 +2439,84 @@
         }
       }
     },
+    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
+      "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
+      "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
+      "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
+      "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
+      "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
+      "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@noble/hashes": {
       "version": "1.8.0",
       "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz",
@@ -5098,6 +5154,21 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/bullmq": {
+      "version": "5.65.1",
+      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.65.1.tgz",
+      "integrity": "sha512-QgDAzX1G9L5IRy4Orva5CfQTXZT+5K+OfO/kbPrAqN+pmL9LJekCzxijXehlm/u2eXfWPfWvIdJJIqiuz3WJSg==",
+      "license": "MIT",
+      "dependencies": {
+        "cron-parser": "^4.9.0",
+        "ioredis": "^5.8.2",
+        "msgpackr": "^1.11.2",
+        "node-abort-controller": "^3.1.1",
+        "semver": "^7.5.4",
+        "tslib": "^2.0.0",
+        "uuid": "^11.1.0"
+      }
+    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -5522,6 +5593,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/code-block-writer": {
       "version": "13.0.3",
       "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-13.0.3.tgz",
@@ -5710,6 +5790,18 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/cron-parser": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
+      "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+      "license": "MIT",
+      "dependencies": {
+        "luxon": "^3.2.1"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -5768,9 +5860,9 @@
       "license": "MIT"
     },
     "node_modules/debug": {
-      "version": "4.4.1",
+      "version": "4.4.3",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
-      "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
       "license": "MIT",
       "dependencies": {
         "ms": "^2.1.3"
@@ -7001,58 +7093,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/flydrive": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/flydrive/-/flydrive-1.2.0.tgz",
-      "integrity": "sha512-l9ix5MhBE8bVwxyHdFku6z5KhGOCOXQDI9xGNIlACSz9UrDFQxAB1I6W0qffZiOBBDambiJZlEYBCxlvF4U7fw==",
-      "license": "MIT",
-      "dependencies": {
-        "@humanwhocodes/retry": "^0.4.2",
-        "@poppinss/utils": "^6.9.2",
-        "etag": "^1.8.1",
-        "mime-types": "^2.1.35"
-      },
-      "engines": {
-        "node": ">=20.6.0"
-      },
-      "peerDependencies": {
-        "@aws-sdk/client-s3": "^3.577.0",
-        "@aws-sdk/s3-request-presigner": "^3.577.0",
-        "@google-cloud/storage": "^7.10.2"
-      },
-      "peerDependenciesMeta": {
-        "@aws-sdk/client-s3": {
-          "optional": true
-        },
-        "@aws-sdk/s3-request-presigner": {
-          "optional": true
-        },
-        "@google-cloud/storage": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/flydrive/node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/flydrive/node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "license": "MIT",
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
     "node_modules/follow-redirects": {
       "version": "1.15.9",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
@@ -7751,6 +7791,30 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/ioredis": {
+      "version": "5.8.2",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz",
+      "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@ioredis/commands": "1.4.0",
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.4",
+        "denque": "^2.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.isarguments": "^3.1.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=12.22.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -8469,6 +8533,18 @@
       "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
       "license": "MIT"
     },
+    "node_modules/lodash.defaults": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+      "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isarguments": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
+      "license": "MIT"
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -8900,6 +8976,37 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
+    "node_modules/msgpackr": {
+      "version": "1.11.5",
+      "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
+      "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
+      "license": "MIT",
+      "optionalDependencies": {
+        "msgpackr-extract": "^3.0.2"
+      }
+    },
+    "node_modules/msgpackr-extract": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
+      "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "node-gyp-build-optional-packages": "5.2.2"
+      },
+      "bin": {
+        "download-msgpackr-prebuilds": "bin/download-prebuilds.js"
+      },
+      "optionalDependencies": {
+        "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
+      }
+    },
     "node_modules/murmurhash-js": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz",
@@ -9015,6 +9122,27 @@
         "node": ">=10"
       }
     },
+    "node_modules/node-abort-controller": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
+      "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
+      "license": "MIT"
+    },
+    "node_modules/node-gyp-build-optional-packages": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
+      "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "detect-libc": "^2.0.1"
+      },
+      "bin": {
+        "node-gyp-build-optional-packages": "bin.js",
+        "node-gyp-build-optional-packages-optional": "optional.js",
+        "node-gyp-build-optional-packages-test": "build-test.js"
+      }
+    },
     "node_modules/node-releases": {
       "version": "2.0.19",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
@@ -9921,16 +10049,61 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "3.0.0",
+      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
-      "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==",
+      "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
       "license": "MIT",
       "dependencies": {
-        "bytes": "3.1.2",
+        "bytes": "~3.1.2",
-        "http-errors": "2.0.0",
+        "http-errors": "~2.0.1",
-        "iconv-lite": "0.6.3",
+        "iconv-lite": "~0.7.0",
-        "unpipe": "1.0.0"
+        "unpipe": "~1.0.0"
       },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
+    "node_modules/raw-body/node_modules/http-errors": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
+      "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
+      "license": "MIT",
+      "dependencies": {
+        "depd": "~2.0.0",
+        "inherits": "~2.0.4",
+        "setprototypeof": "~1.2.0",
+        "statuses": "~2.0.2",
+        "toidentifier": "~1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/raw-body/node_modules/iconv-lite": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz",
+      "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/raw-body/node_modules/statuses": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
+      "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
+      "license": "MIT",
       "engines": {
         "node": ">= 0.8"
       }
@@ -10182,6 +10355,27 @@
         "node": ">= 10.13.0"
       }
     },
+    "node_modules/redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+      "license": "MIT",
+      "dependencies": {
+        "redis-errors": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/reflect-metadata": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz",
@@ -10963,6 +11157,12 @@
         "get-source": "^2.0.12"
       }
     },
+    "node_modules/standard-as-callback": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+      "license": "MIT"
+    },
     "node_modules/statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -11777,6 +11977,19 @@
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "license": "MIT"
     },
+    "node_modules/uuid": {
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
+      "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/esm/bin/uuid"
+      }
+    },
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",

admin/package.json

@@ -30,7 +30,8 @@
     "#database/*": "./database/*.js",
     "#tests/*": "./tests/*.js",
     "#start/*": "./start/*.js",
-    "#config/*": "./config/*.js"
+    "#config/*": "./config/*.js",
+    "#jobs/*": "./app/jobs/*.js"
   },
   "devDependencies": {
     "@adonisjs/assembler": "^7.8.2",
@@ -58,7 +59,6 @@
     "@adonisjs/auth": "^9.4.0",
     "@adonisjs/core": "^6.18.0",
     "@adonisjs/cors": "^2.2.1",
-    "@adonisjs/drive": "^3.4.1",
     "@adonisjs/inertia": "^3.1.1",
     "@adonisjs/lucid": "^21.6.1",
     "@adonisjs/session": "^7.5.1",
@@ -82,6 +82,7 @@
     "autoprefixer": "^10.4.21",
     "axios": "^1.13.1",
     "better-sqlite3": "^12.1.1",
+    "bullmq": "^5.65.1",
     "dockerode": "^4.0.7",
     "edge.js": "^6.2.1",
     "fast-xml-parser": "^5.2.5",

admin/providers/map_static_provider.ts

@@ -0,0 +1,26 @@
+import MapsStaticMiddleware from '#middleware/maps_static_middleware'
+import logger from '@adonisjs/core/services/logger'
+import type { ApplicationService } from '@adonisjs/core/types'
+import { defineConfig } from '@adonisjs/static'
+import { join } from 'path'
+
+/**
+ * This is a bit of a hack to serve static files from the
+ * /storage/maps directory using AdonisJS static middleware because
+ * the middleware does not allow us to define a custom path we want
+ * to serve (it always serves from public/ by default).
+ *
+ * We use the same other config options, just change the path
+ * (though we could also separate config if needed).
+ */
+export default class MapStaticProvider {
+  constructor(protected app: ApplicationService) {}
+  register() {
+    this.app.container.singleton(MapsStaticMiddleware, () => {
+      const path = join(process.cwd(), '/storage/maps')
+      logger.debug(`Maps static files will be served from ${path}`)
+      const config = this.app.config.get<any>('static', defineConfig({}))
+      return new MapsStaticMiddleware(path, config)
+    })
+  }
+}

admin/start/env.ts

@@ -26,13 +26,6 @@ export default await Env.create(new URL('../', import.meta.url), {
   */
   //SESSION_DRIVER: Env.schema.enum(['cookie', 'memory'] as const),
 
-  /*
-  |----------------------------------------------------------
-  | Variables for configuring the drive package
-  |----------------------------------------------------------
-  */
-  DRIVE_DISK: Env.schema.enum(['fs'] as const),
-
 
   /*
   |----------------------------------------------------------
@@ -45,4 +38,12 @@ export default await Env.create(new URL('../', import.meta.url), {
   DB_PASSWORD: Env.schema.string.optional(),
   DB_DATABASE: Env.schema.string(),
   DB_SSL: Env.schema.boolean.optional(),
+
+  /*
+  |----------------------------------------------------------
+  | Variables for configuring the Redis connection
+  |----------------------------------------------------------
+  */
+  REDIS_HOST: Env.schema.string({ format: 'host' }),
+  REDIS_PORT: Env.schema.number(),
 })

admin/start/kernel.ts

@@ -27,7 +27,8 @@ server.use([
   () => import('@adonisjs/cors/cors_middleware'),
   () => import('@adonisjs/vite/vite_middleware'),
   () => import('@adonisjs/inertia/inertia_middleware'),
-  () => import('@adonisjs/static/static_middleware')
+  () => import('@adonisjs/static/static_middleware'),
+  () => import('#middleware/maps_static_middleware')
 ])
 
 /**

admin/start/routes.ts

@@ -7,6 +7,7 @@
 |
 */
 import DocsController from '#controllers/docs_controller'
+import DownloadsController from '#controllers/downloads_controller'
 import HomeController from '#controllers/home_controller'
 import MapsController from '#controllers/maps_controller'
 import SettingsController from '#controllers/settings_controller'
@@ -64,6 +65,13 @@ router
   })
   .prefix('/api/docs')
 
+router
+  .group(() => {
+    router.get('/jobs', [DownloadsController, 'index'])
+    router.get('/jobs/:filetype', [DownloadsController, 'filetype'])
+  })
+  .prefix('/api/downloads')
+
 router
   .group(() => {
     router.get('/info', [SystemController, 'getSystemInfo'])
@@ -77,7 +85,6 @@ router
   .group(() => {
     router.get('/list', [ZimController, 'list'])
     router.get('/list-remote', [ZimController, 'listRemote'])
-    router.get('/active-downloads', [ZimController, 'listActiveDownloads'])
     router.get('/curated-collections', [ZimController, 'listCuratedCollections'])
     router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections'])
     router.post('/download-remote', [ZimController, 'downloadRemote'])

admin/types/downloads.ts

@@ -12,6 +12,7 @@ export type DoResumableDownloadParams = {
   allowedMimeTypes: string[]
   signal?: AbortSignal
   onProgress?: (progress: DoResumableDownloadProgress) => void
+  onComplete?: (url: string, path: string) => void | Promise<void>
   forceNew?: boolean
 }
 
@@ -29,15 +30,6 @@ export type DoResumableDownloadProgress = {
   url: string
 }
 
-export type DoBackgroundDownloadParams = Omit<
-  DoResumableDownloadWithRetryParams,
-  'onProgress' | 'onAttemptError' | 'signal'
-> & {
-  channel: string
-  activeDownloads: Map<string, AbortController>
-  onComplete?: (url: string, path: string) => void | Promise<void>
-}
-
 export type CuratedCollection = {
   name: string
   slug: string
@@ -59,3 +51,18 @@ export type CuratedCollectionWithStatus = CuratedCollection & {
 export type CuratedCollectionsFile = {
   collections: CuratedCollection[]
 }
+
+export type RunDownloadJobParams = Omit<
+  DoResumableDownloadParams,
+  'onProgress' | 'onComplete' | 'signal'
+> & {
+  filetype: string
+}
+
+export type DownloadJobWithProgress = {
+  jobId: string
+  url: string
+  progress: number
+  filepath: string
+  filetype: string
+}

admin/util/broadcast_channels.ts

@@ -1,5 +1,3 @@
-
 export const BROADCAST_CHANNELS = {
-    ZIM: 'zim-downloads',
+  DOWNLOADS: 'downloads',
-    MAP: 'map-downloads',
+}
-}

admin/util/docs.ts

@@ -1,6 +1,5 @@
-import { Readable } from 'stream';
 
-export const streamToString = async (stream: Readable): Promise<string> => {
+export const streamToString = async (stream: NodeJS.ReadableStream): Promise<string> => {
     const chunks: Buffer[] = [];
     for await (const chunk of stream) {
         chunks.push(Buffer.from(chunk));

install/entrypoint.sh

@@ -17,6 +17,10 @@ node ace migration:run --force
 echo "Seeding the database..."
 node ace db:seed
 
+# Start background worker for queues
+echo "Starting background worker for queues..."
+node ace queue:work --queue=downloads &
+
 # Start the AdonisJS application
 echo "Starting AdonisJS application..."
 exec node bin/server.js

install/management_compose.yaml

@@ -15,7 +15,6 @@ services:
       - NODE_ENV=production
       - PORT=8080
       - LOG_LEVEL=debug
-      - DRIVE_DISK=fs
       - APP_KEY=replaceme
       - HOST=0.0.0.0
       - URL=replaceme
@@ -29,6 +28,8 @@ services:
     depends_on:
       mysql:
         condition: service_healthy
+      redis:
+        condition: service_healthy
     entrypoint: ["/usr/local/bin/entrypoint.sh"]
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health"]
@@ -63,4 +64,17 @@ services:
       test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
       interval: 30s
       timeout: 10s
+      retries: 3
+  redis:
+    image: redis:7-alpine
+    container_name: nomad_redis
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    volumes:
+      - /opt/project-nomad/redis:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 30s
+      timeout: 10s
       retries: 3