diff --git a/admin/.env.example b/admin/.env.example index cc9260b..6ae2d58 100644 --- a/admin/.env.example +++ b/admin/.env.example @@ -4,7 +4,6 @@ LOG_LEVEL=info APP_KEY=some_random_key NODE_ENV=development SESSION_DRIVER=cookie -DRIVE_DISK=fs DB_HOST=localhost DB_PORT=3306 DB_USER=root diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 5e505e0..6cab20d 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -52,8 +52,8 @@ export default defineConfig({ () => import('@adonisjs/cors/cors_provider'), () => import('@adonisjs/lucid/database_provider'), () => import('@adonisjs/inertia/inertia_provider'), - () => import('@adonisjs/drive/drive_provider'), - () => import('@adonisjs/transmit/transmit_provider') + () => import('@adonisjs/transmit/transmit_provider'), + () => import('#providers/map_static_provider') ], /* diff --git a/admin/app/controllers/downloads_controller.ts b/admin/app/controllers/downloads_controller.ts new file mode 100644 index 0000000..bd58790 --- /dev/null +++ b/admin/app/controllers/downloads_controller.ts @@ -0,0 +1,18 @@ +import type { HttpContext } from '@adonisjs/core/http' +import { DownloadService } from '#services/download_service' +import { downloadJobsByFiletypeSchema } from '#validators/download' +import { inject } from '@adonisjs/core' + +@inject() +export default class DownloadsController { + constructor(private downloadService: DownloadService) {} + + async index() { + return this.downloadService.listDownloadJobs() + } + + async filetype({ request }: HttpContext) { + const payload = await request.validateUsing(downloadJobsByFiletypeSchema) + return this.downloadService.listDownloadJobs(payload.params.filetype) + } +} diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index 23bf39f..1bf8fc7 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -1,6 +1,6 @@ import { MapService } from '#services/map_service' import { - filenameValidator, + filenameParamValidator, remoteDownloadValidator, remoteDownloadValidatorOptional, } from '#validators/common' @@ -53,14 +53,14 @@ export default class MapsController { } async delete({ request, response }: HttpContext) { - const payload = await request.validateUsing(filenameValidator) + const payload = await request.validateUsing(filenameParamValidator) try { - await this.mapService.delete(payload.filename) + await this.mapService.delete(payload.params.filename) } catch (error) { if (error.message === 'not_found') { return response.status(404).send({ - message: `Map file with key ${payload.filename} not found`, + message: `Map file with key ${payload.params.filename} not found`, }) } throw error // Re-throw any other errors and let the global error handler catch diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 5e4a029..e556964 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -1,7 +1,7 @@ import { ZimService } from '#services/zim_service' import { downloadCollectionValidator, - filenameValidator, + filenameParamValidator, remoteDownloadValidator, } from '#validators/common' import { listRemoteZimValidator } from '#validators/zim' @@ -24,11 +24,12 @@ export default class ZimController { async downloadRemote({ request }: HttpContext) { const payload = await request.validateUsing(remoteDownloadValidator) - const filename = await this.zimService.downloadRemote(payload.url) + const { filename, jobId } = await this.zimService.downloadRemote(payload.url) return { message: 'Download started successfully', filename, + jobId, url: payload.url, } } @@ -44,10 +45,6 @@ export default class ZimController { } } - async listActiveDownloads({}: HttpContext) { - return this.zimService.listActiveDownloads() - } - async listCuratedCollections({}: HttpContext) { return this.zimService.listCuratedCollections() } @@ -58,14 +55,14 @@ export default class ZimController { } async delete({ request, response }: HttpContext) { - const payload = await request.validateUsing(filenameValidator) + const payload = await request.validateUsing(filenameParamValidator) try { - await this.zimService.delete(payload.filename) + await this.zimService.delete(payload.params.filename) } catch (error) { if (error.message === 'not_found') { return response.status(404).send({ - message: `ZIM file with key ${payload.filename} not found`, + message: `ZIM file with key ${payload.params.filename} not found`, }) } throw error // Re-throw any other errors and let the global error handler catch diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts new file mode 100644 index 0000000..0f788d0 --- /dev/null +++ b/admin/app/jobs/run_download_job.ts @@ -0,0 +1,107 @@ +import { Job } from 'bullmq' +import { RunDownloadJobParams } from '../../types/downloads.js' +import { QueueService } from '#services/queue_service' +import { doResumableDownload } from '../utils/downloads.js' +import { createHash } from 'crypto' +import { DockerService } from '#services/docker_service' +import { ZimService } from '#services/zim_service' + +export class RunDownloadJob { + static get queue() { + return 'downloads' + } + + static get key() { + return 'run-download' + } + + static getJobId(url: string): string { + return createHash('sha256').update(url).digest('hex').slice(0, 16) + } + + async handle(job: Job) { + const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } = + job.data as RunDownloadJobParams + + // console.log("Simulating delay for job for URL:", url) + // await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay + // console.log("Starting download for URL:", url) + + // // simulate progress updates for demonstration + // for (let progress = 0; progress <= 100; progress += 10) { + // await new Promise((resolve) => setTimeout(resolve, 20000)) // Simulate time taken for each progress step + // job.updateProgress(progress) + // console.log(`Job progress for URL ${url}: ${progress}%`) + // } + + await doResumableDownload({ + url, + filepath, + timeout, + allowedMimeTypes, + forceNew, + onProgress(progress) { + const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100 + job.updateProgress(Math.floor(progressPercent)) + }, + async onComplete(url) { + if (filetype === 'zim') { + try { + const dockerService = new DockerService() + const zimService = new ZimService(dockerService) + await zimService.downloadRemoteSuccessCallback([url], true) + } catch (error) { + console.error( + `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`, + error + ) + } + } + job.updateProgress(100) + }, + }) + + return { + url, + filepath, + } + } + + static async getByUrl(url: string): Promise { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const jobId = this.getJobId(url) + return await queue.getJob(jobId) + } + + static async dispatch(params: RunDownloadJobParams) { + const queueService = new QueueService() + const queue = queueService.getQueue(this.queue) + const jobId = this.getJobId(params.url) + + try { + const job = await queue.add(this.key, params, { + jobId, + attempts: 3, + backoff: { type: 'exponential', delay: 2000 }, + removeOnComplete: true, + }) + + return { + job, + created: true, + message: `Dispatched download job for URL ${params.url}`, + } + } catch (error) { + if (error.message.includes('job already exists')) { + const existing = await queue.getJob(jobId) + return { + job: existing, + created: false, + message: `Job already exists for URL ${params.url}`, + } + } + throw error + } + } +} diff --git a/admin/app/middleware/maps_static_middleware.ts b/admin/app/middleware/maps_static_middleware.ts new file mode 100644 index 0000000..0766d41 --- /dev/null +++ b/admin/app/middleware/maps_static_middleware.ts @@ -0,0 +1,20 @@ +import type { HttpContext } from '@adonisjs/core/http' +import type { NextFn } from '@adonisjs/core/types/http' +import StaticMiddleware from '@adonisjs/static/static_middleware' +import { AssetsConfig } from '@adonisjs/static/types' + +/** + * See #providers/map_static_provider.ts for explanation + * of why this middleware exists. + */ +export default class MapsStaticMiddleware { + constructor( + private path: string, + private config: AssetsConfig + ) {} + + async handle(ctx: HttpContext, next: NextFn) { + const staticMiddleware = new StaticMiddleware(this.path, this.config) + return staticMiddleware.handle(ctx, next) + } +} diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 7d07492..d332fc3 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -4,7 +4,7 @@ import logger from '@adonisjs/core/services/logger' import { inject } from '@adonisjs/core' import { ServiceStatus } from '../../types/services.js' import transmit from '@adonisjs/transmit/services/main' -import { doSimpleDownload } from '../utils/downloads.js' +import { doResumableDownloadWithRetry } from '../utils/downloads.js' import path from 'path' @inject() @@ -347,10 +347,15 @@ export class DockerService { ) try { - await doSimpleDownload({ + await doResumableDownloadWithRetry({ url: WIKIPEDIA_ZIM_URL, filepath, timeout: 60000, + allowedMimeTypes: [ + 'application/x-zim', + 'application/x-openzim', + 'application/octet-stream', + ], }) this._broadcast( diff --git a/admin/app/services/docs_service.ts b/admin/app/services/docs_service.ts index e314965..2c27e75 100644 --- a/admin/app/services/docs_service.ts +++ b/admin/app/services/docs_service.ts @@ -1,73 +1,65 @@ -import drive from '@adonisjs/drive/services/main'; -import Markdoc from '@markdoc/markdoc'; -import { streamToString } from '../../util/docs.js'; +import Markdoc from '@markdoc/markdoc' +import { streamToString } from '../../util/docs.js' +import { getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js' +import path from 'path' export class DocsService { + private docsPath = path.join(process.cwd(), 'docs') async getDocs() { - const disk = drive.use('docs'); - if (!disk) { - throw new Error('Docs disk not configured'); - } + const contents = await listDirectoryContentsRecursive(this.docsPath) + const files: Array<{ title: string; slug: string }> = [] - const contents = await disk.listAll('/'); - const files: Array<{ title: string; slug: string }> = []; - - for (const item of contents.objects) { - if (item.isFile && item.name.endsWith('.md')) { - const cleaned = this.prettify(item.name); + for (const item of contents) { + if (item.type === 'file' && item.name.endsWith('.md')) { + const cleaned = this.prettify(item.name) files.push({ title: cleaned, - slug: item.name.replace(/\.md$/, '') - }); + slug: item.name.replace(/\.md$/, ''), + }) } } - return files.sort((a, b) => a.title.localeCompare(b.title)); + return files.sort((a, b) => a.title.localeCompare(b.title)) } parse(content: string) { - const ast = Markdoc.parse(content); - const config = this.getConfig(); - const errors = Markdoc.validate(ast, config); + const ast = Markdoc.parse(content) + const config = this.getConfig() + const errors = Markdoc.validate(ast, config) if (errors.length > 0) { - throw new Error(`Markdoc validation errors: ${errors.map(e => e.error).join(', ')}`); + throw new Error(`Markdoc validation errors: ${errors.map((e) => e.error).join(', ')}`) } - return Markdoc.transform(ast, config); + return Markdoc.transform(ast, config) } async parseFile(_filename: string) { - const disk = drive.use('docs'); - if (!disk) { - throw new Error('Docs disk not configured'); - } - if (!_filename) { - throw new Error('Filename is required'); + throw new Error('Filename is required') } - const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`; + const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md` - const fileExists = await disk.exists(filename); + const fileExists = await getFileStatsIfExists(path.join(this.docsPath, filename)) if (!fileExists) { - throw new Error(`File not found: ${filename}`); + throw new Error(`File not found: ${filename}`) } - const fileStream = await disk.getStream(filename); + const fileStream = await getFile(path.join(this.docsPath, filename), 'stream') if (!fileStream) { - throw new Error(`Failed to read file stream: ${filename}`); + throw new Error(`Failed to read file stream: ${filename}`) } - const content = await streamToString(fileStream); - return this.parse(content); + const content = await streamToString(fileStream) + return this.parse(content) } private prettify(filename: string) { // Remove hyphens, underscores, and file extension - const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' '); + const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ') // Convert to Title Case - const titleCased = cleaned.replace(/\b\w/g, char => char.toUpperCase()); - return titleCased.charAt(0).toUpperCase() + titleCased.slice(1); + const titleCased = cleaned.replace(/\b\w/g, (char) => char.toUpperCase()) + return titleCased.charAt(0).toUpperCase() + titleCased.slice(1) } private getConfig() { @@ -79,12 +71,12 @@ export class DocsService { type: { type: String, default: 'info', - matches: ['info', 'warning', 'error', 'success'] + matches: ['info', 'warning', 'error', 'success'], }, title: { - type: String - } - } + type: String, + }, + }, }, }, nodes: { @@ -92,10 +84,10 @@ export class DocsService { render: 'Heading', attributes: { level: { type: Number, required: true }, - id: { type: String } - } - } - } + id: { type: String }, + }, + }, + }, } } -} \ No newline at end of file +} diff --git a/admin/app/services/download_service.ts b/admin/app/services/download_service.ts new file mode 100644 index 0000000..67d05aa --- /dev/null +++ b/admin/app/services/download_service.ts @@ -0,0 +1,25 @@ +import { inject } from '@adonisjs/core' +import { QueueService } from './queue_service.js' +import { RunDownloadJob } from '#jobs/run_download_job' +import { DownloadJobWithProgress } from '../../types/downloads.js' +import { normalize } from 'path' + +@inject() +export class DownloadService { + constructor(private queueService: QueueService) {} + + async listDownloadJobs(filetype?: string): Promise { + const queue = this.queueService.getQueue(RunDownloadJob.queue) + const jobs = await queue.getJobs(['waiting', 'active', 'delayed']) + + return jobs + .map((job) => ({ + jobId: job.id!.toString(), + url: job.data.url, + progress: parseInt(job.progress.toString(), 10), + filepath: normalize(job.data.filepath), + filetype: job.data.filetype, + })) + .filter((job) => !filetype || job.filetype === filetype) + } +} diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index 330d09e..0637ffe 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -1,6 +1,6 @@ import { BaseStylesFile, MapLayer } from '../../types/maps.js' import { FileEntry } from '../../types/files.js' -import { doBackgroundDownload, doResumableDownloadWithRetry } from '../utils/downloads.js' +import { doResumableDownloadWithRetry } from '../utils/downloads.js' import { extract } from 'tar' import env from '#start/env' import { @@ -14,7 +14,8 @@ import { import { join } from 'path' import urlJoin from 'url-join' import axios from 'axios' -import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js' +import { RunDownloadJob } from '#jobs/run_download_job' +import logger from '@adonisjs/core/services/logger' const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', @@ -32,7 +33,6 @@ export class MapService { private readonly basemapsAssetsDir = 'basemaps-assets' private readonly baseAssetsTarFile = 'base-assets.tar.gz' private readonly baseDirPath = join(process.cwd(), this.mapStoragePath) - private activeDownloads = new Map() async listRegions() { const files = (await this.listAllMapStorageItems()).filter( @@ -80,13 +80,13 @@ export class MapService { return true } - async downloadRemote(url: string): Promise { + async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> { const parsed = new URL(url) if (!parsed.pathname.endsWith('.pmtiles')) { throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`) } - const existing = this.activeDownloads.get(url) + const existing = await RunDownloadJob.getByUrl(url) if (existing) { throw new Error(`Download already in progress for URL ${url}`) } @@ -98,18 +98,26 @@ export class MapService { const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) - // Don't await the download, run it in the background - doBackgroundDownload({ + // Dispatch background job + const result = await RunDownloadJob.dispatch({ url, filepath, timeout: 30000, allowedMimeTypes: PMTILES_MIME_TYPES, forceNew: true, - channel: BROADCAST_CHANNELS.MAP, - activeDownloads: this.activeDownloads, + filetype: 'pmtiles', }) - return filename + if (!result.job) { + throw new Error('Failed to dispatch download job') + } + + logger.info(`[MapService] Dispatched download job ${result.job.id} for URL ${url}`) + + return { + filename, + jobId: result.job?.id, + } } async downloadRemotePreflight( diff --git a/admin/app/services/queue_service.ts b/admin/app/services/queue_service.ts new file mode 100644 index 0000000..fa3a050 --- /dev/null +++ b/admin/app/services/queue_service.ts @@ -0,0 +1,22 @@ +import { Queue } from 'bullmq' +import queueConfig from '#config/queue' + +export class QueueService { + private queues: Map = new Map() + + getQueue(name: string): Queue { + if (!this.queues.has(name)) { + const queue = new Queue(name, { + connection: queueConfig.connection, + }) + this.queues.set(name, queue) + } + return this.queues.get(name)! + } + + async close() { + for (const queue of this.queues.values()) { + await queue.close() + } + } +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 2d5424a..f3fbf4d 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -6,11 +6,9 @@ import { import axios from 'axios' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' -import transmit from '@adonisjs/transmit/services/main' import logger from '@adonisjs/core/services/logger' import { DockerService } from './docker_service.js' import { inject } from '@adonisjs/core' -import { doBackgroundDownload } from '../utils/downloads.js' import { deleteFileIfExists, ensureDirectoryExists, @@ -23,7 +21,7 @@ import vine from '@vinejs/vine' import { curatedCollectionsFileSchema } from '#validators/curated_collections' import CuratedCollection from '#models/curated_collection' import CuratedCollectionResource from '#models/curated_collection_resource' -import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js' +import { RunDownloadJob } from '#jobs/run_download_job' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const COLLECTIONS_URL = @@ -32,7 +30,6 @@ const COLLECTIONS_URL = @inject() export class ZimService { private zimStoragePath = '/storage/zim' - private activeDownloads = new Map() constructor(private dockerService: DockerService) {} @@ -140,15 +137,15 @@ export class ZimService { } } - async downloadRemote(url: string): Promise { + async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> { const parsed = new URL(url) if (!parsed.pathname.endsWith('.zim')) { throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`) } - const existing = this.activeDownloads.get(url) + const existing = await RunDownloadJob.getByUrl(url) if (existing) { - throw new Error(`Download already in progress for URL ${url}`) + throw new Error('A download for this URL is already in progress') } // Extract the filename from the URL @@ -159,19 +156,26 @@ export class ZimService { const filepath = join(process.cwd(), this.zimStoragePath, filename) - // Don't await the download, run it in the background - doBackgroundDownload({ + // Dispatch a background download job + const result = await RunDownloadJob.dispatch({ url, filepath, - channel: BROADCAST_CHANNELS.ZIM, - activeDownloads: this.activeDownloads, - allowedMimeTypes: ZIM_MIME_TYPES, timeout: 30000, + allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, - onComplete: (url) => this._downloadRemoteSuccessCallback([url]), + filetype: 'zim', }) - return filename + if (!result || !result.job) { + throw new Error('Failed to dispatch download job') + } + + logger.info(`[ZimService] Dispatched background download job for ZIM file: ${filename}`) + + return { + filename, + jobId: result.job.id, + } } async downloadCollection(slug: string): Promise { @@ -188,49 +192,43 @@ export class ZimService { const downloadUrls = resources.map((res) => res.url) const downloadFilenames: string[] = [] - for (const [idx, url] of downloadUrls.entries()) { - const existing = this.activeDownloads.get(url) + for (const url of downloadUrls) { + const existing = await RunDownloadJob.getByUrl(url) if (existing) { - logger.warn(`Download already in progress for URL ${url}, skipping.`) + logger.warn(`[ZimService] Download already in progress for URL ${url}, skipping.`) continue } // Extract the filename from the URL const filename = url.split('/').pop() if (!filename) { - logger.warn(`Could not determine filename from URL ${url}, skipping.`) + logger.warn(`[ZimService] Could not determine filename from URL ${url}, skipping.`) continue } - const filepath = join(process.cwd(), this.zimStoragePath, filename) downloadFilenames.push(filename) + const filepath = join(process.cwd(), this.zimStoragePath, filename) - const isLastDownload = idx === downloadUrls.length - 1 - - // Don't await the download, run it in the background - doBackgroundDownload({ + await RunDownloadJob.dispatch({ url, filepath, - channel: BROADCAST_CHANNELS.ZIM, - activeDownloads: this.activeDownloads, - allowedMimeTypes: ZIM_MIME_TYPES, timeout: 30000, + allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, - onComplete: (url) => - this._downloadRemoteSuccessCallback([url], isLastDownload), + filetype: 'zim', }) } return downloadFilenames.length > 0 ? downloadFilenames : null } - async _downloadRemoteSuccessCallback(urls: string[], restart = true) { + async downloadRemoteSuccessCallback(urls: string[], restart = true) { // Restart KIWIX container to pick up new ZIM file if (restart) { await this.dockerService .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart') .catch((error) => { - logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. + logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. }) } @@ -242,21 +240,6 @@ export class ZimService { } } - listActiveDownloads(): string[] { - return Array.from(this.activeDownloads.keys()) - } - - cancelDownload(url: string): boolean { - const entry = this.activeDownloads.get(url) - if (entry) { - entry.abort() - this.activeDownloads.delete(url) - transmit.broadcast(BROADCAST_CHANNELS.ZIM, { url, status: 'cancelled' }) - return true - } - return false - } - async listCuratedCollections(): Promise { const collections = await CuratedCollection.query().preload('resources') return collections.map((collection) => ({ @@ -282,17 +265,17 @@ export class ZimService { type: 'zim', } ) - logger.info(`Upserted curated collection: ${collection.slug}`) + logger.info(`[ZimService] Upserted curated collection: ${collection.slug}`) await collectionResult.related('resources').createMany(collection.resources) logger.info( - `Upserted ${collection.resources.length} resources for collection: ${collection.slug}` + `[ZimService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}` ) } return true } catch (error) { - logger.error('Failed to download latest Kiwix collections:', error) + logger.error(`[ZimService] Failed to download latest Kiwix collections:`, error) return false } } diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index 8e3b5eb..d8fb0d4 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -1,7 +1,5 @@ import { - DoBackgroundDownloadParams, DoResumableDownloadParams, - DoResumableDownloadProgress, DoResumableDownloadWithRetryParams, DoSimpleDownloadParams, } from '../../types/downloads.js' @@ -9,9 +7,6 @@ import axios, { AxiosResponse } from 'axios' import { Transform } from 'stream' import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js' import { createWriteStream } from 'fs' -import { formatSpeed } from './misc.js' -import { DownloadProgress } from '../../types/files.js' -import transmit from '@adonisjs/transmit/services/main' import logger from '@adonisjs/core/services/logger' import path from 'path' @@ -86,6 +81,7 @@ export async function doResumableDownload({ timeout = 30000, signal, onProgress, + onComplete, forceNew = false, allowedMimeTypes, }: DoResumableDownloadParams): Promise { @@ -200,7 +196,7 @@ export async function doResumableDownload({ cleanup(new Error('Download aborted')) }) - writeStream.on('finish', () => { + writeStream.on('finish', async () => { if (onProgress) { onProgress({ downloadedBytes, @@ -210,6 +206,9 @@ export async function doResumableDownload({ url, }) } + if (onComplete) { + await onComplete(url, filepath) + } resolve(filepath) }) @@ -276,82 +275,6 @@ export async function doResumableDownloadWithRetry({ throw lastError || new Error('Unknown error during download') } -export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise { - const { url, filepath, channel, activeDownloads, onComplete, ...restParams } = params - - try { - const dirname = path.dirname(filepath) - await ensureDirectoryExists(dirname) - - const abortController = new AbortController() - activeDownloads.set(url, abortController) - - await doResumableDownloadWithRetry({ - url, - filepath, - signal: abortController.signal, - ...restParams, - onProgress: (progressData) => { - sendProgressBroadcast(channel, progressData) - }, - }) - - sendCompletedBroadcast(channel, url, filepath) - - if (onComplete) { - await onComplete(url, filepath) - } - } catch (error) { - logger.error(`Background download failed for ${url}: ${error.message}`) - sendErrorBroadcast(channel, url, error.message) - } finally { - activeDownloads.delete(url) - } -} - -export function sendProgressBroadcast( - channel: string, - progressData: DoResumableDownloadProgress, - status = 'in_progress' -) { - const { downloadedBytes, totalBytes, lastProgressTime, lastDownloadedBytes, url } = progressData - const now = Date.now() - const timeDiff = (now - lastProgressTime) / 1000 - const bytesDiff = downloadedBytes - lastDownloadedBytes - const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0 - const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0 - const speed = formatSpeed(rawSpeed) - - const progress: DownloadProgress = { - downloaded_bytes: downloadedBytes, - total_bytes: totalBytes, - percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0, - speed, - time_remaining: timeRemaining, - } - - transmit.broadcast(channel, { url, progress, status }) -} - -export function sendCompletedBroadcast(channel: string, url: string, path: string) { - transmit.broadcast(channel, { - url, - path, - status: 'completed', - progress: { - downloaded_bytes: 0, - total_bytes: 0, - percentage: 100, - speed: '0 B/s', - time_remaining: 0, - }, - }) -} - -export function sendErrorBroadcast(channel: string, url: string, errorMessage: string) { - transmit.broadcast(channel, { url, error: errorMessage, status: 'failed' }) -} - async function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } diff --git a/admin/app/utils/fs.ts b/admin/app/utils/fs.ts index c1484fa..0580344 100644 --- a/admin/app/utils/fs.ts +++ b/admin/app/utils/fs.ts @@ -1,9 +1,7 @@ import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises' import { join } from 'path' -import { DriveDisks } from '@adonisjs/drive/types' -import driveConfig from '#config/drive' -import app from '@adonisjs/core/services/app' import { FileEntry } from '../../types/files.js' +import { createReadStream } from 'fs' export async function listDirectoryContents(path: string): Promise { const entries = await readdir(path, { withFileTypes: true }) @@ -56,14 +54,22 @@ export async function ensureDirectoryExists(path: string): Promise { } export async function getFile(path: string, returnType: 'buffer'): Promise +export async function getFile( + path: string, + returnType: 'stream' +): Promise export async function getFile(path: string, returnType: 'string'): Promise -export async function getFile(path: string, returnType: 'buffer' | 'string' = 'buffer'): Promise { +export async function getFile( + path: string, + returnType: 'buffer' | 'string' | 'stream' = 'buffer' +): Promise { try { - if (returnType === 'buffer') { - return await readFile(path) - } else { + if (returnType === 'string') { return await readFile(path, 'utf-8') + } else if (returnType === 'stream') { + return createReadStream(path) } + return await readFile(path) } catch (error) { if (error.code === 'ENOENT') { return null @@ -98,18 +104,3 @@ export async function deleteFileIfExists(path: string): Promise { } } } - -export async function getFullDrivePath(diskName: keyof DriveDisks): Promise { - const config = await driveConfig.resolver(app) - const serviceConfig = config.config.services[diskName] - const resolved = serviceConfig() - if (!resolved) { - throw new Error(`Disk ${diskName} not configured`) - } - - let path = resolved.options.location - if (path instanceof URL) { - return path.pathname - } - return path -} diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 285d4b9..93b68ab 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -23,9 +23,11 @@ export const remoteDownloadValidatorOptional = vine.compile( }) ) -export const filenameValidator = vine.compile( +export const filenameParamValidator = vine.compile( vine.object({ - filename: vine.string().trim().minLength(1).maxLength(4096), + params: vine.object({ + filename: vine.string().trim().minLength(1).maxLength(4096), + }), }) ) diff --git a/admin/app/validators/download.ts b/admin/app/validators/download.ts new file mode 100644 index 0000000..67ac46a --- /dev/null +++ b/admin/app/validators/download.ts @@ -0,0 +1,9 @@ +import vine from '@vinejs/vine' + +export const downloadJobsByFiletypeSchema = vine.compile( + vine.object({ + params: vine.object({ + filetype: vine.string(), + }), + }) +) diff --git a/admin/commands/queue/work.ts b/admin/commands/queue/work.ts new file mode 100644 index 0000000..3dd013c --- /dev/null +++ b/admin/commands/queue/work.ts @@ -0,0 +1,67 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import { Worker } from 'bullmq' +import queueConfig from '#config/queue' + +export default class QueueWork extends BaseCommand { + static commandName = 'queue:work' + static description = 'Start processing jobs from the queue' + + @flags.string({ description: 'Queue name to process', required: true }) + declare queue: string + + static options: CommandOptions = { + startApp: true, + staysAlive: true, + } + + async run() { + const queueName = this.queue || 'default' + + const jobHandlers = await this.loadJobHandlers() + + const worker = new Worker( + queueName, + async (job) => { + this.logger.info(`Processing job: ${job.id} of type: ${job.name}`) + const jobHandler = jobHandlers.get(job.name) + if (!jobHandler) { + throw new Error(`No handler found for job: ${job.name}`) + } + + return await jobHandler.handle(job) + }, + { + connection: queueConfig.connection, + concurrency: 3, + autorun: true, + } + ) + + worker.on('failed', (job, err) => { + this.logger.error(`Job failed: ${job?.id}, Error: ${err.message}`) + }) + + worker.on('completed', (job) => { + this.logger.info(`Job completed: ${job.id}`) + }) + + this.logger.info(`Worker started for queue: ${queueName}`) + + process.on('SIGTERM', async () => { + this.logger.info('SIGTERM received. Shutting down worker...') + await worker.close() + this.logger.info('Worker shut down gracefully.') + process.exit(0) + }) + } + + private async loadJobHandlers() { + const handlers = new Map() + + const { RunDownloadJob } = await import('#jobs/run_download_job') + handlers.set(RunDownloadJob.key, new RunDownloadJob()) + + return handlers + } +} diff --git a/admin/config/drive.ts b/admin/config/drive.ts deleted file mode 100644 index 72aa24e..0000000 --- a/admin/config/drive.ts +++ /dev/null @@ -1,31 +0,0 @@ -import env from '#start/env' -import app from '@adonisjs/core/services/app' -import { defineConfig, services } from '@adonisjs/drive' - -const driveConfig = defineConfig({ - default: env.get('DRIVE_DISK'), - - /** - * The services object can be used to configure multiple file system - * services each using the same or a different driver. - */ - services: { - fs: services.fs({ - location: app.makePath('storage'), - serveFiles: true, - routeBasePath: '/storage', - visibility: 'public', - }), - docs: services.fs({ - location: app.makePath('docs'), - serveFiles: false, // Don't serve files directly - we handle this via routes/Inertia - visibility: 'public', - }), - }, -}) - -export default driveConfig - -declare module '@adonisjs/drive/types' { - export interface DriveDisks extends InferDriveDisks { } -} \ No newline at end of file diff --git a/admin/config/queue.ts b/admin/config/queue.ts new file mode 100644 index 0000000..25053b8 --- /dev/null +++ b/admin/config/queue.ts @@ -0,0 +1,10 @@ +import env from '#start/env' + +const queueConfig = { + connection: { + host: env.get('REDIS_HOST'), + port: env.get('REDIS_PORT') ?? 6379, + }, +} + +export default queueConfig diff --git a/admin/config/static.ts b/admin/config/static.ts index 2e20fc0..4babdc0 100644 --- a/admin/config/static.ts +++ b/admin/config/static.ts @@ -12,6 +12,7 @@ const staticServerConfig = defineConfig({ etag: true, lastModified: true, dotFiles: 'ignore', + acceptRanges: true, }) export default staticServerConfig diff --git a/admin/inertia/components/systeminfo/HorizontalBarChart.tsx b/admin/inertia/components/HorizontalBarChart.tsx similarity index 59% rename from admin/inertia/components/systeminfo/HorizontalBarChart.tsx rename to admin/inertia/components/HorizontalBarChart.tsx index ea59c63..7c90a36 100644 --- a/admin/inertia/components/systeminfo/HorizontalBarChart.tsx +++ b/admin/inertia/components/HorizontalBarChart.tsx @@ -8,11 +8,23 @@ interface HorizontalBarChartProps { used: string type?: string }> - maxValue?: number + statuses?: Array<{ + label: string + min_threshold: number + color_class: string + }> + progressiveBarColor?: boolean } -export default function HorizontalBarChart({ items, maxValue = 100 }: HorizontalBarChartProps) { +export default function HorizontalBarChart({ + items, + statuses, + progressiveBarColor = false, +}: HorizontalBarChartProps) { + const sortedStatus = statuses?.sort((a, b) => b.min_threshold - a.min_threshold) || [] + const getBarColor = (value: number) => { + if (!progressiveBarColor) return 'bg-desert-green' if (value >= 90) return 'bg-desert-red' if (value >= 75) return 'bg-desert-orange' if (value >= 50) return 'bg-desert-tan' @@ -26,6 +38,26 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal return 'shadow-desert-olive/50' } + const getStatusLabel = (value: number) => { + if (sortedStatus.length === 0) return '' + for (const status of sortedStatus) { + if (value >= status.min_threshold) { + return status.label + } + } + return '' + } + + const getStatusColor = (value: number) => { + if (sortedStatus.length === 0) return '' + for (const status of sortedStatus) { + if (value >= status.min_threshold) { + return status.color_class + } + } + return '' + } + return (
{items.map((item, index) => ( @@ -56,28 +88,7 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal width: `${item.value}%`, animationDelay: `${index * 100}ms`, }} - > - {/* Animated shine effect */} - {/*
*/} - {/*
*/} -
+ >
-
-
= 90 - ? 'bg-desert-red' - : item.value >= 75 - ? 'bg-desert-orange' - : 'bg-desert-olive' - )} - /> - - {item.value >= 90 - ? 'Critical - Disk Almost Full' - : item.value >= 75 - ? 'Warning - Usage High' - : 'Normal'} - -
+ {getStatusLabel(item.value) && ( +
+
+ {getStatusLabel(item.value)} +
+ )}
))}
diff --git a/admin/inertia/hooks/useDownloads.ts b/admin/inertia/hooks/useDownloads.ts new file mode 100644 index 0000000..6d9b33b --- /dev/null +++ b/admin/inertia/hooks/useDownloads.ts @@ -0,0 +1,31 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { useMemo } from 'react' +import api from '~/lib/api' + +export type useDownloadsProps = { + filetype?: string + enabled?: boolean +} + +const useDownloads = (props: useDownloadsProps) => { + const queryClient = useQueryClient() + + const queryKey = useMemo(() => { + return props.filetype ? ['download-jobs', props.filetype] : ['download-jobs'] + }, [props.filetype]) + + const queryData = useQuery({ + queryKey: queryKey, + queryFn: () => api.listDownloadJobs(props.filetype), + refetchInterval: 2000, // Refetch every 2 seconds to get updated progress + enabled: props.enabled ?? true, + }) + + const invalidate = () => { + queryClient.invalidateQueries({ queryKey: queryKey }) + } + + return { ...queryData, invalidate } +} + +export default useDownloads diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index b9ef741..3384f2c 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -3,7 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { SystemInformationResponse } from '../../types/system' -import { CuratedCollectionWithStatus } from '../../types/downloads' +import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads' class API { private client: AxiosInstance @@ -142,16 +142,6 @@ class API { } } - async listActiveZimDownloads(): Promise { - try { - const response = await this.client.get('/zim/active-downloads') - return response.data - } catch (error) { - console.error('Error listing active ZIM downloads:', error) - throw error - } - } - async downloadRemoteZimFile(url: string): Promise<{ message: string filename: string @@ -209,6 +199,17 @@ class API { throw error } } + + async listDownloadJobs(filetype?: string): Promise { + try { + const endpoint = filetype ? `/downloads/jobs/${filetype}` : '/downloads/jobs' + const response = await this.client.get(endpoint) + return response.data + } catch (error) { + console.error('Error listing download jobs:', error) + throw error + } + } } export default new API() diff --git a/admin/inertia/pages/settings/system.tsx b/admin/inertia/pages/settings/system.tsx index 1615055..64ce6a6 100644 --- a/admin/inertia/pages/settings/system.tsx +++ b/admin/inertia/pages/settings/system.tsx @@ -3,7 +3,7 @@ import SettingsLayout from '~/layouts/SettingsLayout' import { SystemInformationResponse } from '../../../types/system' import { formatBytes } from '~/lib/util' import CircularGauge from '~/components/systeminfo/CircularGauge' -import HorizontalBarChart from '~/components/systeminfo/HorizontalBarChart' +import HorizontalBarChart from '~/components/HorizontalBarChart' import InfoCard from '~/components/systeminfo/InfoCard' import { CpuChipIcon, @@ -195,7 +195,27 @@ export default function SettingsPage(props: {
{diskData && diskData.length > 0 ? ( - + ) : (
No storage devices detected diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index 0b86dc8..3aa3d53 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -16,8 +16,6 @@ import { formatBytes } from '~/lib/util' import StyledButton from '~/components/StyledButton' import { useModals } from '~/context/ModalContext' import StyledModal from '~/components/StyledModal' -import { useTransmit } from 'react-adonis-transmit' -import ProgressBar from '~/components/ProgressBar' import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' import Alert from '~/components/Alert' @@ -28,7 +26,8 @@ import useDebounce from '~/hooks/useDebounce' import CuratedCollectionCard from '~/components/CuratedCollectionCard' import StyledSectionHeader from '~/components/StyledSectionHeader' import { CuratedCollectionWithStatus } from '../../../../types/downloads' -import { BROADCAST_CHANNELS } from '../../../../util/broadcast_channels' +import useDownloads from '~/hooks/useDownloads' +import HorizontalBarChart from '~/components/HorizontalBarChart' const CURATED_COLLECTIONS_KEY = 'curated-zim-collections' @@ -36,7 +35,6 @@ export default function ZimRemoteExplorer() { const queryClient = useQueryClient() const tableParentRef = useRef(null) - const { subscribe } = useTransmit() const { openModal, closeAllModals } = useModals() const { addNotification } = useNotifications() const { isOnline } = useInternetStatus() @@ -45,44 +43,22 @@ export default function ZimRemoteExplorer() { const [query, setQuery] = useState('') const [queryUI, setQueryUI] = useState('') - const [activeDownloads, setActiveDownloads] = useState< - Map - >(new Map()) const debouncedSetQuery = debounce((val: string) => { setQuery(val) }, 400) - useEffect(() => { - const unsubscribe = subscribe(BROADCAST_CHANNELS.ZIM, (data: any) => { - if (data.url && data.progress?.percentage) { - setActiveDownloads((prev) => - new Map(prev).set(data.url, { - status: data.status, - progress: data.progress.percentage || 0, - speed: data.progress.speed || '0 KB/s', - }) - ) - if (data.status === 'completed') { - addNotification({ - message: `The download for ${data.url} has completed successfully.`, - type: 'success', - }) - } - } - }) - - return () => { - unsubscribe() - } - }, []) - const { data: curatedCollections } = useQuery({ queryKey: [CURATED_COLLECTIONS_KEY], queryFn: () => api.listCuratedZimCollections(), refetchOnWindowFocus: false, }) + const { data: downloads, invalidate: invalidateDownloads } = useDownloads({ + filetype: 'zim', + enabled: true, + }) + const { data, fetchNextPage, isFetching, isLoading } = useInfiniteQuery({ queryKey: ['remote-zim-files', query], @@ -103,7 +79,17 @@ export default function ZimRemoteExplorer() { placeholderData: keepPreviousData, }) - const flatData = useMemo(() => data?.pages.flatMap((page) => page.items) || [], [data]) + const flatData = useMemo(() => { + const mapped = data?.pages.flatMap((page) => page.items) || [] + // remove items that are currently downloading + return mapped.filter((item) => { + const isDownloading = downloads?.some((download) => { + const filename = item.download_url.split('/').pop() + return filename && download.filepath.endsWith(filename) + }) + return !isDownloading + }) + }, [data, downloads]) const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data]) const fetchOnBottomReached = useCallback( @@ -176,6 +162,7 @@ export default function ZimRemoteExplorer() { async function downloadFile(record: RemoteZimFileEntry) { try { await api.downloadRemoteZimFile(record.download_url) + invalidateDownloads() } catch (error) { console.error('Error downloading file:', error) } @@ -184,19 +171,12 @@ export default function ZimRemoteExplorer() { async function downloadCollection(record: CuratedCollectionWithStatus) { try { await api.downloadZimCollection(record.slug) + invalidateDownloads() } catch (error) { console.error('Error downloading collection:', error) } } - const EntryProgressBar = useCallback( - ({ url }: { url: string }) => { - const entry = activeDownloads.get(url) - return - }, - [activeDownloads] - ) - const fetchLatestCollections = useMutation({ mutationFn: () => api.fetchLatestZimCollections(), onSuccess: () => { @@ -208,6 +188,17 @@ export default function ZimRemoteExplorer() { }, }) + const extractFileName = (path: string) => { + if (!path) return '' + if (path.includes('/')) { + return path.substring(path.lastIndexOf('/') + 1) + } + if (path.includes('\\')) { + return path.substring(path.lastIndexOf('\\') + 1) + } + return path + } + return ( @@ -307,20 +298,16 @@ export default function ZimRemoteExplorer() { { accessor: 'actions', render(record) { - const isDownloading = activeDownloads.has(record.download_url) return (
- {!isDownloading && ( - { - confirmDownload(record) - }} - > - Download - - )} - {isDownloading && } + { + confirmDownload(record) + }} + > + Download +
) }, @@ -337,6 +324,28 @@ export default function ZimRemoteExplorer() { compact rowLines /> + +
+ {downloads && downloads.length > 0 ? ( + downloads.map((download) => ( +
+ +
+ )) + ) : ( +

No active downloads

+ )} +
diff --git a/admin/package-lock.json b/admin/package-lock.json index 9d31116..6fe2d3c 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -12,7 +12,6 @@ "@adonisjs/auth": "^9.4.0", "@adonisjs/core": "^6.18.0", "@adonisjs/cors": "^2.2.1", - "@adonisjs/drive": "^3.4.1", "@adonisjs/inertia": "^3.1.1", "@adonisjs/lucid": "^21.6.1", "@adonisjs/session": "^7.5.1", @@ -36,6 +35,7 @@ "autoprefixer": "^10.4.21", "axios": "^1.13.1", "better-sqlite3": "^12.1.1", + "bullmq": "^5.65.1", "dockerode": "^4.0.7", "edge.js": "^6.2.1", "fast-xml-parser": "^5.2.5", @@ -339,35 +339,6 @@ "@adonisjs/core": "^6.2.0" } }, - "node_modules/@adonisjs/drive": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@adonisjs/drive/-/drive-3.4.1.tgz", - "integrity": "sha512-oDYY4wJ7wDMlO4E+dZPYBu+T3Av7Mj+JL8+J33qgyxtiJylnZgoZDuRfFjZZix/bFNNuWX2sLwTMnyiDcK+YsA==", - "license": "MIT", - "dependencies": { - "flydrive": "^1.1.0" - }, - "engines": { - "node": ">=20.6.0" - }, - "peerDependencies": { - "@adonisjs/core": "^6.2.0", - "@aws-sdk/client-s3": "^3.577.0", - "@aws-sdk/s3-request-presigner": "^3.577.0", - "@google-cloud/storage": "^7.10.2" - }, - "peerDependenciesMeta": { - "@aws-sdk/client-s3": { - "optional": true - }, - "@aws-sdk/s3-request-presigner": { - "optional": true - }, - "@google-cloud/storage": { - "optional": true - } - } - }, "node_modules/@adonisjs/encryption": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@adonisjs/encryption/-/encryption-6.0.2.tgz", @@ -2083,6 +2054,7 @@ "version": "0.4.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18.18" @@ -2116,6 +2088,12 @@ "react": "^16.9.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz", + "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==", + "license": "MIT" + }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -2461,6 +2439,84 @@ } } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@noble/hashes": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", @@ -5098,6 +5154,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bullmq": { + "version": "5.65.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.65.1.tgz", + "integrity": "sha512-QgDAzX1G9L5IRy4Orva5CfQTXZT+5K+OfO/kbPrAqN+pmL9LJekCzxijXehlm/u2eXfWPfWvIdJJIqiuz3WJSg==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.9.0", + "ioredis": "^5.8.2", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^11.1.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -5522,6 +5593,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/code-block-writer": { "version": "13.0.3", "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-13.0.3.tgz", @@ -5710,6 +5790,18 @@ "dev": true, "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -5768,9 +5860,9 @@ "license": "MIT" }, "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -7001,58 +7093,6 @@ "node": ">=8" } }, - "node_modules/flydrive": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/flydrive/-/flydrive-1.2.0.tgz", - "integrity": "sha512-l9ix5MhBE8bVwxyHdFku6z5KhGOCOXQDI9xGNIlACSz9UrDFQxAB1I6W0qffZiOBBDambiJZlEYBCxlvF4U7fw==", - "license": "MIT", - "dependencies": { - "@humanwhocodes/retry": "^0.4.2", - "@poppinss/utils": "^6.9.2", - "etag": "^1.8.1", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">=20.6.0" - }, - "peerDependencies": { - "@aws-sdk/client-s3": "^3.577.0", - "@aws-sdk/s3-request-presigner": "^3.577.0", - "@google-cloud/storage": "^7.10.2" - }, - "peerDependenciesMeta": { - "@aws-sdk/client-s3": { - "optional": true - }, - "@aws-sdk/s3-request-presigner": { - "optional": true - }, - "@google-cloud/storage": { - "optional": true - } - } - }, - "node_modules/flydrive/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/flydrive/node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/follow-redirects": { "version": "1.15.9", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", @@ -7751,6 +7791,30 @@ "node": ">= 0.10" } }, + "node_modules/ioredis": { + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz", + "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.4.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8469,6 +8533,18 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -8900,6 +8976,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", + "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/murmurhash-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz", @@ -9015,6 +9122,27 @@ "node": ">=10" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-releases": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", @@ -9921,16 +10049,61 @@ } }, "node_modules/raw-body": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", - "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.6.3", - "unpipe": "1.0.0" + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -10182,6 +10355,27 @@ "node": ">= 10.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -10963,6 +11157,12 @@ "get-source": "^2.0.12" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -11777,6 +11977,19 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/admin/package.json b/admin/package.json index 8d03582..8b12b58 100644 --- a/admin/package.json +++ b/admin/package.json @@ -30,7 +30,8 @@ "#database/*": "./database/*.js", "#tests/*": "./tests/*.js", "#start/*": "./start/*.js", - "#config/*": "./config/*.js" + "#config/*": "./config/*.js", + "#jobs/*": "./app/jobs/*.js" }, "devDependencies": { "@adonisjs/assembler": "^7.8.2", @@ -58,7 +59,6 @@ "@adonisjs/auth": "^9.4.0", "@adonisjs/core": "^6.18.0", "@adonisjs/cors": "^2.2.1", - "@adonisjs/drive": "^3.4.1", "@adonisjs/inertia": "^3.1.1", "@adonisjs/lucid": "^21.6.1", "@adonisjs/session": "^7.5.1", @@ -82,6 +82,7 @@ "autoprefixer": "^10.4.21", "axios": "^1.13.1", "better-sqlite3": "^12.1.1", + "bullmq": "^5.65.1", "dockerode": "^4.0.7", "edge.js": "^6.2.1", "fast-xml-parser": "^5.2.5", diff --git a/admin/providers/map_static_provider.ts b/admin/providers/map_static_provider.ts new file mode 100644 index 0000000..07a0e2c --- /dev/null +++ b/admin/providers/map_static_provider.ts @@ -0,0 +1,26 @@ +import MapsStaticMiddleware from '#middleware/maps_static_middleware' +import logger from '@adonisjs/core/services/logger' +import type { ApplicationService } from '@adonisjs/core/types' +import { defineConfig } from '@adonisjs/static' +import { join } from 'path' + +/** + * This is a bit of a hack to serve static files from the + * /storage/maps directory using AdonisJS static middleware because + * the middleware does not allow us to define a custom path we want + * to serve (it always serves from public/ by default). + * + * We use the same other config options, just change the path + * (though we could also separate config if needed). + */ +export default class MapStaticProvider { + constructor(protected app: ApplicationService) {} + register() { + this.app.container.singleton(MapsStaticMiddleware, () => { + const path = join(process.cwd(), '/storage/maps') + logger.debug(`Maps static files will be served from ${path}`) + const config = this.app.config.get('static', defineConfig({})) + return new MapsStaticMiddleware(path, config) + }) + } +} diff --git a/admin/start/env.ts b/admin/start/env.ts index 4cb7593..ab1ec20 100644 --- a/admin/start/env.ts +++ b/admin/start/env.ts @@ -26,13 +26,6 @@ export default await Env.create(new URL('../', import.meta.url), { */ //SESSION_DRIVER: Env.schema.enum(['cookie', 'memory'] as const), - /* - |---------------------------------------------------------- - | Variables for configuring the drive package - |---------------------------------------------------------- - */ - DRIVE_DISK: Env.schema.enum(['fs'] as const), - /* |---------------------------------------------------------- @@ -45,4 +38,12 @@ export default await Env.create(new URL('../', import.meta.url), { DB_PASSWORD: Env.schema.string.optional(), DB_DATABASE: Env.schema.string(), DB_SSL: Env.schema.boolean.optional(), + + /* + |---------------------------------------------------------- + | Variables for configuring the Redis connection + |---------------------------------------------------------- + */ + REDIS_HOST: Env.schema.string({ format: 'host' }), + REDIS_PORT: Env.schema.number(), }) diff --git a/admin/start/kernel.ts b/admin/start/kernel.ts index 740125a..56e1afa 100644 --- a/admin/start/kernel.ts +++ b/admin/start/kernel.ts @@ -27,7 +27,8 @@ server.use([ () => import('@adonisjs/cors/cors_middleware'), () => import('@adonisjs/vite/vite_middleware'), () => import('@adonisjs/inertia/inertia_middleware'), - () => import('@adonisjs/static/static_middleware') + () => import('@adonisjs/static/static_middleware'), + () => import('#middleware/maps_static_middleware') ]) /** diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 9881c13..e208e41 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -7,6 +7,7 @@ | */ import DocsController from '#controllers/docs_controller' +import DownloadsController from '#controllers/downloads_controller' import HomeController from '#controllers/home_controller' import MapsController from '#controllers/maps_controller' import SettingsController from '#controllers/settings_controller' @@ -64,6 +65,13 @@ router }) .prefix('/api/docs') +router + .group(() => { + router.get('/jobs', [DownloadsController, 'index']) + router.get('/jobs/:filetype', [DownloadsController, 'filetype']) + }) + .prefix('/api/downloads') + router .group(() => { router.get('/info', [SystemController, 'getSystemInfo']) @@ -77,7 +85,6 @@ router .group(() => { router.get('/list', [ZimController, 'list']) router.get('/list-remote', [ZimController, 'listRemote']) - router.get('/active-downloads', [ZimController, 'listActiveDownloads']) router.get('/curated-collections', [ZimController, 'listCuratedCollections']) router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections']) router.post('/download-remote', [ZimController, 'downloadRemote']) diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index 21f398d..52637a2 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -12,6 +12,7 @@ export type DoResumableDownloadParams = { allowedMimeTypes: string[] signal?: AbortSignal onProgress?: (progress: DoResumableDownloadProgress) => void + onComplete?: (url: string, path: string) => void | Promise forceNew?: boolean } @@ -29,15 +30,6 @@ export type DoResumableDownloadProgress = { url: string } -export type DoBackgroundDownloadParams = Omit< - DoResumableDownloadWithRetryParams, - 'onProgress' | 'onAttemptError' | 'signal' -> & { - channel: string - activeDownloads: Map - onComplete?: (url: string, path: string) => void | Promise -} - export type CuratedCollection = { name: string slug: string @@ -59,3 +51,18 @@ export type CuratedCollectionWithStatus = CuratedCollection & { export type CuratedCollectionsFile = { collections: CuratedCollection[] } + +export type RunDownloadJobParams = Omit< + DoResumableDownloadParams, + 'onProgress' | 'onComplete' | 'signal' +> & { + filetype: string +} + +export type DownloadJobWithProgress = { + jobId: string + url: string + progress: number + filepath: string + filetype: string +} diff --git a/admin/util/broadcast_channels.ts b/admin/util/broadcast_channels.ts index 18c0114..386cb39 100644 --- a/admin/util/broadcast_channels.ts +++ b/admin/util/broadcast_channels.ts @@ -1,5 +1,3 @@ - export const BROADCAST_CHANNELS = { - ZIM: 'zim-downloads', - MAP: 'map-downloads', -} \ No newline at end of file + DOWNLOADS: 'downloads', +} diff --git a/admin/util/docs.ts b/admin/util/docs.ts index 86f026f..8b3632a 100644 --- a/admin/util/docs.ts +++ b/admin/util/docs.ts @@ -1,6 +1,5 @@ -import { Readable } from 'stream'; -export const streamToString = async (stream: Readable): Promise => { +export const streamToString = async (stream: NodeJS.ReadableStream): Promise => { const chunks: Buffer[] = []; for await (const chunk of stream) { chunks.push(Buffer.from(chunk)); diff --git a/install/entrypoint.sh b/install/entrypoint.sh index bd3199b..a11c107 100644 --- a/install/entrypoint.sh +++ b/install/entrypoint.sh @@ -17,6 +17,10 @@ node ace migration:run --force echo "Seeding the database..." node ace db:seed +# Start background worker for queues +echo "Starting background worker for queues..." +node ace queue:work --queue=downloads & + # Start the AdonisJS application echo "Starting AdonisJS application..." exec node bin/server.js \ No newline at end of file diff --git a/install/management_compose.yaml b/install/management_compose.yaml index 634c5b6..a4ad5d2 100644 --- a/install/management_compose.yaml +++ b/install/management_compose.yaml @@ -15,7 +15,6 @@ services: - NODE_ENV=production - PORT=8080 - LOG_LEVEL=debug - - DRIVE_DISK=fs - APP_KEY=replaceme - HOST=0.0.0.0 - URL=replaceme @@ -29,6 +28,8 @@ services: depends_on: mysql: condition: service_healthy + redis: + condition: service_healthy entrypoint: ["/usr/local/bin/entrypoint.sh"] healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health"] @@ -63,4 +64,17 @@ services: test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] interval: 30s timeout: 10s + retries: 3 + redis: + image: redis:7-alpine + container_name: nomad_redis + restart: unless-stopped + ports: + - "6379:6379" + volumes: + - /opt/project-nomad/redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s retries: 3 \ No newline at end of file