mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-27 19:19:25 +01:00
feat: background job overhaul with bullmq
This commit is contained in:
Jake Turner
parent
2985929079
commit
7569aa935d
|
|
@ -4,7 +4,6 @@ LOG_LEVEL=info
|
|||
APP_KEY=some_random_key
|
||||
NODE_ENV=development
|
||||
SESSION_DRIVER=cookie
|
||||
DRIVE_DISK=fs
|
||||
DB_HOST=localhost
|
||||
DB_PORT=3306
|
||||
DB_USER=root
|
||||
|
|
|
|||
|
|
@ -52,8 +52,8 @@ export default defineConfig({
|
|||
() => import('@adonisjs/cors/cors_provider'),
|
||||
() => import('@adonisjs/lucid/database_provider'),
|
||||
() => import('@adonisjs/inertia/inertia_provider'),
|
||||
() => import('@adonisjs/drive/drive_provider'),
|
||||
() => import('@adonisjs/transmit/transmit_provider')
|
||||
() => import('@adonisjs/transmit/transmit_provider'),
|
||||
() => import('#providers/map_static_provider')
|
||||
],
|
||||
|
||||
/*
|
||||
|
|
|
|||
18
admin/app/controllers/downloads_controller.ts
Normal file
18
admin/app/controllers/downloads_controller.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import { DownloadService } from '#services/download_service'
|
||||
import { downloadJobsByFiletypeSchema } from '#validators/download'
|
||||
import { inject } from '@adonisjs/core'
|
||||
|
||||
@inject()
|
||||
export default class DownloadsController {
|
||||
constructor(private downloadService: DownloadService) {}
|
||||
|
||||
async index() {
|
||||
return this.downloadService.listDownloadJobs()
|
||||
}
|
||||
|
||||
async filetype({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(downloadJobsByFiletypeSchema)
|
||||
return this.downloadService.listDownloadJobs(payload.params.filetype)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { MapService } from '#services/map_service'
|
||||
import {
|
||||
filenameValidator,
|
||||
filenameParamValidator,
|
||||
remoteDownloadValidator,
|
||||
remoteDownloadValidatorOptional,
|
||||
} from '#validators/common'
|
||||
|
|
@ -53,14 +53,14 @@ export default class MapsController {
|
|||
}
|
||||
|
||||
async delete({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(filenameValidator)
|
||||
const payload = await request.validateUsing(filenameParamValidator)
|
||||
|
||||
try {
|
||||
await this.mapService.delete(payload.filename)
|
||||
await this.mapService.delete(payload.params.filename)
|
||||
} catch (error) {
|
||||
if (error.message === 'not_found') {
|
||||
return response.status(404).send({
|
||||
message: `Map file with key ${payload.filename} not found`,
|
||||
message: `Map file with key ${payload.params.filename} not found`,
|
||||
})
|
||||
}
|
||||
throw error // Re-throw any other errors and let the global error handler catch
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { ZimService } from '#services/zim_service'
|
||||
import {
|
||||
downloadCollectionValidator,
|
||||
filenameValidator,
|
||||
filenameParamValidator,
|
||||
remoteDownloadValidator,
|
||||
} from '#validators/common'
|
||||
import { listRemoteZimValidator } from '#validators/zim'
|
||||
|
|
@ -24,11 +24,12 @@ export default class ZimController {
|
|||
|
||||
async downloadRemote({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(remoteDownloadValidator)
|
||||
const filename = await this.zimService.downloadRemote(payload.url)
|
||||
const { filename, jobId } = await this.zimService.downloadRemote(payload.url)
|
||||
|
||||
return {
|
||||
message: 'Download started successfully',
|
||||
filename,
|
||||
jobId,
|
||||
url: payload.url,
|
||||
}
|
||||
}
|
||||
|
|
@ -44,10 +45,6 @@ export default class ZimController {
|
|||
}
|
||||
}
|
||||
|
||||
async listActiveDownloads({}: HttpContext) {
|
||||
return this.zimService.listActiveDownloads()
|
||||
}
|
||||
|
||||
async listCuratedCollections({}: HttpContext) {
|
||||
return this.zimService.listCuratedCollections()
|
||||
}
|
||||
|
|
@ -58,14 +55,14 @@ export default class ZimController {
|
|||
}
|
||||
|
||||
async delete({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(filenameValidator)
|
||||
const payload = await request.validateUsing(filenameParamValidator)
|
||||
|
||||
try {
|
||||
await this.zimService.delete(payload.filename)
|
||||
await this.zimService.delete(payload.params.filename)
|
||||
} catch (error) {
|
||||
if (error.message === 'not_found') {
|
||||
return response.status(404).send({
|
||||
message: `ZIM file with key ${payload.filename} not found`,
|
||||
message: `ZIM file with key ${payload.params.filename} not found`,
|
||||
})
|
||||
}
|
||||
throw error // Re-throw any other errors and let the global error handler catch
|
||||
|
|
|
|||
107
admin/app/jobs/run_download_job.ts
Normal file
107
admin/app/jobs/run_download_job.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { Job } from 'bullmq'
|
||||
import { RunDownloadJobParams } from '../../types/downloads.js'
|
||||
import { QueueService } from '#services/queue_service'
|
||||
import { doResumableDownload } from '../utils/downloads.js'
|
||||
import { createHash } from 'crypto'
|
||||
import { DockerService } from '#services/docker_service'
|
||||
import { ZimService } from '#services/zim_service'
|
||||
|
||||
export class RunDownloadJob {
|
||||
static get queue() {
|
||||
return 'downloads'
|
||||
}
|
||||
|
||||
static get key() {
|
||||
return 'run-download'
|
||||
}
|
||||
|
||||
static getJobId(url: string): string {
|
||||
return createHash('sha256').update(url).digest('hex').slice(0, 16)
|
||||
}
|
||||
|
||||
async handle(job: Job) {
|
||||
const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } =
|
||||
job.data as RunDownloadJobParams
|
||||
|
||||
// console.log("Simulating delay for job for URL:", url)
|
||||
// await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
|
||||
// console.log("Starting download for URL:", url)
|
||||
|
||||
// // simulate progress updates for demonstration
|
||||
// for (let progress = 0; progress <= 100; progress += 10) {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 20000)) // Simulate time taken for each progress step
|
||||
// job.updateProgress(progress)
|
||||
// console.log(`Job progress for URL ${url}: ${progress}%`)
|
||||
// }
|
||||
|
||||
await doResumableDownload({
|
||||
url,
|
||||
filepath,
|
||||
timeout,
|
||||
allowedMimeTypes,
|
||||
forceNew,
|
||||
onProgress(progress) {
|
||||
const progressPercent = (progress.downloadedBytes / (progress.totalBytes || 1)) * 100
|
||||
job.updateProgress(Math.floor(progressPercent))
|
||||
},
|
||||
async onComplete(url) {
|
||||
if (filetype === 'zim') {
|
||||
try {
|
||||
const dockerService = new DockerService()
|
||||
const zimService = new ZimService(dockerService)
|
||||
await zimService.downloadRemoteSuccessCallback([url], true)
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
job.updateProgress(100)
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
url,
|
||||
filepath,
|
||||
}
|
||||
}
|
||||
|
||||
static async getByUrl(url: string): Promise<Job | undefined> {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const jobId = this.getJobId(url)
|
||||
return await queue.getJob(jobId)
|
||||
}
|
||||
|
||||
static async dispatch(params: RunDownloadJobParams) {
|
||||
const queueService = new QueueService()
|
||||
const queue = queueService.getQueue(this.queue)
|
||||
const jobId = this.getJobId(params.url)
|
||||
|
||||
try {
|
||||
const job = await queue.add(this.key, params, {
|
||||
jobId,
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
removeOnComplete: true,
|
||||
})
|
||||
|
||||
return {
|
||||
job,
|
||||
created: true,
|
||||
message: `Dispatched download job for URL ${params.url}`,
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.message.includes('job already exists')) {
|
||||
const existing = await queue.getJob(jobId)
|
||||
return {
|
||||
job: existing,
|
||||
created: false,
|
||||
message: `Job already exists for URL ${params.url}`,
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
20
admin/app/middleware/maps_static_middleware.ts
Normal file
20
admin/app/middleware/maps_static_middleware.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import StaticMiddleware from '@adonisjs/static/static_middleware'
|
||||
import { AssetsConfig } from '@adonisjs/static/types'
|
||||
|
||||
/**
|
||||
* See #providers/map_static_provider.ts for explanation
|
||||
* of why this middleware exists.
|
||||
*/
|
||||
export default class MapsStaticMiddleware {
|
||||
constructor(
|
||||
private path: string,
|
||||
private config: AssetsConfig
|
||||
) {}
|
||||
|
||||
async handle(ctx: HttpContext, next: NextFn) {
|
||||
const staticMiddleware = new StaticMiddleware(this.path, this.config)
|
||||
return staticMiddleware.handle(ctx, next)
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ import logger from '@adonisjs/core/services/logger'
|
|||
import { inject } from '@adonisjs/core'
|
||||
import { ServiceStatus } from '../../types/services.js'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import { doSimpleDownload } from '../utils/downloads.js'
|
||||
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
|
||||
import path from 'path'
|
||||
|
||||
@inject()
|
||||
|
|
@ -347,10 +347,15 @@ export class DockerService {
|
|||
)
|
||||
|
||||
try {
|
||||
await doSimpleDownload({
|
||||
await doResumableDownloadWithRetry({
|
||||
url: WIKIPEDIA_ZIM_URL,
|
||||
filepath,
|
||||
timeout: 60000,
|
||||
allowedMimeTypes: [
|
||||
'application/x-zim',
|
||||
'application/x-openzim',
|
||||
'application/octet-stream',
|
||||
],
|
||||
})
|
||||
|
||||
this._broadcast(
|
||||
|
|
|
|||
|
|
@ -1,73 +1,65 @@
|
|||
import drive from '@adonisjs/drive/services/main';
|
||||
import Markdoc from '@markdoc/markdoc';
|
||||
import { streamToString } from '../../util/docs.js';
|
||||
import Markdoc from '@markdoc/markdoc'
|
||||
import { streamToString } from '../../util/docs.js'
|
||||
import { getFile, getFileStatsIfExists, listDirectoryContentsRecursive } from '../utils/fs.js'
|
||||
import path from 'path'
|
||||
|
||||
export class DocsService {
|
||||
private docsPath = path.join(process.cwd(), 'docs')
|
||||
async getDocs() {
|
||||
const disk = drive.use('docs');
|
||||
if (!disk) {
|
||||
throw new Error('Docs disk not configured');
|
||||
}
|
||||
const contents = await listDirectoryContentsRecursive(this.docsPath)
|
||||
const files: Array<{ title: string; slug: string }> = []
|
||||
|
||||
const contents = await disk.listAll('/');
|
||||
const files: Array<{ title: string; slug: string }> = [];
|
||||
|
||||
for (const item of contents.objects) {
|
||||
if (item.isFile && item.name.endsWith('.md')) {
|
||||
const cleaned = this.prettify(item.name);
|
||||
for (const item of contents) {
|
||||
if (item.type === 'file' && item.name.endsWith('.md')) {
|
||||
const cleaned = this.prettify(item.name)
|
||||
files.push({
|
||||
title: cleaned,
|
||||
slug: item.name.replace(/\.md$/, '')
|
||||
});
|
||||
slug: item.name.replace(/\.md$/, ''),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return files.sort((a, b) => a.title.localeCompare(b.title));
|
||||
return files.sort((a, b) => a.title.localeCompare(b.title))
|
||||
}
|
||||
|
||||
parse(content: string) {
|
||||
const ast = Markdoc.parse(content);
|
||||
const config = this.getConfig();
|
||||
const errors = Markdoc.validate(ast, config);
|
||||
const ast = Markdoc.parse(content)
|
||||
const config = this.getConfig()
|
||||
const errors = Markdoc.validate(ast, config)
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Markdoc validation errors: ${errors.map(e => e.error).join(', ')}`);
|
||||
throw new Error(`Markdoc validation errors: ${errors.map((e) => e.error).join(', ')}`)
|
||||
}
|
||||
|
||||
return Markdoc.transform(ast, config);
|
||||
return Markdoc.transform(ast, config)
|
||||
}
|
||||
|
||||
async parseFile(_filename: string) {
|
||||
const disk = drive.use('docs');
|
||||
if (!disk) {
|
||||
throw new Error('Docs disk not configured');
|
||||
}
|
||||
|
||||
if (!_filename) {
|
||||
throw new Error('Filename is required');
|
||||
throw new Error('Filename is required')
|
||||
}
|
||||
|
||||
const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`;
|
||||
const filename = _filename.endsWith('.md') ? _filename : `${_filename}.md`
|
||||
|
||||
const fileExists = await disk.exists(filename);
|
||||
const fileExists = await getFileStatsIfExists(path.join(this.docsPath, filename))
|
||||
if (!fileExists) {
|
||||
throw new Error(`File not found: ${filename}`);
|
||||
throw new Error(`File not found: ${filename}`)
|
||||
}
|
||||
|
||||
const fileStream = await disk.getStream(filename);
|
||||
const fileStream = await getFile(path.join(this.docsPath, filename), 'stream')
|
||||
if (!fileStream) {
|
||||
throw new Error(`Failed to read file stream: ${filename}`);
|
||||
throw new Error(`Failed to read file stream: ${filename}`)
|
||||
}
|
||||
const content = await streamToString(fileStream);
|
||||
return this.parse(content);
|
||||
const content = await streamToString(fileStream)
|
||||
return this.parse(content)
|
||||
}
|
||||
|
||||
private prettify(filename: string) {
|
||||
// Remove hyphens, underscores, and file extension
|
||||
const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ');
|
||||
const cleaned = filename.replace(/_/g, ' ').replace(/\.md$/, '').replace(/-/g, ' ')
|
||||
// Convert to Title Case
|
||||
const titleCased = cleaned.replace(/\b\w/g, char => char.toUpperCase());
|
||||
return titleCased.charAt(0).toUpperCase() + titleCased.slice(1);
|
||||
const titleCased = cleaned.replace(/\b\w/g, (char) => char.toUpperCase())
|
||||
return titleCased.charAt(0).toUpperCase() + titleCased.slice(1)
|
||||
}
|
||||
|
||||
private getConfig() {
|
||||
|
|
@ -79,12 +71,12 @@ export class DocsService {
|
|||
type: {
|
||||
type: String,
|
||||
default: 'info',
|
||||
matches: ['info', 'warning', 'error', 'success']
|
||||
matches: ['info', 'warning', 'error', 'success'],
|
||||
},
|
||||
title: {
|
||||
type: String
|
||||
}
|
||||
}
|
||||
type: String,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
nodes: {
|
||||
|
|
@ -92,10 +84,10 @@ export class DocsService {
|
|||
render: 'Heading',
|
||||
attributes: {
|
||||
level: { type: Number, required: true },
|
||||
id: { type: String }
|
||||
}
|
||||
}
|
||||
}
|
||||
id: { type: String },
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
25
admin/app/services/download_service.ts
Normal file
25
admin/app/services/download_service.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { inject } from '@adonisjs/core'
|
||||
import { QueueService } from './queue_service.js'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { DownloadJobWithProgress } from '../../types/downloads.js'
|
||||
import { normalize } from 'path'
|
||||
|
||||
@inject()
|
||||
export class DownloadService {
|
||||
constructor(private queueService: QueueService) {}
|
||||
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
const queue = this.queueService.getQueue(RunDownloadJob.queue)
|
||||
const jobs = await queue.getJobs(['waiting', 'active', 'delayed'])
|
||||
|
||||
return jobs
|
||||
.map((job) => ({
|
||||
jobId: job.id!.toString(),
|
||||
url: job.data.url,
|
||||
progress: parseInt(job.progress.toString(), 10),
|
||||
filepath: normalize(job.data.filepath),
|
||||
filetype: job.data.filetype,
|
||||
}))
|
||||
.filter((job) => !filetype || job.filetype === filetype)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { BaseStylesFile, MapLayer } from '../../types/maps.js'
|
||||
import { FileEntry } from '../../types/files.js'
|
||||
import { doBackgroundDownload, doResumableDownloadWithRetry } from '../utils/downloads.js'
|
||||
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
|
||||
import { extract } from 'tar'
|
||||
import env from '#start/env'
|
||||
import {
|
||||
|
|
@ -14,7 +14,8 @@ import {
|
|||
import { join } from 'path'
|
||||
import urlJoin from 'url-join'
|
||||
import axios from 'axios'
|
||||
import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
const BASE_ASSETS_MIME_TYPES = [
|
||||
'application/gzip',
|
||||
|
|
@ -32,7 +33,6 @@ export class MapService {
|
|||
private readonly basemapsAssetsDir = 'basemaps-assets'
|
||||
private readonly baseAssetsTarFile = 'base-assets.tar.gz'
|
||||
private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
|
||||
private activeDownloads = new Map<string, AbortController>()
|
||||
|
||||
async listRegions() {
|
||||
const files = (await this.listAllMapStorageItems()).filter(
|
||||
|
|
@ -80,13 +80,13 @@ export class MapService {
|
|||
return true
|
||||
}
|
||||
|
||||
async downloadRemote(url: string): Promise<string> {
|
||||
async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> {
|
||||
const parsed = new URL(url)
|
||||
if (!parsed.pathname.endsWith('.pmtiles')) {
|
||||
throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
|
||||
}
|
||||
|
||||
const existing = this.activeDownloads.get(url)
|
||||
const existing = await RunDownloadJob.getByUrl(url)
|
||||
if (existing) {
|
||||
throw new Error(`Download already in progress for URL ${url}`)
|
||||
}
|
||||
|
|
@ -98,18 +98,26 @@ export class MapService {
|
|||
|
||||
const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
|
||||
|
||||
// Don't await the download, run it in the background
|
||||
doBackgroundDownload({
|
||||
// Dispatch background job
|
||||
const result = await RunDownloadJob.dispatch({
|
||||
url,
|
||||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: PMTILES_MIME_TYPES,
|
||||
forceNew: true,
|
||||
channel: BROADCAST_CHANNELS.MAP,
|
||||
activeDownloads: this.activeDownloads,
|
||||
filetype: 'pmtiles',
|
||||
})
|
||||
|
||||
return filename
|
||||
if (!result.job) {
|
||||
throw new Error('Failed to dispatch download job')
|
||||
}
|
||||
|
||||
logger.info(`[MapService] Dispatched download job ${result.job.id} for URL ${url}`)
|
||||
|
||||
return {
|
||||
filename,
|
||||
jobId: result.job?.id,
|
||||
}
|
||||
}
|
||||
|
||||
async downloadRemotePreflight(
|
||||
|
|
|
|||
22
admin/app/services/queue_service.ts
Normal file
22
admin/app/services/queue_service.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { Queue } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
|
||||
export class QueueService {
|
||||
private queues: Map<string, Queue> = new Map()
|
||||
|
||||
getQueue(name: string): Queue {
|
||||
if (!this.queues.has(name)) {
|
||||
const queue = new Queue(name, {
|
||||
connection: queueConfig.connection,
|
||||
})
|
||||
this.queues.set(name, queue)
|
||||
}
|
||||
return this.queues.get(name)!
|
||||
}
|
||||
|
||||
async close() {
|
||||
for (const queue of this.queues.values()) {
|
||||
await queue.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -6,11 +6,9 @@ import {
|
|||
import axios from 'axios'
|
||||
import { XMLParser } from 'fast-xml-parser'
|
||||
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { DockerService } from './docker_service.js'
|
||||
import { inject } from '@adonisjs/core'
|
||||
import { doBackgroundDownload } from '../utils/downloads.js'
|
||||
import {
|
||||
deleteFileIfExists,
|
||||
ensureDirectoryExists,
|
||||
|
|
@ -23,7 +21,7 @@ import vine from '@vinejs/vine'
|
|||
import { curatedCollectionsFileSchema } from '#validators/curated_collections'
|
||||
import CuratedCollection from '#models/curated_collection'
|
||||
import CuratedCollectionResource from '#models/curated_collection_resource'
|
||||
import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
|
||||
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
|
||||
const COLLECTIONS_URL =
|
||||
|
|
@ -32,7 +30,6 @@ const COLLECTIONS_URL =
|
|||
@inject()
|
||||
export class ZimService {
|
||||
private zimStoragePath = '/storage/zim'
|
||||
private activeDownloads = new Map<string, AbortController>()
|
||||
|
||||
constructor(private dockerService: DockerService) {}
|
||||
|
||||
|
|
@ -140,15 +137,15 @@ export class ZimService {
|
|||
}
|
||||
}
|
||||
|
||||
async downloadRemote(url: string): Promise<string> {
|
||||
async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> {
|
||||
const parsed = new URL(url)
|
||||
if (!parsed.pathname.endsWith('.zim')) {
|
||||
throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`)
|
||||
}
|
||||
|
||||
const existing = this.activeDownloads.get(url)
|
||||
const existing = await RunDownloadJob.getByUrl(url)
|
||||
if (existing) {
|
||||
throw new Error(`Download already in progress for URL ${url}`)
|
||||
throw new Error('A download for this URL is already in progress')
|
||||
}
|
||||
|
||||
// Extract the filename from the URL
|
||||
|
|
@ -159,19 +156,26 @@ export class ZimService {
|
|||
|
||||
const filepath = join(process.cwd(), this.zimStoragePath, filename)
|
||||
|
||||
// Don't await the download, run it in the background
|
||||
doBackgroundDownload({
|
||||
// Dispatch a background download job
|
||||
const result = await RunDownloadJob.dispatch({
|
||||
url,
|
||||
filepath,
|
||||
channel: BROADCAST_CHANNELS.ZIM,
|
||||
activeDownloads: this.activeDownloads,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
onComplete: (url) => this._downloadRemoteSuccessCallback([url]),
|
||||
filetype: 'zim',
|
||||
})
|
||||
|
||||
return filename
|
||||
if (!result || !result.job) {
|
||||
throw new Error('Failed to dispatch download job')
|
||||
}
|
||||
|
||||
logger.info(`[ZimService] Dispatched background download job for ZIM file: ${filename}`)
|
||||
|
||||
return {
|
||||
filename,
|
||||
jobId: result.job.id,
|
||||
}
|
||||
}
|
||||
|
||||
async downloadCollection(slug: string): Promise<string[] | null> {
|
||||
|
|
@ -188,49 +192,43 @@ export class ZimService {
|
|||
const downloadUrls = resources.map((res) => res.url)
|
||||
const downloadFilenames: string[] = []
|
||||
|
||||
for (const [idx, url] of downloadUrls.entries()) {
|
||||
const existing = this.activeDownloads.get(url)
|
||||
for (const url of downloadUrls) {
|
||||
const existing = await RunDownloadJob.getByUrl(url)
|
||||
if (existing) {
|
||||
logger.warn(`Download already in progress for URL ${url}, skipping.`)
|
||||
logger.warn(`[ZimService] Download already in progress for URL ${url}, skipping.`)
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract the filename from the URL
|
||||
const filename = url.split('/').pop()
|
||||
if (!filename) {
|
||||
logger.warn(`Could not determine filename from URL ${url}, skipping.`)
|
||||
logger.warn(`[ZimService] Could not determine filename from URL ${url}, skipping.`)
|
||||
continue
|
||||
}
|
||||
|
||||
const filepath = join(process.cwd(), this.zimStoragePath, filename)
|
||||
downloadFilenames.push(filename)
|
||||
const filepath = join(process.cwd(), this.zimStoragePath, filename)
|
||||
|
||||
const isLastDownload = idx === downloadUrls.length - 1
|
||||
|
||||
// Don't await the download, run it in the background
|
||||
doBackgroundDownload({
|
||||
await RunDownloadJob.dispatch({
|
||||
url,
|
||||
filepath,
|
||||
channel: BROADCAST_CHANNELS.ZIM,
|
||||
activeDownloads: this.activeDownloads,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
onComplete: (url) =>
|
||||
this._downloadRemoteSuccessCallback([url], isLastDownload),
|
||||
filetype: 'zim',
|
||||
})
|
||||
}
|
||||
|
||||
return downloadFilenames.length > 0 ? downloadFilenames : null
|
||||
}
|
||||
|
||||
async _downloadRemoteSuccessCallback(urls: string[], restart = true) {
|
||||
async downloadRemoteSuccessCallback(urls: string[], restart = true) {
|
||||
// Restart KIWIX container to pick up new ZIM file
|
||||
if (restart) {
|
||||
await this.dockerService
|
||||
.affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
|
||||
.catch((error) => {
|
||||
logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
|
||||
logger.error(`[ZimService] Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -242,21 +240,6 @@ export class ZimService {
|
|||
}
|
||||
}
|
||||
|
||||
listActiveDownloads(): string[] {
|
||||
return Array.from(this.activeDownloads.keys())
|
||||
}
|
||||
|
||||
cancelDownload(url: string): boolean {
|
||||
const entry = this.activeDownloads.get(url)
|
||||
if (entry) {
|
||||
entry.abort()
|
||||
this.activeDownloads.delete(url)
|
||||
transmit.broadcast(BROADCAST_CHANNELS.ZIM, { url, status: 'cancelled' })
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
|
||||
const collections = await CuratedCollection.query().preload('resources')
|
||||
return collections.map((collection) => ({
|
||||
|
|
@ -282,17 +265,17 @@ export class ZimService {
|
|||
type: 'zim',
|
||||
}
|
||||
)
|
||||
logger.info(`Upserted curated collection: ${collection.slug}`)
|
||||
logger.info(`[ZimService] Upserted curated collection: ${collection.slug}`)
|
||||
|
||||
await collectionResult.related('resources').createMany(collection.resources)
|
||||
logger.info(
|
||||
`Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
|
||||
`[ZimService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
|
||||
)
|
||||
}
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
logger.error('Failed to download latest Kiwix collections:', error)
|
||||
logger.error(`[ZimService] Failed to download latest Kiwix collections:`, error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import {
|
||||
DoBackgroundDownloadParams,
|
||||
DoResumableDownloadParams,
|
||||
DoResumableDownloadProgress,
|
||||
DoResumableDownloadWithRetryParams,
|
||||
DoSimpleDownloadParams,
|
||||
} from '../../types/downloads.js'
|
||||
|
|
@ -9,9 +7,6 @@ import axios, { AxiosResponse } from 'axios'
|
|||
import { Transform } from 'stream'
|
||||
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
|
||||
import { createWriteStream } from 'fs'
|
||||
import { formatSpeed } from './misc.js'
|
||||
import { DownloadProgress } from '../../types/files.js'
|
||||
import transmit from '@adonisjs/transmit/services/main'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import path from 'path'
|
||||
|
||||
|
|
@ -86,6 +81,7 @@ export async function doResumableDownload({
|
|||
timeout = 30000,
|
||||
signal,
|
||||
onProgress,
|
||||
onComplete,
|
||||
forceNew = false,
|
||||
allowedMimeTypes,
|
||||
}: DoResumableDownloadParams): Promise<string> {
|
||||
|
|
@ -200,7 +196,7 @@ export async function doResumableDownload({
|
|||
cleanup(new Error('Download aborted'))
|
||||
})
|
||||
|
||||
writeStream.on('finish', () => {
|
||||
writeStream.on('finish', async () => {
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
downloadedBytes,
|
||||
|
|
@ -210,6 +206,9 @@ export async function doResumableDownload({
|
|||
url,
|
||||
})
|
||||
}
|
||||
if (onComplete) {
|
||||
await onComplete(url, filepath)
|
||||
}
|
||||
resolve(filepath)
|
||||
})
|
||||
|
||||
|
|
@ -276,82 +275,6 @@ export async function doResumableDownloadWithRetry({
|
|||
throw lastError || new Error('Unknown error during download')
|
||||
}
|
||||
|
||||
export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise<void> {
|
||||
const { url, filepath, channel, activeDownloads, onComplete, ...restParams } = params
|
||||
|
||||
try {
|
||||
const dirname = path.dirname(filepath)
|
||||
await ensureDirectoryExists(dirname)
|
||||
|
||||
const abortController = new AbortController()
|
||||
activeDownloads.set(url, abortController)
|
||||
|
||||
await doResumableDownloadWithRetry({
|
||||
url,
|
||||
filepath,
|
||||
signal: abortController.signal,
|
||||
...restParams,
|
||||
onProgress: (progressData) => {
|
||||
sendProgressBroadcast(channel, progressData)
|
||||
},
|
||||
})
|
||||
|
||||
sendCompletedBroadcast(channel, url, filepath)
|
||||
|
||||
if (onComplete) {
|
||||
await onComplete(url, filepath)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Background download failed for ${url}: ${error.message}`)
|
||||
sendErrorBroadcast(channel, url, error.message)
|
||||
} finally {
|
||||
activeDownloads.delete(url)
|
||||
}
|
||||
}
|
||||
|
||||
export function sendProgressBroadcast(
|
||||
channel: string,
|
||||
progressData: DoResumableDownloadProgress,
|
||||
status = 'in_progress'
|
||||
) {
|
||||
const { downloadedBytes, totalBytes, lastProgressTime, lastDownloadedBytes, url } = progressData
|
||||
const now = Date.now()
|
||||
const timeDiff = (now - lastProgressTime) / 1000
|
||||
const bytesDiff = downloadedBytes - lastDownloadedBytes
|
||||
const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0
|
||||
const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0
|
||||
const speed = formatSpeed(rawSpeed)
|
||||
|
||||
const progress: DownloadProgress = {
|
||||
downloaded_bytes: downloadedBytes,
|
||||
total_bytes: totalBytes,
|
||||
percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
|
||||
speed,
|
||||
time_remaining: timeRemaining,
|
||||
}
|
||||
|
||||
transmit.broadcast(channel, { url, progress, status })
|
||||
}
|
||||
|
||||
export function sendCompletedBroadcast(channel: string, url: string, path: string) {
|
||||
transmit.broadcast(channel, {
|
||||
url,
|
||||
path,
|
||||
status: 'completed',
|
||||
progress: {
|
||||
downloaded_bytes: 0,
|
||||
total_bytes: 0,
|
||||
percentage: 100,
|
||||
speed: '0 B/s',
|
||||
time_remaining: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function sendErrorBroadcast(channel: string, url: string, errorMessage: string) {
|
||||
transmit.broadcast(channel, { url, error: errorMessage, status: 'failed' })
|
||||
}
|
||||
|
||||
async function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { DriveDisks } from '@adonisjs/drive/types'
|
||||
import driveConfig from '#config/drive'
|
||||
import app from '@adonisjs/core/services/app'
|
||||
import { FileEntry } from '../../types/files.js'
|
||||
import { createReadStream } from 'fs'
|
||||
|
||||
export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
|
||||
const entries = await readdir(path, { withFileTypes: true })
|
||||
|
|
@ -56,14 +54,22 @@ export async function ensureDirectoryExists(path: string): Promise<void> {
|
|||
}
|
||||
|
||||
export async function getFile(path: string, returnType: 'buffer'): Promise<Buffer | null>
|
||||
export async function getFile(
|
||||
path: string,
|
||||
returnType: 'stream'
|
||||
): Promise<NodeJS.ReadableStream | null>
|
||||
export async function getFile(path: string, returnType: 'string'): Promise<string | null>
|
||||
export async function getFile(path: string, returnType: 'buffer' | 'string' = 'buffer'): Promise<Buffer | string | null> {
|
||||
export async function getFile(
|
||||
path: string,
|
||||
returnType: 'buffer' | 'string' | 'stream' = 'buffer'
|
||||
): Promise<Buffer | string | NodeJS.ReadableStream | null> {
|
||||
try {
|
||||
if (returnType === 'buffer') {
|
||||
return await readFile(path)
|
||||
} else {
|
||||
if (returnType === 'string') {
|
||||
return await readFile(path, 'utf-8')
|
||||
} else if (returnType === 'stream') {
|
||||
return createReadStream(path)
|
||||
}
|
||||
return await readFile(path)
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
return null
|
||||
|
|
@ -98,18 +104,3 @@ export async function deleteFileIfExists(path: string): Promise<void> {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function getFullDrivePath(diskName: keyof DriveDisks): Promise<string> {
|
||||
const config = await driveConfig.resolver(app)
|
||||
const serviceConfig = config.config.services[diskName]
|
||||
const resolved = serviceConfig()
|
||||
if (!resolved) {
|
||||
throw new Error(`Disk ${diskName} not configured`)
|
||||
}
|
||||
|
||||
let path = resolved.options.location
|
||||
if (path instanceof URL) {
|
||||
return path.pathname
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,11 @@ export const remoteDownloadValidatorOptional = vine.compile(
|
|||
})
|
||||
)
|
||||
|
||||
export const filenameValidator = vine.compile(
|
||||
export const filenameParamValidator = vine.compile(
|
||||
vine.object({
|
||||
filename: vine.string().trim().minLength(1).maxLength(4096),
|
||||
params: vine.object({
|
||||
filename: vine.string().trim().minLength(1).maxLength(4096),
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
||||
|
|
|
|||
9
admin/app/validators/download.ts
Normal file
9
admin/app/validators/download.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import vine from '@vinejs/vine'
|
||||
|
||||
export const downloadJobsByFiletypeSchema = vine.compile(
|
||||
vine.object({
|
||||
params: vine.object({
|
||||
filetype: vine.string(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
67
admin/commands/queue/work.ts
Normal file
67
admin/commands/queue/work.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { Worker } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
|
||||
export default class QueueWork extends BaseCommand {
|
||||
static commandName = 'queue:work'
|
||||
static description = 'Start processing jobs from the queue'
|
||||
|
||||
@flags.string({ description: 'Queue name to process', required: true })
|
||||
declare queue: string
|
||||
|
||||
static options: CommandOptions = {
|
||||
startApp: true,
|
||||
staysAlive: true,
|
||||
}
|
||||
|
||||
async run() {
|
||||
const queueName = this.queue || 'default'
|
||||
|
||||
const jobHandlers = await this.loadJobHandlers()
|
||||
|
||||
const worker = new Worker(
|
||||
queueName,
|
||||
async (job) => {
|
||||
this.logger.info(`Processing job: ${job.id} of type: ${job.name}`)
|
||||
const jobHandler = jobHandlers.get(job.name)
|
||||
if (!jobHandler) {
|
||||
throw new Error(`No handler found for job: ${job.name}`)
|
||||
}
|
||||
|
||||
return await jobHandler.handle(job)
|
||||
},
|
||||
{
|
||||
connection: queueConfig.connection,
|
||||
concurrency: 3,
|
||||
autorun: true,
|
||||
}
|
||||
)
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
this.logger.error(`Job failed: ${job?.id}, Error: ${err.message}`)
|
||||
})
|
||||
|
||||
worker.on('completed', (job) => {
|
||||
this.logger.info(`Job completed: ${job.id}`)
|
||||
})
|
||||
|
||||
this.logger.info(`Worker started for queue: ${queueName}`)
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
this.logger.info('SIGTERM received. Shutting down worker...')
|
||||
await worker.close()
|
||||
this.logger.info('Worker shut down gracefully.')
|
||||
process.exit(0)
|
||||
})
|
||||
}
|
||||
|
||||
private async loadJobHandlers() {
|
||||
const handlers = new Map<string, any>()
|
||||
|
||||
const { RunDownloadJob } = await import('#jobs/run_download_job')
|
||||
handlers.set(RunDownloadJob.key, new RunDownloadJob())
|
||||
|
||||
return handlers
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
import env from '#start/env'
|
||||
import app from '@adonisjs/core/services/app'
|
||||
import { defineConfig, services } from '@adonisjs/drive'
|
||||
|
||||
const driveConfig = defineConfig({
|
||||
default: env.get('DRIVE_DISK'),
|
||||
|
||||
/**
|
||||
* The services object can be used to configure multiple file system
|
||||
* services each using the same or a different driver.
|
||||
*/
|
||||
services: {
|
||||
fs: services.fs({
|
||||
location: app.makePath('storage'),
|
||||
serveFiles: true,
|
||||
routeBasePath: '/storage',
|
||||
visibility: 'public',
|
||||
}),
|
||||
docs: services.fs({
|
||||
location: app.makePath('docs'),
|
||||
serveFiles: false, // Don't serve files directly - we handle this via routes/Inertia
|
||||
visibility: 'public',
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
export default driveConfig
|
||||
|
||||
declare module '@adonisjs/drive/types' {
|
||||
export interface DriveDisks extends InferDriveDisks<typeof driveConfig> { }
|
||||
}
|
||||
10
admin/config/queue.ts
Normal file
10
admin/config/queue.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import env from '#start/env'
|
||||
|
||||
const queueConfig = {
|
||||
connection: {
|
||||
host: env.get('REDIS_HOST'),
|
||||
port: env.get('REDIS_PORT') ?? 6379,
|
||||
},
|
||||
}
|
||||
|
||||
export default queueConfig
|
||||
|
|
@ -12,6 +12,7 @@ const staticServerConfig = defineConfig({
|
|||
etag: true,
|
||||
lastModified: true,
|
||||
dotFiles: 'ignore',
|
||||
acceptRanges: true,
|
||||
})
|
||||
|
||||
export default staticServerConfig
|
||||
|
|
|
|||
|
|
@ -8,11 +8,23 @@ interface HorizontalBarChartProps {
|
|||
used: string
|
||||
type?: string
|
||||
}>
|
||||
maxValue?: number
|
||||
statuses?: Array<{
|
||||
label: string
|
||||
min_threshold: number
|
||||
color_class: string
|
||||
}>
|
||||
progressiveBarColor?: boolean
|
||||
}
|
||||
|
||||
export default function HorizontalBarChart({ items, maxValue = 100 }: HorizontalBarChartProps) {
|
||||
export default function HorizontalBarChart({
|
||||
items,
|
||||
statuses,
|
||||
progressiveBarColor = false,
|
||||
}: HorizontalBarChartProps) {
|
||||
const sortedStatus = statuses?.sort((a, b) => b.min_threshold - a.min_threshold) || []
|
||||
|
||||
const getBarColor = (value: number) => {
|
||||
if (!progressiveBarColor) return 'bg-desert-green'
|
||||
if (value >= 90) return 'bg-desert-red'
|
||||
if (value >= 75) return 'bg-desert-orange'
|
||||
if (value >= 50) return 'bg-desert-tan'
|
||||
|
|
@ -26,6 +38,26 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
|
|||
return 'shadow-desert-olive/50'
|
||||
}
|
||||
|
||||
const getStatusLabel = (value: number) => {
|
||||
if (sortedStatus.length === 0) return ''
|
||||
for (const status of sortedStatus) {
|
||||
if (value >= status.min_threshold) {
|
||||
return status.label
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
const getStatusColor = (value: number) => {
|
||||
if (sortedStatus.length === 0) return ''
|
||||
for (const status of sortedStatus) {
|
||||
if (value >= status.min_threshold) {
|
||||
return status.color_class
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{items.map((item, index) => (
|
||||
|
|
@ -56,28 +88,7 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
|
|||
width: `${item.value}%`,
|
||||
animationDelay: `${index * 100}ms`,
|
||||
}}
|
||||
>
|
||||
{/* Animated shine effect */}
|
||||
{/* <div
|
||||
className="absolute inset-0 bg-gradient-to-r from-transparent via-white to-transparent opacity-20 animate-shimmer"
|
||||
style={{
|
||||
animation: 'shimmer 3s infinite',
|
||||
animationDelay: `${index * 0.5}s`,
|
||||
}}
|
||||
/> */}
|
||||
{/* <div
|
||||
className="absolute inset-0 opacity-10"
|
||||
style={{
|
||||
backgroundImage: `repeating-linear-gradient(
|
||||
90deg,
|
||||
transparent,
|
||||
transparent 10px,
|
||||
rgba(255, 255, 255, 0.1) 10px,
|
||||
rgba(255, 255, 255, 0.1) 11px
|
||||
)`,
|
||||
}}
|
||||
/> */}
|
||||
</div>
|
||||
></div>
|
||||
</div>
|
||||
<div
|
||||
className={classNames(
|
||||
|
|
@ -90,25 +101,17 @@ export default function HorizontalBarChart({ items, maxValue = 100 }: Horizontal
|
|||
{Math.round(item.value)}%
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className={classNames(
|
||||
'w-2 h-2 rounded-full animate-pulse',
|
||||
item.value >= 90
|
||||
? 'bg-desert-red'
|
||||
: item.value >= 75
|
||||
? 'bg-desert-orange'
|
||||
: 'bg-desert-olive'
|
||||
)}
|
||||
/>
|
||||
<span className="text-xs text-desert-stone">
|
||||
{item.value >= 90
|
||||
? 'Critical - Disk Almost Full'
|
||||
: item.value >= 75
|
||||
? 'Warning - Usage High'
|
||||
: 'Normal'}
|
||||
</span>
|
||||
</div>
|
||||
{getStatusLabel(item.value) && (
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className={classNames(
|
||||
'w-2 h-2 rounded-full animate-pulse',
|
||||
getStatusColor(item.value)
|
||||
)}
|
||||
/>
|
||||
<span className="text-xs text-desert-stone">{getStatusLabel(item.value)}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
31
admin/inertia/hooks/useDownloads.ts
Normal file
31
admin/inertia/hooks/useDownloads.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useMemo } from 'react'
|
||||
import api from '~/lib/api'
|
||||
|
||||
export type useDownloadsProps = {
|
||||
filetype?: string
|
||||
enabled?: boolean
|
||||
}
|
||||
|
||||
const useDownloads = (props: useDownloadsProps) => {
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const queryKey = useMemo(() => {
|
||||
return props.filetype ? ['download-jobs', props.filetype] : ['download-jobs']
|
||||
}, [props.filetype])
|
||||
|
||||
const queryData = useQuery({
|
||||
queryKey: queryKey,
|
||||
queryFn: () => api.listDownloadJobs(props.filetype),
|
||||
refetchInterval: 2000, // Refetch every 2 seconds to get updated progress
|
||||
enabled: props.enabled ?? true,
|
||||
})
|
||||
|
||||
const invalidate = () => {
|
||||
queryClient.invalidateQueries({ queryKey: queryKey })
|
||||
}
|
||||
|
||||
return { ...queryData, invalidate }
|
||||
}
|
||||
|
||||
export default useDownloads
|
||||
|
|
@ -3,7 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi
|
|||
import { ServiceSlim } from '../../types/services'
|
||||
import { FileEntry } from '../../types/files'
|
||||
import { SystemInformationResponse } from '../../types/system'
|
||||
import { CuratedCollectionWithStatus } from '../../types/downloads'
|
||||
import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads'
|
||||
|
||||
class API {
|
||||
private client: AxiosInstance
|
||||
|
|
@ -142,16 +142,6 @@ class API {
|
|||
}
|
||||
}
|
||||
|
||||
async listActiveZimDownloads(): Promise<string[]> {
|
||||
try {
|
||||
const response = await this.client.get<string[]>('/zim/active-downloads')
|
||||
return response.data
|
||||
} catch (error) {
|
||||
console.error('Error listing active ZIM downloads:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async downloadRemoteZimFile(url: string): Promise<{
|
||||
message: string
|
||||
filename: string
|
||||
|
|
@ -209,6 +199,17 @@ class API {
|
|||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async listDownloadJobs(filetype?: string): Promise<DownloadJobWithProgress[]> {
|
||||
try {
|
||||
const endpoint = filetype ? `/downloads/jobs/${filetype}` : '/downloads/jobs'
|
||||
const response = await this.client.get<DownloadJobWithProgress[]>(endpoint)
|
||||
return response.data
|
||||
} catch (error) {
|
||||
console.error('Error listing download jobs:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new API()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import SettingsLayout from '~/layouts/SettingsLayout'
|
|||
import { SystemInformationResponse } from '../../../types/system'
|
||||
import { formatBytes } from '~/lib/util'
|
||||
import CircularGauge from '~/components/systeminfo/CircularGauge'
|
||||
import HorizontalBarChart from '~/components/systeminfo/HorizontalBarChart'
|
||||
import HorizontalBarChart from '~/components/HorizontalBarChart'
|
||||
import InfoCard from '~/components/systeminfo/InfoCard'
|
||||
import {
|
||||
CpuChipIcon,
|
||||
|
|
@ -195,7 +195,27 @@ export default function SettingsPage(props: {
|
|||
|
||||
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow">
|
||||
{diskData && diskData.length > 0 ? (
|
||||
<HorizontalBarChart items={diskData} />
|
||||
<HorizontalBarChart
|
||||
items={diskData}
|
||||
progressiveBarColor={true}
|
||||
statuses={[
|
||||
{
|
||||
label: 'Normal',
|
||||
min_threshold: 0,
|
||||
color_class: 'bg-desert-olive',
|
||||
},
|
||||
{
|
||||
label: 'Warning - Usage High',
|
||||
min_threshold: 75,
|
||||
color_class: 'bg-desert-orange',
|
||||
},
|
||||
{
|
||||
label: 'Critical - Disk Almost Full',
|
||||
min_threshold: 90,
|
||||
color_class: 'bg-desert-red',
|
||||
},
|
||||
]}
|
||||
/>
|
||||
) : (
|
||||
<div className="text-center text-desert-stone-dark py-8">
|
||||
No storage devices detected
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@ import { formatBytes } from '~/lib/util'
|
|||
import StyledButton from '~/components/StyledButton'
|
||||
import { useModals } from '~/context/ModalContext'
|
||||
import StyledModal from '~/components/StyledModal'
|
||||
import { useTransmit } from 'react-adonis-transmit'
|
||||
import ProgressBar from '~/components/ProgressBar'
|
||||
import { useNotifications } from '~/context/NotificationContext'
|
||||
import useInternetStatus from '~/hooks/useInternetStatus'
|
||||
import Alert from '~/components/Alert'
|
||||
|
|
@ -28,7 +26,8 @@ import useDebounce from '~/hooks/useDebounce'
|
|||
import CuratedCollectionCard from '~/components/CuratedCollectionCard'
|
||||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
import { CuratedCollectionWithStatus } from '../../../../types/downloads'
|
||||
import { BROADCAST_CHANNELS } from '../../../../util/broadcast_channels'
|
||||
import useDownloads from '~/hooks/useDownloads'
|
||||
import HorizontalBarChart from '~/components/HorizontalBarChart'
|
||||
|
||||
const CURATED_COLLECTIONS_KEY = 'curated-zim-collections'
|
||||
|
||||
|
|
@ -36,7 +35,6 @@ export default function ZimRemoteExplorer() {
|
|||
const queryClient = useQueryClient()
|
||||
const tableParentRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
const { subscribe } = useTransmit()
|
||||
const { openModal, closeAllModals } = useModals()
|
||||
const { addNotification } = useNotifications()
|
||||
const { isOnline } = useInternetStatus()
|
||||
|
|
@ -45,44 +43,22 @@ export default function ZimRemoteExplorer() {
|
|||
|
||||
const [query, setQuery] = useState('')
|
||||
const [queryUI, setQueryUI] = useState('')
|
||||
const [activeDownloads, setActiveDownloads] = useState<
|
||||
Map<string, { status: string; progress: number; speed: string }>
|
||||
>(new Map<string, { status: string; progress: number; speed: string }>())
|
||||
|
||||
const debouncedSetQuery = debounce((val: string) => {
|
||||
setQuery(val)
|
||||
}, 400)
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = subscribe(BROADCAST_CHANNELS.ZIM, (data: any) => {
|
||||
if (data.url && data.progress?.percentage) {
|
||||
setActiveDownloads((prev) =>
|
||||
new Map(prev).set(data.url, {
|
||||
status: data.status,
|
||||
progress: data.progress.percentage || 0,
|
||||
speed: data.progress.speed || '0 KB/s',
|
||||
})
|
||||
)
|
||||
if (data.status === 'completed') {
|
||||
addNotification({
|
||||
message: `The download for ${data.url} has completed successfully.`,
|
||||
type: 'success',
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return () => {
|
||||
unsubscribe()
|
||||
}
|
||||
}, [])
|
||||
|
||||
const { data: curatedCollections } = useQuery({
|
||||
queryKey: [CURATED_COLLECTIONS_KEY],
|
||||
queryFn: () => api.listCuratedZimCollections(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
const { data: downloads, invalidate: invalidateDownloads } = useDownloads({
|
||||
filetype: 'zim',
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const { data, fetchNextPage, isFetching, isLoading } =
|
||||
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
||||
queryKey: ['remote-zim-files', query],
|
||||
|
|
@ -103,7 +79,17 @@ export default function ZimRemoteExplorer() {
|
|||
placeholderData: keepPreviousData,
|
||||
})
|
||||
|
||||
const flatData = useMemo(() => data?.pages.flatMap((page) => page.items) || [], [data])
|
||||
const flatData = useMemo(() => {
|
||||
const mapped = data?.pages.flatMap((page) => page.items) || []
|
||||
// remove items that are currently downloading
|
||||
return mapped.filter((item) => {
|
||||
const isDownloading = downloads?.some((download) => {
|
||||
const filename = item.download_url.split('/').pop()
|
||||
return filename && download.filepath.endsWith(filename)
|
||||
})
|
||||
return !isDownloading
|
||||
})
|
||||
}, [data, downloads])
|
||||
const hasMore = useMemo(() => data?.pages[data.pages.length - 1]?.has_more || false, [data])
|
||||
|
||||
const fetchOnBottomReached = useCallback(
|
||||
|
|
@ -176,6 +162,7 @@ export default function ZimRemoteExplorer() {
|
|||
async function downloadFile(record: RemoteZimFileEntry) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(record.download_url)
|
||||
invalidateDownloads()
|
||||
} catch (error) {
|
||||
console.error('Error downloading file:', error)
|
||||
}
|
||||
|
|
@ -184,19 +171,12 @@ export default function ZimRemoteExplorer() {
|
|||
async function downloadCollection(record: CuratedCollectionWithStatus) {
|
||||
try {
|
||||
await api.downloadZimCollection(record.slug)
|
||||
invalidateDownloads()
|
||||
} catch (error) {
|
||||
console.error('Error downloading collection:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const EntryProgressBar = useCallback(
|
||||
({ url }: { url: string }) => {
|
||||
const entry = activeDownloads.get(url)
|
||||
return <ProgressBar progress={entry?.progress || 0} speed={entry?.speed} />
|
||||
},
|
||||
[activeDownloads]
|
||||
)
|
||||
|
||||
const fetchLatestCollections = useMutation({
|
||||
mutationFn: () => api.fetchLatestZimCollections(),
|
||||
onSuccess: () => {
|
||||
|
|
@ -208,6 +188,17 @@ export default function ZimRemoteExplorer() {
|
|||
},
|
||||
})
|
||||
|
||||
const extractFileName = (path: string) => {
|
||||
if (!path) return ''
|
||||
if (path.includes('/')) {
|
||||
return path.substring(path.lastIndexOf('/') + 1)
|
||||
}
|
||||
if (path.includes('\\')) {
|
||||
return path.substring(path.lastIndexOf('\\') + 1)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingsLayout>
|
||||
<Head title="ZIM Remote Explorer | Project N.O.M.A.D." />
|
||||
|
|
@ -307,20 +298,16 @@ export default function ZimRemoteExplorer() {
|
|||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
const isDownloading = activeDownloads.has(record.download_url)
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
{!isDownloading && (
|
||||
<StyledButton
|
||||
icon={'ArrowDownTrayIcon'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
)}
|
||||
{isDownloading && <EntryProgressBar url={record.download_url} />}
|
||||
<StyledButton
|
||||
icon={'ArrowDownTrayIcon'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
|
|
@ -337,6 +324,28 @@ export default function ZimRemoteExplorer() {
|
|||
compact
|
||||
rowLines
|
||||
/>
|
||||
<StyledSectionHeader title="Active Downloads" className="mt-12 mb-4" />
|
||||
<div className="space-y-4">
|
||||
{downloads && downloads.length > 0 ? (
|
||||
downloads.map((download) => (
|
||||
<div className="bg-desert-white rounded-lg p-4 border border-desert-stone-light shadow-sm hover:shadow-lg transition-shadow">
|
||||
<HorizontalBarChart
|
||||
items={[
|
||||
{
|
||||
label: extractFileName(download.filepath) || download.url,
|
||||
value: download.progress,
|
||||
total: '100%',
|
||||
used: `${download.progress}%`,
|
||||
type: download.filetype,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-gray-500">No active downloads</p>
|
||||
)}
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
</SettingsLayout>
|
||||
|
|
|
|||
397
admin/package-lock.json
generated
397
admin/package-lock.json
generated
|
|
@ -12,7 +12,6 @@
|
|||
"@adonisjs/auth": "^9.4.0",
|
||||
"@adonisjs/core": "^6.18.0",
|
||||
"@adonisjs/cors": "^2.2.1",
|
||||
"@adonisjs/drive": "^3.4.1",
|
||||
"@adonisjs/inertia": "^3.1.1",
|
||||
"@adonisjs/lucid": "^21.6.1",
|
||||
"@adonisjs/session": "^7.5.1",
|
||||
|
|
@ -36,6 +35,7 @@
|
|||
"autoprefixer": "^10.4.21",
|
||||
"axios": "^1.13.1",
|
||||
"better-sqlite3": "^12.1.1",
|
||||
"bullmq": "^5.65.1",
|
||||
"dockerode": "^4.0.7",
|
||||
"edge.js": "^6.2.1",
|
||||
"fast-xml-parser": "^5.2.5",
|
||||
|
|
@ -339,35 +339,6 @@
|
|||
"@adonisjs/core": "^6.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@adonisjs/drive": {
|
||||
"version": "3.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@adonisjs/drive/-/drive-3.4.1.tgz",
|
||||
"integrity": "sha512-oDYY4wJ7wDMlO4E+dZPYBu+T3Av7Mj+JL8+J33qgyxtiJylnZgoZDuRfFjZZix/bFNNuWX2sLwTMnyiDcK+YsA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"flydrive": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@adonisjs/core": "^6.2.0",
|
||||
"@aws-sdk/client-s3": "^3.577.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.577.0",
|
||||
"@google-cloud/storage": "^7.10.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@aws-sdk/client-s3": {
|
||||
"optional": true
|
||||
},
|
||||
"@aws-sdk/s3-request-presigner": {
|
||||
"optional": true
|
||||
},
|
||||
"@google-cloud/storage": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@adonisjs/encryption": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@adonisjs/encryption/-/encryption-6.0.2.tgz",
|
||||
|
|
@ -2083,6 +2054,7 @@
|
|||
"version": "0.4.3",
|
||||
"resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz",
|
||||
"integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18.18"
|
||||
|
|
@ -2116,6 +2088,12 @@
|
|||
"react": "^16.9.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@ioredis/commands": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",
|
||||
"integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@isaacs/fs-minipass": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
|
||||
|
|
@ -2461,6 +2439,84 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
|
||||
"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@noble/hashes": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz",
|
||||
|
|
@ -5098,6 +5154,21 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/bullmq": {
|
||||
"version": "5.65.1",
|
||||
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.65.1.tgz",
|
||||
"integrity": "sha512-QgDAzX1G9L5IRy4Orva5CfQTXZT+5K+OfO/kbPrAqN+pmL9LJekCzxijXehlm/u2eXfWPfWvIdJJIqiuz3WJSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cron-parser": "^4.9.0",
|
||||
"ioredis": "^5.8.2",
|
||||
"msgpackr": "^1.11.2",
|
||||
"node-abort-controller": "^3.1.1",
|
||||
"semver": "^7.5.4",
|
||||
"tslib": "^2.0.0",
|
||||
"uuid": "^11.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
|
|
@ -5522,6 +5593,15 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/code-block-writer": {
|
||||
"version": "13.0.3",
|
||||
"resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-13.0.3.tgz",
|
||||
|
|
@ -5710,6 +5790,18 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cron-parser": {
|
||||
"version": "4.9.0",
|
||||
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
|
||||
"integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"luxon": "^3.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
|
||||
|
|
@ -5768,9 +5860,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
|
||||
"integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
|
|
@ -7001,58 +7093,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/flydrive": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/flydrive/-/flydrive-1.2.0.tgz",
|
||||
"integrity": "sha512-l9ix5MhBE8bVwxyHdFku6z5KhGOCOXQDI9xGNIlACSz9UrDFQxAB1I6W0qffZiOBBDambiJZlEYBCxlvF4U7fw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@humanwhocodes/retry": "^0.4.2",
|
||||
"@poppinss/utils": "^6.9.2",
|
||||
"etag": "^1.8.1",
|
||||
"mime-types": "^2.1.35"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.6.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@aws-sdk/client-s3": "^3.577.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.577.0",
|
||||
"@google-cloud/storage": "^7.10.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@aws-sdk/client-s3": {
|
||||
"optional": true
|
||||
},
|
||||
"@aws-sdk/s3-request-presigner": {
|
||||
"optional": true
|
||||
},
|
||||
"@google-cloud/storage": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/flydrive/node_modules/mime-db": {
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/flydrive/node_modules/mime-types": {
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.9",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
|
||||
|
|
@ -7751,6 +7791,30 @@
|
|||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/ioredis": {
|
||||
"version": "5.8.2",
|
||||
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz",
|
||||
"integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@ioredis/commands": "1.4.0",
|
||||
"cluster-key-slot": "^1.1.0",
|
||||
"debug": "^4.3.4",
|
||||
"denque": "^2.1.0",
|
||||
"lodash.defaults": "^4.2.0",
|
||||
"lodash.isarguments": "^3.1.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0",
|
||||
"standard-as-callback": "^2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.22.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/ioredis"
|
||||
}
|
||||
},
|
||||
"node_modules/ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
|
|
@ -8469,6 +8533,18 @@
|
|||
"integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.defaults": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
|
||||
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isarguments": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
|
||||
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
|
|
@ -8900,6 +8976,37 @@
|
|||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/msgpackr": {
|
||||
"version": "1.11.5",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
|
||||
"integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"msgpackr-extract": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/msgpackr-extract": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
|
||||
"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"node-gyp-build-optional-packages": "5.2.2"
|
||||
},
|
||||
"bin": {
|
||||
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/murmurhash-js": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz",
|
||||
|
|
@ -9015,6 +9122,27 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/node-abort-controller": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
|
||||
"integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-gyp-build-optional-packages": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
|
||||
"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"node-gyp-build-optional-packages": "bin.js",
|
||||
"node-gyp-build-optional-packages-optional": "optional.js",
|
||||
"node-gyp-build-optional-packages-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.19",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
|
||||
|
|
@ -9921,16 +10049,61 @@
|
|||
}
|
||||
},
|
||||
"node_modules/raw-body": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz",
|
||||
"integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==",
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
|
||||
"integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bytes": "3.1.2",
|
||||
"http-errors": "2.0.0",
|
||||
"iconv-lite": "0.6.3",
|
||||
"unpipe": "1.0.0"
|
||||
"bytes": "~3.1.2",
|
||||
"http-errors": "~2.0.1",
|
||||
"iconv-lite": "~0.7.0",
|
||||
"unpipe": "~1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/raw-body/node_modules/http-errors": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
|
||||
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"depd": "~2.0.0",
|
||||
"inherits": "~2.0.4",
|
||||
"setprototypeof": "~1.2.0",
|
||||
"statuses": "~2.0.2",
|
||||
"toidentifier": "~1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/raw-body/node_modules/iconv-lite": {
|
||||
"version": "0.7.0",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz",
|
||||
"integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/raw-body/node_modules/statuses": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
|
||||
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
|
|
@ -10182,6 +10355,27 @@
|
|||
"node": ">= 10.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-errors": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-parser": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"redis-errors": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/reflect-metadata": {
|
||||
"version": "0.2.2",
|
||||
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz",
|
||||
|
|
@ -10963,6 +11157,12 @@
|
|||
"get-source": "^2.0.12"
|
||||
}
|
||||
},
|
||||
"node_modules/standard-as-callback": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
|
||||
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/statuses": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
|
||||
|
|
@ -11777,6 +11977,19 @@
|
|||
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "11.1.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
|
||||
"integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/esm/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/v8-compile-cache-lib": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@
|
|||
"#database/*": "./database/*.js",
|
||||
"#tests/*": "./tests/*.js",
|
||||
"#start/*": "./start/*.js",
|
||||
"#config/*": "./config/*.js"
|
||||
"#config/*": "./config/*.js",
|
||||
"#jobs/*": "./app/jobs/*.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@adonisjs/assembler": "^7.8.2",
|
||||
|
|
@ -58,7 +59,6 @@
|
|||
"@adonisjs/auth": "^9.4.0",
|
||||
"@adonisjs/core": "^6.18.0",
|
||||
"@adonisjs/cors": "^2.2.1",
|
||||
"@adonisjs/drive": "^3.4.1",
|
||||
"@adonisjs/inertia": "^3.1.1",
|
||||
"@adonisjs/lucid": "^21.6.1",
|
||||
"@adonisjs/session": "^7.5.1",
|
||||
|
|
@ -82,6 +82,7 @@
|
|||
"autoprefixer": "^10.4.21",
|
||||
"axios": "^1.13.1",
|
||||
"better-sqlite3": "^12.1.1",
|
||||
"bullmq": "^5.65.1",
|
||||
"dockerode": "^4.0.7",
|
||||
"edge.js": "^6.2.1",
|
||||
"fast-xml-parser": "^5.2.5",
|
||||
|
|
|
|||
26
admin/providers/map_static_provider.ts
Normal file
26
admin/providers/map_static_provider.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import MapsStaticMiddleware from '#middleware/maps_static_middleware'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { ApplicationService } from '@adonisjs/core/types'
|
||||
import { defineConfig } from '@adonisjs/static'
|
||||
import { join } from 'path'
|
||||
|
||||
/**
|
||||
* This is a bit of a hack to serve static files from the
|
||||
* /storage/maps directory using AdonisJS static middleware because
|
||||
* the middleware does not allow us to define a custom path we want
|
||||
* to serve (it always serves from public/ by default).
|
||||
*
|
||||
* We use the same other config options, just change the path
|
||||
* (though we could also separate config if needed).
|
||||
*/
|
||||
export default class MapStaticProvider {
|
||||
constructor(protected app: ApplicationService) {}
|
||||
register() {
|
||||
this.app.container.singleton(MapsStaticMiddleware, () => {
|
||||
const path = join(process.cwd(), '/storage/maps')
|
||||
logger.debug(`Maps static files will be served from ${path}`)
|
||||
const config = this.app.config.get<any>('static', defineConfig({}))
|
||||
return new MapsStaticMiddleware(path, config)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -26,13 +26,6 @@ export default await Env.create(new URL('../', import.meta.url), {
|
|||
*/
|
||||
//SESSION_DRIVER: Env.schema.enum(['cookie', 'memory'] as const),
|
||||
|
||||
/*
|
||||
|----------------------------------------------------------
|
||||
| Variables for configuring the drive package
|
||||
|----------------------------------------------------------
|
||||
*/
|
||||
DRIVE_DISK: Env.schema.enum(['fs'] as const),
|
||||
|
||||
|
||||
/*
|
||||
|----------------------------------------------------------
|
||||
|
|
@ -45,4 +38,12 @@ export default await Env.create(new URL('../', import.meta.url), {
|
|||
DB_PASSWORD: Env.schema.string.optional(),
|
||||
DB_DATABASE: Env.schema.string(),
|
||||
DB_SSL: Env.schema.boolean.optional(),
|
||||
|
||||
/*
|
||||
|----------------------------------------------------------
|
||||
| Variables for configuring the Redis connection
|
||||
|----------------------------------------------------------
|
||||
*/
|
||||
REDIS_HOST: Env.schema.string({ format: 'host' }),
|
||||
REDIS_PORT: Env.schema.number(),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ server.use([
|
|||
() => import('@adonisjs/cors/cors_middleware'),
|
||||
() => import('@adonisjs/vite/vite_middleware'),
|
||||
() => import('@adonisjs/inertia/inertia_middleware'),
|
||||
() => import('@adonisjs/static/static_middleware')
|
||||
() => import('@adonisjs/static/static_middleware'),
|
||||
() => import('#middleware/maps_static_middleware')
|
||||
])
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
|
||||
*/
|
||||
import DocsController from '#controllers/docs_controller'
|
||||
import DownloadsController from '#controllers/downloads_controller'
|
||||
import HomeController from '#controllers/home_controller'
|
||||
import MapsController from '#controllers/maps_controller'
|
||||
import SettingsController from '#controllers/settings_controller'
|
||||
|
|
@ -64,6 +65,13 @@ router
|
|||
})
|
||||
.prefix('/api/docs')
|
||||
|
||||
router
|
||||
.group(() => {
|
||||
router.get('/jobs', [DownloadsController, 'index'])
|
||||
router.get('/jobs/:filetype', [DownloadsController, 'filetype'])
|
||||
})
|
||||
.prefix('/api/downloads')
|
||||
|
||||
router
|
||||
.group(() => {
|
||||
router.get('/info', [SystemController, 'getSystemInfo'])
|
||||
|
|
@ -77,7 +85,6 @@ router
|
|||
.group(() => {
|
||||
router.get('/list', [ZimController, 'list'])
|
||||
router.get('/list-remote', [ZimController, 'listRemote'])
|
||||
router.get('/active-downloads', [ZimController, 'listActiveDownloads'])
|
||||
router.get('/curated-collections', [ZimController, 'listCuratedCollections'])
|
||||
router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections'])
|
||||
router.post('/download-remote', [ZimController, 'downloadRemote'])
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export type DoResumableDownloadParams = {
|
|||
allowedMimeTypes: string[]
|
||||
signal?: AbortSignal
|
||||
onProgress?: (progress: DoResumableDownloadProgress) => void
|
||||
onComplete?: (url: string, path: string) => void | Promise<void>
|
||||
forceNew?: boolean
|
||||
}
|
||||
|
||||
|
|
@ -29,15 +30,6 @@ export type DoResumableDownloadProgress = {
|
|||
url: string
|
||||
}
|
||||
|
||||
export type DoBackgroundDownloadParams = Omit<
|
||||
DoResumableDownloadWithRetryParams,
|
||||
'onProgress' | 'onAttemptError' | 'signal'
|
||||
> & {
|
||||
channel: string
|
||||
activeDownloads: Map<string, AbortController>
|
||||
onComplete?: (url: string, path: string) => void | Promise<void>
|
||||
}
|
||||
|
||||
export type CuratedCollection = {
|
||||
name: string
|
||||
slug: string
|
||||
|
|
@ -59,3 +51,18 @@ export type CuratedCollectionWithStatus = CuratedCollection & {
|
|||
export type CuratedCollectionsFile = {
|
||||
collections: CuratedCollection[]
|
||||
}
|
||||
|
||||
export type RunDownloadJobParams = Omit<
|
||||
DoResumableDownloadParams,
|
||||
'onProgress' | 'onComplete' | 'signal'
|
||||
> & {
|
||||
filetype: string
|
||||
}
|
||||
|
||||
export type DownloadJobWithProgress = {
|
||||
jobId: string
|
||||
url: string
|
||||
progress: number
|
||||
filepath: string
|
||||
filetype: string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
|
||||
export const BROADCAST_CHANNELS = {
|
||||
ZIM: 'zim-downloads',
|
||||
MAP: 'map-downloads',
|
||||
}
|
||||
DOWNLOADS: 'downloads',
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { Readable } from 'stream';
|
||||
|
||||
export const streamToString = async (stream: Readable): Promise<string> => {
|
||||
export const streamToString = async (stream: NodeJS.ReadableStream): Promise<string> => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ node ace migration:run --force
|
|||
echo "Seeding the database..."
|
||||
node ace db:seed
|
||||
|
||||
# Start background worker for queues
|
||||
echo "Starting background worker for queues..."
|
||||
node ace queue:work --queue=downloads &
|
||||
|
||||
# Start the AdonisJS application
|
||||
echo "Starting AdonisJS application..."
|
||||
exec node bin/server.js
|
||||
|
|
@ -15,7 +15,6 @@ services:
|
|||
- NODE_ENV=production
|
||||
- PORT=8080
|
||||
- LOG_LEVEL=debug
|
||||
- DRIVE_DISK=fs
|
||||
- APP_KEY=replaceme
|
||||
- HOST=0.0.0.0
|
||||
- URL=replaceme
|
||||
|
|
@ -29,6 +28,8 @@ services:
|
|||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/usr/local/bin/entrypoint.sh"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/api/v1/health"]
|
||||
|
|
@ -63,4 +64,17 @@ services:
|
|||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: nomad_redis
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- /opt/project-nomad/redis:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
Loading…
Reference in New Issue
Block a user