import { ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, } from '../../types/zim.js' import axios from 'axios' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' import transmit from '@adonisjs/transmit/services/main' import logger from '@adonisjs/core/services/logger' import { DockerService } from './docker_service.js' import { inject } from '@adonisjs/core' import { doBackgroundDownload } from '../utils/downloads.js' import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists, listDirectoryContents, } from '../utils/fs.js' import { join } from 'path' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const BROADCAST_CHANNEL = 'zim-downloads' @inject() export class ZimService { private zimStoragePath = '/storage/zim' private activeDownloads = new Map() constructor(private dockerService: DockerService) {} async list() { const dirPath = join(process.cwd(), this.zimStoragePath) await ensureDirectoryExists(dirPath) const files = await listDirectoryContents(dirPath) return { files, } } async listRemote({ start, count, query, }: { start: number count: number query?: string }): Promise { const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' const res = await axios.get(LIBRARY_BASE_URL, { params: { start: start, count: count, lang: 'eng', ...(query ? { q: query } : {}), }, responseType: 'text', }) const data = res.data const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '', textNodeName: '#text', }) const result = parser.parse(data) if (!isRawListRemoteZimFilesResponse(result)) { throw new Error('Invalid response format from remote library') } const entries = Array.isArray(result.feed.entry) ? result.feed.entry : [result.feed.entry] const filtered = entries.filter((entry: any) => { return isRawRemoteZimFileEntry(entry) }) const mapped: (RemoteZimFileEntry | null)[] = filtered.map((entry: RawRemoteZimFileEntry) => { const downloadLink = entry.link.find((link: any) => { return ( typeof link === 'object' && 'rel' in link && 'length' in link && 'href' in link && 'type' in link && link.type === 'application/x-zim' ) }) if (!downloadLink) { return null } // downloadLink['href'] will end with .meta4, we need to remove that to get the actual download URL const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6) const file_name = download_url.split('/').pop() || `${entry.title}.zim` const sizeBytes = parseInt(downloadLink['length'], 10) return { id: entry.id, title: entry.title, updated: entry.updated, summary: entry.summary, size_bytes: sizeBytes || 0, download_url: download_url, author: entry.author.name, file_name: file_name, } }) // Filter out any null entries (those without a valid download link) // or files that already exist in the local storage const existing = await this.list() const existingKeys = new Set(existing.files.map((file) => file.name)) const withoutExisting = mapped.filter( (entry): entry is RemoteZimFileEntry => entry !== null && !existingKeys.has(entry.file_name) ) return { items: withoutExisting, has_more: result.feed.totalResults > start, total_count: result.feed.totalResults, } } async downloadRemote(url: string): Promise { const parsed = new URL(url) if (!parsed.pathname.endsWith('.zim')) { throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`) } const existing = this.activeDownloads.get(url) if (existing) { throw new Error(`Download already in progress for URL ${url}`) } await ensureDirectoryExists(join(process.cwd(), this.zimStoragePath)) // Extract the filename from the URL const filename = url.split('/').pop() if (!filename) { throw new Error('Could not determine filename from URL') } const path = join(process.cwd(), this.zimStoragePath, filename) // Don't await the download, run it in the background doBackgroundDownload({ url, path, channel: BROADCAST_CHANNEL, activeDownloads: this.activeDownloads, allowedMimeTypes: ZIM_MIME_TYPES, timeout: 30000, forceNew: true, onComplete: async () => { // Restart KIWIX container to pick up new ZIM file await this.dockerService .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart') .catch((error) => { logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. }) }, }) return filename } listActiveDownloads(): string[] { return Array.from(this.activeDownloads.keys()) } cancelDownload(url: string): boolean { const entry = this.activeDownloads.get(url) if (entry) { entry.abort() this.activeDownloads.delete(url) transmit.broadcast(BROADCAST_CHANNEL, { url, status: 'cancelled' }) return true } return false } async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.zim')) { fileName += '.zim' } const fullPath = join(process.cwd(), this.zimStoragePath, fileName) const exists = await getFileStatsIfExists(fullPath) if (!exists) { throw new Error('not_found') } await deleteFileIfExists(fullPath) } }