diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index 1bf8fc7..ab91f32 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -1,5 +1,6 @@ import { MapService } from '#services/map_service' import { + downloadCollectionValidator, filenameParamValidator, remoteDownloadValidator, remoteDownloadValidatorOptional, @@ -36,6 +37,16 @@ export default class MapsController { } } + async downloadCollection({ request }: HttpContext) { + const payload = await request.validateUsing(downloadCollectionValidator) + const resources = await this.mapService.downloadCollection(payload.slug) + return { + message: 'Collection download started successfully', + slug: payload.slug, + resources, + } + } + // For providing a "preflight" check in the UI before actually starting a background download async downloadRemotePreflight({ request }: HttpContext) { const payload = await request.validateUsing(remoteDownloadValidator) @@ -43,6 +54,15 @@ export default class MapsController { return info } + async fetchLatestCollections({}: HttpContext) { + const success = await this.mapService.fetchLatestCollections() + return { success } + } + + async listCuratedCollections({}: HttpContext) { + return await this.mapService.listCuratedCollections() + } + async listRegions({}: HttpContext) { return await this.mapService.listRegions() } diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index e556964..68a1c44 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -36,12 +36,12 @@ export default class ZimController { async downloadCollection({ request }: HttpContext) { const payload = await request.validateUsing(downloadCollectionValidator) - const resource_count = await this.zimService.downloadCollection(payload.slug) + const resources = await this.zimService.downloadCollection(payload.slug) return { message: 'Download started successfully', slug: payload.slug, - resource_count, + resources, } } diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index 0f788d0..f899c5d 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -5,6 +5,7 @@ import { doResumableDownload } from '../utils/downloads.js' import { createHash } from 'crypto' import { DockerService } from '#services/docker_service' import { ZimService } from '#services/zim_service' +import { MapService } from '#services/map_service' export class RunDownloadJob { static get queue() { @@ -23,9 +24,9 @@ export class RunDownloadJob { const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } = job.data as RunDownloadJobParams - // console.log("Simulating delay for job for URL:", url) - // await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay - // console.log("Starting download for URL:", url) + // console.log("Simulating delay for job for URL:", url) + // await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay + // console.log("Starting download for URL:", url) // // simulate progress updates for demonstration // for (let progress = 0; progress <= 100; progress += 10) { @@ -45,17 +46,20 @@ export class RunDownloadJob { job.updateProgress(Math.floor(progressPercent)) }, async onComplete(url) { - if (filetype === 'zim') { - try { + try { + if (filetype === 'zim') { const dockerService = new DockerService() const zimService = new ZimService(dockerService) await zimService.downloadRemoteSuccessCallback([url], true) - } catch (error) { - console.error( - `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`, - error - ) + } else if (filetype === 'map') { + const mapsService = new MapService() + await mapsService.downloadRemoteSuccessCallback([url], false) } + } catch (error) { + console.error( + `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`, + error + ) } job.updateProgress(100) }, diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index 0637ffe..5f2336c 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -1,5 +1,9 @@ import { BaseStylesFile, MapLayer } from '../../types/maps.js' -import { FileEntry } from '../../types/files.js' +import { + DownloadCollectionOperation, + DownloadRemoteSuccessCallback, + FileEntry, +} from '../../types/files.js' import { doResumableDownloadWithRetry } from '../utils/downloads.js' import { extract } from 'tar' import env from '#start/env' @@ -16,6 +20,11 @@ import urlJoin from 'url-join' import axios from 'axios' import { RunDownloadJob } from '#jobs/run_download_job' import logger from '@adonisjs/core/services/logger' +import { CuratedCollectionsFile, CuratedCollectionWithStatus } from '../../types/downloads.js' +import CuratedCollection from '#models/curated_collection' +import vine from '@vinejs/vine' +import { curatedCollectionsFileSchema } from '#validators/curated_collections' +import CuratedCollectionResource from '#models/curated_collection_resource' const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', @@ -23,11 +32,19 @@ const BASE_ASSETS_MIME_TYPES = [ 'application/octet-stream', ] +const COLLECTIONS_URL = + 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json' + const PMTILES_ATTRIBUTION = 'Protomaps © OpenStreetMap' const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream'] -export class MapService { +interface IMapService { + downloadCollection: DownloadCollectionOperation + downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback +} + +export class MapService implements IMapService { private readonly mapStoragePath = '/storage/maps' private readonly baseStylesFile = 'nomad-base-styles.json' private readonly basemapsAssetsDir = 'basemaps-assets' @@ -80,6 +97,62 @@ export class MapService { return true } + async downloadCollection(slug: string) { + const collection = await CuratedCollection.query() + .where('slug', slug) + .andWhere('type', 'map') + .first() + if (!collection) { + return null + } + + const resources = await collection.related('resources').query().where('downloaded', false) + if (resources.length === 0) { + return null + } + + const downloadUrls = resources.map((res) => res.url) + const downloadFilenames: string[] = [] + + for (const url of downloadUrls) { + const existing = await RunDownloadJob.getByUrl(url) + if (existing) { + logger.warn(`[MapService] Download already in progress for URL ${url}, skipping.`) + continue + } + + // Extract the filename from the URL + const filename = url.split('/').pop() + if (!filename) { + logger.warn(`[MapService] Could not determine filename from URL ${url}, skipping.`) + continue + } + + downloadFilenames.push(filename) + const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) + + await RunDownloadJob.dispatch({ + url, + filepath, + timeout: 30000, + allowedMimeTypes: PMTILES_MIME_TYPES, + forceNew: true, + filetype: 'map', + }) + } + + return downloadFilenames.length > 0 ? downloadFilenames : null + } + + async downloadRemoteSuccessCallback(urls: string[], _: boolean) { + const resources = await CuratedCollectionResource.query().whereIn('url', urls) + for (const resource of resources) { + resource.downloaded = true + await resource.save() + logger.info(`[MapService] Marked resource as downloaded: ${resource.url}`) + } + } + async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> { const parsed = new URL(url) if (!parsed.pathname.endsWith('.pmtiles')) { @@ -105,7 +178,7 @@ export class MapService { timeout: 30000, allowedMimeTypes: PMTILES_MIME_TYPES, forceNew: true, - filetype: 'pmtiles', + filetype: 'map', }) if (!result.job) { @@ -193,6 +266,47 @@ export class MapService { return !!baseStyleItem && !!basemapsAssetsItem } + async listCuratedCollections(): Promise { + const collections = await CuratedCollection.query().where('type', 'map').preload('resources') + return collections.map((collection) => ({ + ...(collection.serialize() as CuratedCollection), + all_downloaded: collection.resources.every((res) => res.downloaded), + })) + } + + async fetchLatestCollections(): Promise { + try { + const response = await axios.get(COLLECTIONS_URL) + + const validated = await vine.validate({ + schema: curatedCollectionsFileSchema, + data: response.data, + }) + + for (const collection of validated.collections) { + const collectionResult = await CuratedCollection.updateOrCreate( + { slug: collection.slug }, + { + ...collection, + type: 'map', + } + ) + logger.info(`[MapService] Upserted curated collection: ${collection.slug}`) + + await collectionResult.related('resources').createMany(collection.resources) + logger.info( + `[MapService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}` + ) + } + + return true + } catch (error) { + console.error(error) + logger.error(`[MapService] Failed to download latest Kiwix collections:`, error) + return false + } + } + private async listMapStorageItems(): Promise { await ensureDirectoryExists(this.baseDirPath) return await listDirectoryContents(this.baseDirPath) diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 6ea8d89..9cb1cc1 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -23,13 +23,20 @@ import { curatedCollectionsFileSchema } from '#validators/curated_collections' import CuratedCollection from '#models/curated_collection' import CuratedCollectionResource from '#models/curated_collection_resource' import { RunDownloadJob } from '#jobs/run_download_job' +import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const COLLECTIONS_URL = 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json' + +interface IZimService { + downloadCollection: DownloadCollectionOperation + downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback +} + @inject() -export class ZimService { +export class ZimService implements IZimService { constructor(private dockerService: DockerService) {} async list() { @@ -176,8 +183,8 @@ export class ZimService { } } - async downloadCollection(slug: string): Promise { - const collection = await CuratedCollection.find(slug) + async downloadCollection(slug: string) { + const collection = await CuratedCollection.query().where('slug', slug).andWhere('type', 'zim').first() if (!collection) { return null } @@ -218,7 +225,7 @@ export class ZimService { } return downloadFilenames.length > 0 ? downloadFilenames : null - } + } async downloadRemoteSuccessCallback(urls: string[], restart = true) { // Restart KIWIX container to pick up new ZIM file @@ -239,7 +246,7 @@ export class ZimService { } async listCuratedCollections(): Promise { - const collections = await CuratedCollection.query().preload('resources') + const collections = await CuratedCollection.query().where('type', 'zim').preload('resources') return collections.map((collection) => ({ ...(collection.serialize() as CuratedCollection), all_downloaded: collection.resources.every((res) => res.downloaded), diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 3384f2c..76f3fb1 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -4,6 +4,7 @@ import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { SystemInformationResponse } from '../../types/system' import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads' +import { catchInternal } from './util' class API { private client: AxiosInstance @@ -17,99 +18,134 @@ class API { }) } - async listDocs() { - try { - const response = await this.client.get>('/docs/list') - return response.data - } catch (error) { - console.error('Error listing docs:', error) - throw error - } - } - - async listMapRegionFiles() { - try { - const response = await this.client.get<{ files: FileEntry[] }>('/maps/regions') - return response.data.files - } catch (error) { - console.error('Error listing map region files:', error) - throw error - } - } - - async downloadBaseMapAssets() { - try { - const response = await this.client.post<{ success: boolean }>('/maps/download-base-assets') - return response.data - } catch (error) { - console.error('Error downloading base map assets:', error) - throw error - } - } - - async downloadRemoteMapRegion(url: string) { - try { - const response = await this.client.post<{ message: string; filename: string; url: string }>( - '/maps/download-remote', - { url } - ) - return response.data - } catch (error) { - console.error('Error downloading remote map region:', error) - throw error - } - } - - async downloadRemoteMapRegionPreflight(url: string) { - try { - const response = await this.client.post< - { filename: string; size: number } | { message: string } - >('/maps/download-remote-preflight', { url }) - return response.data - } catch (error) { - console.error('Error preflighting remote map region download:', error) - throw error - } - } - - async listServices() { - try { - const response = await this.client.get>('/system/services') - return response.data - } catch (error) { - console.error('Error listing services:', error) - throw error - } - } - - async installService(service_name: string) { - try { - const response = await this.client.post<{ success: boolean; message: string }>( - '/system/services/install', - { service_name } - ) - return response.data - } catch (error) { - console.error('Error installing service:', error) - throw error - } - } - async affectService(service_name: string, action: 'start' | 'stop' | 'restart') { - try { + return catchInternal(async () => { const response = await this.client.post<{ success: boolean; message: string }>( '/system/services/affect', { service_name, action } ) return response.data - } catch (error) { - console.error('Error affecting service:', error) - throw error - } + })() } - async listZimFiles() { - return await this.client.get('/zim/list') + async downloadBaseMapAssets() { + return catchInternal(async () => { + const response = await this.client.post<{ success: boolean }>('/maps/download-base-assets') + return response.data + })() + } + + async downloadMapCollection(slug: string): Promise<{ + message: string + slug: string + resources: string[] | null + }> { + return catchInternal(async () => { + const response = await this.client.post('/maps/download-collection', { slug }) + return response.data + })() + } + + async downloadZimCollection(slug: string): Promise<{ + message: string + slug: string + resources: string[] | null + }> { + return catchInternal(async () => { + const response = await this.client.post('/zim/download-collection', { slug }) + return response.data + })() + } + + async downloadRemoteMapRegion(url: string) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string; filename: string; url: string }>( + '/maps/download-remote', + { url } + ) + return response.data + })() + } + + async downloadRemoteMapRegionPreflight(url: string) { + return catchInternal(async () => { + const response = await this.client.post< + { filename: string; size: number } | { message: string } + >('/maps/download-remote-preflight', { url }) + return response.data + })() + } + + async fetchLatestMapCollections(): Promise<{ success: boolean } | undefined> { + return catchInternal(async () => { + const response = await this.client.post<{ success: boolean }>( + '/maps/fetch-latest-collections' + ) + return response.data + })() + } + + async fetchLatestZimCollections(): Promise<{ success: boolean } | undefined> { + return catchInternal(async () => { + const response = await this.client.post<{ success: boolean }>('/zim/fetch-latest-collections') + return response.data + })() + } + + async getSystemInfo() { + return catchInternal(async () => { + const response = await this.client.get('/system/info') + return response.data + })() + } + + async installService(service_name: string) { + return catchInternal(async () => { + const response = await this.client.post<{ success: boolean; message: string }>( + '/system/services/install', + { service_name } + ) + return response.data + })() + } + + async listCuratedMapCollections() { + return catchInternal(async () => { + const response = await this.client.get( + '/maps/curated-collections' + ) + return response.data + })() + } + + async listCuratedZimCollections() { + return catchInternal(async () => { + const response = await this.client.get( + '/zim/curated-collections' + ) + return response.data + })() + } + + async listDocs() { + return catchInternal(async () => { + const response = await this.client.get>('/docs/list') + return response.data + })() + } + + async listMapRegionFiles() { + return catchInternal(async () => { + const response = await this.client.get<{ files: FileEntry[] }>('/maps/regions') + return response.data.files + })() + } + + async listServices() { + return catchInternal(async () => { + const response = await this.client.get>('/system/services') + return response.data + })() } async listRemoteZimFiles({ @@ -121,94 +157,29 @@ class API { count?: number query?: string }) { - return await this.client.get('/zim/list-remote', { - params: { - start, - count, - query, - }, - }) + return catchInternal(async () => { + return await this.client.get('/zim/list-remote', { + params: { + start, + count, + query, + }, + }) + })() } - async listCuratedZimCollections() { - try { - const response = await this.client.get( - '/zim/curated-collections' - ) - return response.data - } catch (error) { - console.error('Error listing curated ZIM collections:', error) - throw error - } + async listZimFiles() { + return catchInternal(async () => { + return await this.client.get('/zim/list') + })() } - async downloadRemoteZimFile(url: string): Promise<{ - message: string - filename: string - url: string - }> { - try { - const response = await this.client.post('/zim/download-remote', { url }) - return response.data - } catch (error) { - console.error('Error downloading remote ZIM file:', error) - throw error - } - } - - async downloadZimCollection(slug: string): Promise<{ - message: string - slug: string - resource_count: number - }> { - try { - const response = await this.client.post('/zim/download-collection', { slug }) - return response.data - } catch (error) { - console.error('Error downloading ZIM collection:', error) - throw error - } - } - - async fetchLatestZimCollections(): Promise<{ success: boolean }> { - try { - const response = await this.client.post<{ success: boolean }>('/zim/fetch-latest-collections') - return response.data - } catch (error) { - console.error('Error fetching latest ZIM collections:', error) - throw error - } - } - - async deleteZimFile(key: string) { - try { - const response = await this.client.delete(`/zim/${key}`) - return response.data - } catch (error) { - console.error('Error deleting ZIM file:', error) - throw error - } - } - - async getSystemInfo() { - try { - const response = await this.client.get('/system/info') - return response.data - } catch (error) { - console.error('Error fetching system info:', error) - throw error - } - } - - async listDownloadJobs(filetype?: string): Promise { - try { + async listDownloadJobs(filetype?: string): Promise { + return catchInternal(async () => { const endpoint = filetype ? `/downloads/jobs/${filetype}` : '/downloads/jobs' const response = await this.client.get(endpoint) return response.data - } catch (error) { - console.error('Error listing download jobs:', error) - throw error - } + })() } } diff --git a/admin/inertia/lib/util.ts b/admin/inertia/lib/util.ts index 7e1d1b5..9b24bde 100644 --- a/admin/inertia/lib/util.ts +++ b/admin/inertia/lib/util.ts @@ -1,55 +1,87 @@ -import axios from "axios"; +import axios from 'axios' export function capitalizeFirstLetter(str?: string | null): string { - if (!str) return ""; - return str.charAt(0).toUpperCase() + str.slice(1); + if (!str) return '' + return str.charAt(0).toUpperCase() + str.slice(1) } export function formatBytes(bytes: number, decimals = 2): string { - if (bytes === 0) return '0 Bytes'; - const k = 1024; - const dm = decimals < 0 ? 0 : decimals; - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; + if (bytes === 0) return '0 Bytes' + const k = 1024 + const dm = decimals < 0 ? 0 : decimals + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i] } export async function testInternetConnection(): Promise { try { const response = await axios.get('https://1.1.1.1/cdn-cgi/trace', { timeout: 5000, - }); - return response.status === 200; + }) + return response.status === 200 } catch (error) { - console.error("Error testing internet connection:", error); - return false; + console.error('Error testing internet connection:', error) + return false } } export function generateRandomString(length: number): string { - const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - let result = ''; + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + let result = '' for (let i = 0; i < length; i++) { - result += characters.charAt(Math.floor(Math.random() * characters.length)); + result += characters.charAt(Math.floor(Math.random() * characters.length)) } - return result; + return result } export function generateUUID(): string { - const arr = new Uint8Array(16); + const arr = new Uint8Array(16) if (window.crypto && window.crypto.getRandomValues) { - window.crypto.getRandomValues(arr); + window.crypto.getRandomValues(arr) } else { // Fallback for non-secure contexts where window.crypto is not available // This is not cryptographically secure, but can be used for non-critical purposes for (let i = 0; i < 16; i++) { - arr[i] = Math.floor(Math.random() * 256); + arr[i] = Math.floor(Math.random() * 256) } } - arr[6] = (arr[6] & 0x0f) | 0x40; // Version 4 - arr[8] = (arr[8] & 0x3f) | 0x80; // Variant bits + arr[6] = (arr[6] & 0x0f) | 0x40 // Version 4 + arr[8] = (arr[8] & 0x3f) | 0x80 // Variant bits - const hex = Array.from(arr, byte => byte.toString(16).padStart(2, '0')).join(''); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + const hex = Array.from(arr, (byte) => byte.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +/** + * Extracts the file name from a given path while handling both forward and backward slashes. + * @param path The full file path. + * @returns The extracted file name. + */ +export const extractFileName = (path: string) => { + if (!path) return '' + if (path.includes('/')) { + return path.substring(path.lastIndexOf('/') + 1) + } + if (path.includes('\\')) { + return path.substring(path.lastIndexOf('\\') + 1) + } + return path +} + +/** + * A higher-order function that wraps an asynchronous function to catch and log internal errors. + * @param fn The asynchronous function to be wrapped. + * @returns A new function that executes the original function and logs any errors. Returns undefined in case of an error. + */ +export function catchInternal any>(fn: Fn): (...args: Parameters) => Promise | undefined> { + return async (...args: any[]) => { + try { + return await fn(...args) + } catch (error) { + console.error('Internal error caught:', error) + return undefined + } + } } \ No newline at end of file diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 343b7d5..49be5b4 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -10,19 +10,44 @@ import { useNotifications } from '~/context/NotificationContext' import { useState } from 'react' import api from '~/lib/api' import DownloadURLModal from '~/components/DownloadURLModal' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import useDownloads from '~/hooks/useDownloads' +import StyledSectionHeader from '~/components/StyledSectionHeader' +import HorizontalBarChart from '~/components/HorizontalBarChart' +import { extractFileName } from '~/lib/util' +import CuratedCollectionCard from '~/components/CuratedCollectionCard' +import { CuratedCollectionWithStatus } from '../../../types/downloads' + +const CURATED_COLLECTIONS_KEY = 'curated-map-collections' export default function MapsManager(props: { maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] } }) { + const queryClient = useQueryClient() const { openModal, closeAllModals } = useModals() const { addNotification } = useNotifications() const [downloading, setDownloading] = useState(false) + const { data: curatedCollections } = useQuery({ + queryKey: [CURATED_COLLECTIONS_KEY], + queryFn: () => api.listCuratedMapCollections(), + refetchOnWindowFocus: false, + }) + + const { data: downloads, invalidate: invalidateDownloads } = useDownloads({ + filetype: 'map', + enabled: true, + }) + async function downloadBaseAssets() { try { setDownloading(true) const res = await api.downloadBaseMapAssets() + if (!res) { + throw new Error('An unknown error occurred while downloading base assets.') + } + if (res.success) { addNotification({ type: 'success', @@ -41,6 +66,24 @@ export default function MapsManager(props: { } } + async function downloadFile(record: string) { + try { + //await api.downloadRemoteZimFile(record.download_url) + invalidateDownloads() + } catch (error) { + console.error('Error downloading file:', error) + } + } + + async function downloadCollection(record: CuratedCollectionWithStatus) { + try { + await api.downloadMapCollection(record.slug) + invalidateDownloads() + } catch (error) { + console.error('Error downloading collection:', error) + } + } + async function confirmDeleteFile(file: FileEntry) { openModal( { + if (isCollection) { + if (record.all_downloaded) { + addNotification({ + message: `All resources in the collection "${record.name}" have already been downloaded.`, + type: 'info', + }) + return + } + downloadCollection(record) + } else { + downloadFile(record) + } + closeAllModals() + }} + onCancel={closeAllModals} + open={true} + confirmText="Download" + cancelText="Cancel" + confirmVariant="primary" + > +

+ Are you sure you want to download {isCollection ? record.name : record}? + It may take some time for it to be available depending on the file size and your internet + connection. +

+ , + 'confirm-download-file-modal' + ) + } + async function openDownloadModal() { openModal( api.fetchLatestMapCollections(), + onSuccess: () => { + addNotification({ + message: 'Successfully fetched the latest map collections.', + type: 'success', + }) + queryClient.invalidateQueries({ queryKey: [CURATED_COLLECTIONS_KEY] }) + }, + }) + return ( @@ -81,20 +171,40 @@ export default function MapsManager(props: {

Maps Manager

-

Manage your stored map data files.

+

Manage your stored map files and explore new regions!

+
+
+ + Download Custom Map File + + fetchLatestCollections.mutate()} + disabled={fetchLatestCollections.isPending} + icon="CloudArrowDownIcon" + > + Fetch Latest Collections +
- - Download New Map File -
{!props.maps.baseAssetsExist && ( )} + +
+ {curatedCollections?.map((collection) => ( + confirmDownload(collection)} + /> + ))} +
+ className="font-semibold mt-4" rowLines={true} @@ -122,6 +232,28 @@ export default function MapsManager(props: { ]} data={props.maps.regionFiles || []} /> + +
+ {downloads && downloads.length > 0 ? ( + downloads.map((download) => ( +
+ +
+ )) + ) : ( +

No active downloads

+ )} +
diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index 3aa3d53..cecac1c 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -12,7 +12,7 @@ import StyledTable from '~/components/StyledTable' import SettingsLayout from '~/layouts/SettingsLayout' import { Head } from '@inertiajs/react' import { ListRemoteZimFilesResponse, RemoteZimFileEntry } from '../../../../types/zim' -import { formatBytes } from '~/lib/util' +import { extractFileName, formatBytes } from '~/lib/util' import StyledButton from '~/components/StyledButton' import { useModals } from '~/context/ModalContext' import StyledModal from '~/components/StyledModal' @@ -188,17 +188,6 @@ export default function ZimRemoteExplorer() { }, }) - const extractFileName = (path: string) => { - if (!path) return '' - if (path.includes('/')) { - return path.substring(path.lastIndexOf('/') + 1) - } - if (path.includes('\\')) { - return path.substring(path.lastIndexOf('\\') + 1) - } - return path - } - return ( diff --git a/admin/package.json b/admin/package.json index 8b12b58..b706d84 100644 --- a/admin/package.json +++ b/admin/package.json @@ -12,7 +12,8 @@ "test": "node ace test", "lint": "eslint .", "format": "prettier --write .", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "work:downloads": "node ace queue:work --queue=downloads" }, "imports": { "#controllers/*": "./app/controllers/*.js", diff --git a/admin/start/routes.ts b/admin/start/routes.ts index e208e41..02adcda 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -52,9 +52,12 @@ router router.get('/regions', [MapsController, 'listRegions']) router.get('/styles', [MapsController, 'styles']) router.get('/preflight', [MapsController, 'checkBaseAssets']) + router.get('/curated-collections', [MapsController, 'listCuratedCollections']) + router.post('/fetch-latest-collections', [MapsController, 'fetchLatestCollections']) router.post('/download-base-assets', [MapsController, 'downloadBaseAssets']) router.post('/download-remote', [MapsController, 'downloadRemote']) router.post('/download-remote-preflight', [MapsController, 'downloadRemotePreflight']) + router.post('/download-collection', [MapsController, 'downloadCollection']) router.delete('/:filename', [MapsController, 'delete']) }) .prefix('/api/maps') diff --git a/admin/types/files.ts b/admin/types/files.ts index d3a8aa6..82d9a87 100644 --- a/admin/types/files.ts +++ b/admin/types/files.ts @@ -28,3 +28,6 @@ export type DownloadOptions = { onError?: (error: Error) => void onComplete?: (filepath: string) => void } + +export type DownloadCollectionOperation = (slug: string) => Promise +export type DownloadRemoteSuccessCallback = (urls: string[], restart: boolean) => Promise \ No newline at end of file