From 8e9131d2ff9d812823a1567ac4451c6736623048 Mon Sep 17 00:00:00 2001 From: Ben Gauger Date: Tue, 24 Mar 2026 15:44:37 -0600 Subject: [PATCH] feat(maps): add global map download from Protomaps --- admin/app/controllers/maps_controller.ts | 12 ++++ admin/app/services/map_service.ts | 74 ++++++++++++++++++++++++ admin/inertia/lib/api.ts | 23 ++++++++ admin/inertia/pages/settings/maps.tsx | 67 +++++++++++++++++++++ admin/start/routes.ts | 2 + 5 files changed, 178 insertions(+) diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index 54f0e8f..9927a19 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -73,6 +73,18 @@ export default class MapsController { return await this.mapService.listRegions() } + async globalMapInfo({}: HttpContext) { + return await this.mapService.getGlobalMapInfo() + } + + async downloadGlobalMap({}: HttpContext) { + const result = await this.mapService.downloadGlobalMap() + return { + message: 'Download started successfully', + ...result, + } + } + async styles({ request, response }: HttpContext) { // Automatically ensure base assets are present before generating styles const baseAssetsExist = await this.mapService.ensureBaseAssets() diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index beb74b2..4c34762 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -21,6 +21,16 @@ import InstalledResource from '#models/installed_resource' import { CollectionManifestService } from './collection_manifest_service.js' import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' +const PROTOMAPS_BUILDS_METADATA_URL = 'https://build-metadata.protomaps.dev/builds.json' +const PROTOMAPS_BUILD_BASE_URL = 'https://build.protomaps.com' + +export interface ProtomapsBuildInfo { + url: string + date: string + size: number + key: string +} + const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', 'application/x-gzip', @@ -398,6 +408,70 @@ export class MapService implements IMapService { return template } + async getGlobalMapInfo(): Promise { + const { default: axios } = await import('axios') + const response = await axios.get(PROTOMAPS_BUILDS_METADATA_URL, { timeout: 15000 }) + const builds = response.data as Array<{ key: string; size: number }> + + if (!builds || builds.length === 0) { + throw new Error('No protomaps builds found') + } + + // Latest build first + const sorted = builds.sort((a, b) => b.key.localeCompare(a.key)) + const latest = sorted[0] + + const dateStr = latest.key.replace('.pmtiles', '') + const date = `${dateStr.slice(0, 4)}-${dateStr.slice(4, 6)}-${dateStr.slice(6, 8)}` + + return { + url: `${PROTOMAPS_BUILD_BASE_URL}/${latest.key}`, + date, + size: latest.size, + key: latest.key, + } + } + + async downloadGlobalMap(): Promise<{ filename: string; jobId?: string }> { + const info = await this.getGlobalMapInfo() + + const existing = await RunDownloadJob.getByUrl(info.url) + if (existing) { + throw new Error(`Download already in progress for URL ${info.url}`) + } + + const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', info.key) + + // First, ensure base assets are present - the global map depends on them + const baseAssetsExist = await this.ensureBaseAssets() + if (!baseAssetsExist) { + throw new Error( + 'Base map assets are missing and could not be downloaded. Please check your connection and try again.' + ) + } + + // forceNew: false so retries resume partial downloads + const result = await RunDownloadJob.dispatch({ + url: info.url, + filepath, + timeout: 30000, + allowedMimeTypes: PMTILES_MIME_TYPES, + forceNew: false, + filetype: 'map', + }) + + if (!result.job) { + throw new Error('Failed to dispatch download job') + } + + logger.info(`[MapService] Dispatched global map download job ${result.job.id}`) + + return { + filename: info.key, + jobId: result.job?.id, + } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.pmtiles')) { diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index a47f326..2186e32 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -486,6 +486,29 @@ class API { } } + async getGlobalMapInfo() { + return catchInternal(async () => { + const response = await this.client.get<{ + url: string + date: string + size: number + key: string + }>('/maps/global-map-info') + return response.data + })() + } + + async downloadGlobalMap() { + return catchInternal(async () => { + const response = await this.client.post<{ + message: string + filename: string + jobId?: string + }>('/maps/download-global-map') + return response.data + })() + } + async listCuratedMapCollections() { return catchInternal(async () => { const response = await this.client.get( diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 153f23a..0212931 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -16,8 +16,10 @@ import CuratedCollectionCard from '~/components/CuratedCollectionCard' import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' +import { formatBytes } from '~/lib/util' const CURATED_COLLECTIONS_KEY = 'curated-map-collections' +const GLOBAL_MAP_INFO_KEY = 'global-map-info' export default function MapsManager(props: { maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] } @@ -38,6 +40,31 @@ export default function MapsManager(props: { enabled: true, }) + const { data: globalMapInfo } = useQuery({ + queryKey: [GLOBAL_MAP_INFO_KEY], + queryFn: () => api.getGlobalMapInfo(), + refetchOnWindowFocus: false, + }) + + const downloadGlobalMap = useMutation({ + mutationFn: () => api.downloadGlobalMap(), + onSuccess: () => { + invalidateDownloads() + addNotification({ + type: 'success', + message: 'Global map download has been queued. This is a large file (~125 GB) and may take a while.', + }) + closeAllModals() + }, + onError: (error) => { + console.error('Error downloading global map:', error) + addNotification({ + type: 'error', + message: 'Failed to start the global map download. Please try again.', + }) + }, + }) + async function downloadBaseAssets() { try { setDownloading(true) @@ -146,6 +173,29 @@ export default function MapsManager(props: { ) } + async function confirmGlobalMapDownload() { + if (!globalMapInfo) return + openModal( + downloadGlobalMap.mutate()} + onCancel={closeAllModals} + open={true} + confirmText="Download" + cancelText="Cancel" + confirmVariant="primary" + confirmLoading={downloadGlobalMap.isPending} + > +

+ This will download the full Protomaps global map ({formatBytes(globalMapInfo.size, 1)}, build {globalMapInfo.date}). + Covers the entire planet so you won't need individual region files. + Make sure you have enough disk space. +

+ , + 'confirm-global-map-download-modal' + ) + } + async function openDownloadModal() { openModal( )} + {globalMapInfo && ( + confirmGlobalMapDownload(), + }} + /> + )}