diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index 7b65a3bd..c655040e 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -44,7 +44,7 @@ import { composeMapStyle, DEFAULT_VECTOR_MAP_ATTRIBUTION, MAP_ARCHIVE_MANIFEST_FILENAME, - parseMapArchiveManifest, + parseMapArchiveManifestJson, } from '../utils/map_styles.js' const execFileAsync = promisify(execFile) @@ -413,18 +413,15 @@ export class MapService implements IMapService { } const manifest = await getFile(manifestPath, 'string') - if (!manifest) return { archives: [], rejectedFilenames: [] } + if (!manifest) return { status: 'absent', archives: [], rejectedFilenames: [] } - try { - return parseMapArchiveManifest(JSON.parse(manifest.toString())) - } catch (error) { + const parsed = parseMapArchiveManifestJson(manifest.toString()) + if (parsed.status === 'invalid') { logger.warn( - `[MapService] Ignoring malformed ${MAP_ARCHIVE_MANIFEST_FILENAME}: ${ - error instanceof Error ? error.message : String(error) - }` + `[MapService] ${MAP_ARCHIVE_MANIFEST_FILENAME} is malformed; blocking unmanifested PMTiles archives` ) - return { archives: [], rejectedFilenames: [] } } + return parsed } async listCuratedCollections(): Promise { diff --git a/admin/app/services/raster_pmtiles_converter.ts b/admin/app/services/raster_pmtiles_converter.ts new file mode 100644 index 00000000..f55bde43 --- /dev/null +++ b/admin/app/services/raster_pmtiles_converter.ts @@ -0,0 +1,220 @@ +import { execFile } from 'node:child_process' +import { createHash, randomBytes } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { access, link, mkdir, rm, stat } from 'node:fs/promises' +import { basename, extname, join, resolve, sep } from 'node:path' +import { promisify } from 'node:util' + +import { PMTILES_BINARY_PATH } from '../../constants/map_regions.js' +import type { RasterMapArchiveMetadata } from '../../types/maps.js' + +const execFileAsync = promisify(execFile) +const SAFE_ARCHIVE_FILENAME = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}\.pmtiles$/ +const SAFE_RESOURCE_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/ +const SAFE_REGION_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/ +const HTTPS_URL = /^https:\/\//i +const TILE_FORMATS: Readonly> = { + jpg: 'jpeg', + jpeg: 'jpeg', + png: 'png', + webp: 'webp', +} + +export type RasterPmtilesConversionRequest = Pick< + RasterMapArchiveMetadata, + | 'filename' + | 'resourceId' + | 'role' + | 'regionId' + | 'tileSize' + | 'attribution' + | 'sourceUrl' + | 'sourceVersion' +> & { + sourceMbtilesPath: string + outputDirectory: string +} + +export type RasterPmtilesCommandResult = { + stdout: string + stderr: string +} + +export type RasterPmtilesConversionDependencies = { + runPmtiles?: (args: string[]) => Promise + now?: () => Date +} + +export type RasterPmtilesConversionResult = { + outputPath: string + archive: RasterMapArchiveMetadata +} + +type PmtilesHeader = Pick + +async function defaultPmtilesRunner(args: string[]): Promise { + const result = await execFileAsync(PMTILES_BINARY_PATH, args, { + maxBuffer: 1024 * 1024, + }) + return { + stdout: result.stdout.toString(), + stderr: result.stderr.toString(), + } +} + +function parseRasterHeader(value: string): PmtilesHeader { + let header: unknown + try { + header = JSON.parse(value) + } catch { + throw new Error('pmtiles show returned malformed header JSON') + } + if (!header || typeof header !== 'object') { + throw new Error('pmtiles show returned an invalid header') + } + + const candidate = header as Record + const tileFormat = + typeof candidate.tile_type === 'string' ? TILE_FORMATS[candidate.tile_type] : undefined + const bounds = candidate.bounds + const minzoom = candidate.minzoom + const maxzoom = candidate.maxzoom + if (!tileFormat) throw new Error('Converted PMTiles archive is not a supported raster tile set') + if ( + !Number.isInteger(minzoom) || + !Number.isInteger(maxzoom) || + (minzoom as number) < 0 || + (maxzoom as number) > 30 || + (minzoom as number) > (maxzoom as number) + ) { + throw new Error('Converted PMTiles archive has invalid zoom metadata') + } + if ( + !Array.isArray(bounds) || + bounds.length !== 4 || + !bounds.every((coordinate) => typeof coordinate === 'number' && Number.isFinite(coordinate)) + ) { + throw new Error('Converted PMTiles archive has invalid bounds metadata') + } + const [west, south, east, north] = bounds as number[] + if ( + west < -180 || + west > 180 || + east < -180 || + east > 180 || + south < -90 || + south > 90 || + north < -90 || + north > 90 || + west >= east || + south >= north + ) { + throw new Error('Converted PMTiles archive has invalid bounds metadata') + } + + return { + tileFormat, + minzoom: minzoom as number, + maxzoom: maxzoom as number, + bounds: [west, south, east, north], + } +} + +async function sha256File(filepath: string): Promise { + const hash = createHash('sha256') + for await (const chunk of createReadStream(filepath)) hash.update(chunk) + return hash.digest('hex') +} + +function validateRequest(request: RasterPmtilesConversionRequest): void { + if ( + !SAFE_ARCHIVE_FILENAME.test(request.filename) || + basename(request.filename) !== request.filename + ) { + throw new Error('Invalid PMTiles output filename') + } + if (extname(request.sourceMbtilesPath).toLowerCase() !== '.mbtiles') { + throw new Error('Raster conversion input must be an MBTiles archive') + } + if (!SAFE_RESOURCE_ID.test(request.resourceId)) throw new Error('Invalid map resource ID') + if (!SAFE_REGION_ID.test(request.regionId)) throw new Error('Invalid map region ID') + if (request.role !== 'aerial' && request.role !== 'topographic') { + throw new Error('Invalid raster map role') + } + if (![128, 256, 512, 1024].includes(request.tileSize)) { + throw new Error('Invalid raster tile size') + } + if (!request.attribution.trim()) throw new Error('Raster attribution is required') + if (!HTTPS_URL.test(request.sourceUrl)) throw new Error('Raster source URL must use HTTPS') + if (!request.sourceVersion.trim()) throw new Error('Raster source version is required') +} + +/** + * Convert a provider-produced raster MBTiles archive into a verified PMTiles + * archive. The converter owns format inspection, checksum generation, and + * atomic activation; provider adapters only supply provenance and staging paths. + */ +export async function convertRasterMbtilesToPmtiles( + request: RasterPmtilesConversionRequest, + dependencies: RasterPmtilesConversionDependencies = {} +): Promise { + validateRequest(request) + const outputDirectory = resolve(request.outputDirectory) + const outputPath = resolve(join(outputDirectory, request.filename)) + if (!outputPath.startsWith(outputDirectory + sep)) throw new Error('Invalid PMTiles output path') + + await mkdir(outputDirectory, { recursive: true }) + try { + await access(outputPath) + throw new Error(`PMTiles output already exists: ${request.filename}`) + } catch (error) { + if (error instanceof Error && !('code' in error && error.code === 'ENOENT')) throw error + } + + const stagingPath = join( + outputDirectory, + `.${request.filename}.${randomBytes(8).toString('hex')}.partial` + ) + const runPmtiles = dependencies.runPmtiles ?? defaultPmtilesRunner + const now = dependencies.now ?? (() => new Date()) + + try { + await runPmtiles(['convert', resolve(request.sourceMbtilesPath), stagingPath]) + await runPmtiles(['verify', stagingPath]) + const inspected = await runPmtiles(['show', stagingPath, '--header-json']) + const header = parseRasterHeader(inspected.stdout) + const [sha256, outputStats] = await Promise.all([sha256File(stagingPath), stat(stagingPath)]) + const archive: RasterMapArchiveMetadata = { + filename: request.filename, + resourceId: request.resourceId, + kind: 'raster', + role: request.role, + regionId: request.regionId, + tileFormat: header.tileFormat, + tileSize: request.tileSize, + minzoom: header.minzoom, + maxzoom: header.maxzoom, + bounds: header.bounds, + attribution: request.attribution, + sourceUrl: request.sourceUrl, + sourceVersion: request.sourceVersion, + sha256, + installedBytes: outputStats.size, + verifiedAt: now().toISOString(), + } + + try { + await link(stagingPath, outputPath) + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'EEXIST') { + throw new Error(`PMTiles output already exists: ${request.filename}`) + } + throw error + } + await rm(stagingPath) + return { outputPath, archive } + } catch (error) { + await rm(stagingPath, { force: true }) + throw error + } +} diff --git a/admin/app/utils/map_styles.ts b/admin/app/utils/map_styles.ts index d1dbd264..2fcc3dc2 100644 --- a/admin/app/utils/map_styles.ts +++ b/admin/app/utils/map_styles.ts @@ -136,6 +136,10 @@ function isMapArchiveMetadata(value: unknown): value is MapArchiveMetadata { } } + if (archive.kind === 'vector' && (archive.role !== 'street' || archive.tileFormat !== 'mvt')) { + return false + } + if (archive.kind === 'raster') { if ( (archive.role !== 'aerial' && archive.role !== 'topographic') || @@ -171,16 +175,23 @@ function candidateFilename(value: unknown): string | null { * their filenames remain blocked from legacy-vector fallback. */ export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifest { - const empty = { archives: [], rejectedFilenames: [] } - if (!value || typeof value !== 'object') return empty + const invalid: ParsedMapArchiveManifest = { + status: 'invalid', + archives: [], + rejectedFilenames: [], + } + if (!value || typeof value !== 'object') return invalid const manifest = value as { schemaVersion?: unknown; archives?: unknown } - if (manifest.schemaVersion !== 1 || !manifest.archives || typeof manifest.archives !== 'object') { - return empty + if ( + manifest.schemaVersion !== 1 || + !manifest.archives || + typeof manifest.archives !== 'object' || + Array.isArray(manifest.archives) + ) { + return invalid } - const entries = Array.isArray(manifest.archives) - ? manifest.archives.map((candidate) => [candidateFilename(candidate), candidate] as const) - : Object.entries(manifest.archives) + const entries = Object.entries(manifest.archives) const archives: MapArchiveMetadata[] = [] const rejectedFilenames = new Set() const filenames = new Set() @@ -210,7 +221,15 @@ export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifes archives.push({ ...candidate }) } - return { archives, rejectedFilenames: [...rejectedFilenames] } + return { status: 'valid', archives, rejectedFilenames: [...rejectedFilenames] } +} + +export function parseMapArchiveManifestJson(value: string): ParsedMapArchiveManifest { + try { + return parseMapArchiveManifest(JSON.parse(value)) + } catch { + return { status: 'invalid', archives: [], rejectedFilenames: [] } + } } function compareMapArchiveVersions(a: string | null, b: string | null): number { @@ -249,7 +268,11 @@ export function sourceFromArchive(baseUrl: string, archive: MapArchiveMetadata): export function buildMapSourceDefinitions( baseUrl: string, regions: FileEntry[], - parsedManifest: ParsedMapArchiveManifest = { archives: [], rejectedFilenames: [] } + parsedManifest: ParsedMapArchiveManifest = { + status: 'absent', + archives: [], + rejectedFilenames: [], + } ): MapSourceDefinition[] { const metadataByFilename = new Map( parsedManifest.archives.map((archive) => [archive.filename, archive]) @@ -275,7 +298,13 @@ export function buildMapSourceDefinitions( }) continue } - if (rejectedFilenames.has(region.name) || RESERVED_RASTER_FILENAME.test(region.name)) continue + if ( + parsedManifest.status === 'invalid' || + rejectedFilenames.has(region.name) || + RESERVED_RASTER_FILENAME.test(region.name) + ) { + continue + } const parsed = parseLegacyMapFilename(region.name) const regionName = parsed ? parsed.resourceId : region.name.replace('.pmtiles', '') diff --git a/admin/tests/unit/map_style_generation.spec.ts b/admin/tests/unit/map_style_generation.spec.ts index 0b3573bc..d044ea3c 100644 --- a/admin/tests/unit/map_style_generation.spec.ts +++ b/admin/tests/unit/map_style_generation.spec.ts @@ -5,6 +5,7 @@ import { buildMapSourceDefinitions, composeMapStyle, parseMapArchiveManifest, + parseMapArchiveManifestJson, } from '../../app/utils/map_styles.js' import type { BaseStylesFile, MapArchiveMetadata, MapSourceDefinition } from '../../types/maps.js' import type { FileEntry } from '../../types/files.js' @@ -64,6 +65,7 @@ test('uses explicit vector archive metadata instead of inferring the source type 'http://nomad.test/maps/pmtiles', [file(metadata.filename)], { + status: 'valid', archives: [metadata], rejectedFilenames: [], } @@ -76,6 +78,31 @@ test('uses explicit vector archive metadata instead of inferring the source type assert.equal(sources[0].source.maxzoom, 15) }) +test('rejects vector metadata with raster roles or tile formats', () => { + const baseVectorMetadata = { + filename: 'us_20260728_z15.pmtiles', + resourceId: 'street-us', + kind: 'vector', + role: 'street', + tileFormat: 'mvt', + } + + for (const metadata of [ + { ...baseVectorMetadata, role: 'aerial' }, + { ...baseVectorMetadata, tileFormat: 'jpeg' }, + ]) { + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: { + [metadata.filename]: metadata, + }, + }) + + assert.deepEqual(parsed.archives, []) + assert.deepEqual(parsed.rejectedFilenames, [metadata.filename]) + } +}) + test('parses a filename-keyed manifest for an arbitrary region ID', () => { const metadata = rasterMetadata('us-ok') const parsed = parseMapArchiveManifest({ @@ -89,6 +116,16 @@ test('parses a filename-keyed manifest for an arbitrary region ID', () => { assert.deepEqual(parsed.rejectedFilenames, []) }) +test('rejects archive arrays because the manifest schema is filename-keyed', () => { + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: [rasterMetadata('us-ok')], + }) + + assert.equal(parsed.status, 'invalid') + assert.deepEqual(parsed.archives, []) +}) + test('rejects invalid raster metadata and blocks legacy-vector fallback for that file', () => { const metadata = { ...rasterMetadata('us-ok'), @@ -114,6 +151,18 @@ test('rejects invalid raster metadata and blocks legacy-vector fallback for that ) }) +test('blocks arbitrary PMTiles filenames from vector fallback when the manifest is malformed', () => { + const parsed = parseMapArchiveManifestJson('{not valid json') + const sources = buildMapSourceDefinitions( + 'http://nomad.test/maps/pmtiles', + [file('provider-aerial-ok.pmtiles'), file('oklahoma.pmtiles')], + parsed + ) + + assert.equal(parsed.status, 'invalid') + assert.deepEqual(sources, []) +}) + test('requires complete provenance and verification metadata for raster archives', () => { const requiredFields = [ 'role', @@ -152,7 +201,7 @@ test('creates non-colliding raster sources for multiple regions and roles', () = const sources = buildMapSourceDefinitions( 'http://nomad.test/maps/pmtiles', metadata.map((archive) => file(archive.filename)), - { archives: metadata, rejectedFilenames: [] } + { status: 'valid', archives: metadata, rejectedFilenames: [] } ) assert.deepEqual( @@ -181,7 +230,7 @@ test('places hidden raster layers below cloned vector layers without mutating th const sources: MapSourceDefinition[] = buildMapSourceDefinitions( 'http://nomad.test/maps/pmtiles', [file(vector.filename), file(raster.filename)], - { archives: [vector, raster], rejectedFilenames: [] } + { status: 'valid', archives: [vector, raster], rejectedFilenames: [] } ) const template = vectorTemplate() const original = structuredClone(template) diff --git a/admin/tests/unit/raster_pmtiles_conversion.spec.ts b/admin/tests/unit/raster_pmtiles_conversion.spec.ts new file mode 100644 index 00000000..25789b25 --- /dev/null +++ b/admin/tests/unit/raster_pmtiles_conversion.spec.ts @@ -0,0 +1,139 @@ +import * as assert from 'node:assert/strict' +import { copyFile, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, test } from 'node:test' + +import { convertRasterMbtilesToPmtiles } from '../../app/services/raster_pmtiles_converter.js' +import { parseMapArchiveManifest } from '../../app/utils/map_styles.js' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true })) + ) +}) + +test('converts a provider-neutral raster MBTiles archive into verified manifest metadata', async () => { + const directory = await mkdtemp(join(tmpdir(), 'nomad-raster-conversion-')) + temporaryDirectories.push(directory) + const sourcePath = join(directory, 'provider-output.mbtiles') + await writeFile(sourcePath, 'mbtiles input') + + const result = await convertRasterMbtilesToPmtiles( + { + sourceMbtilesPath: sourcePath, + outputDirectory: directory, + filename: 'imagery-ok.pmtiles', + resourceId: 'aerial-us-ok', + role: 'aerial', + regionId: 'us-ok', + tileSize: 256, + attribution: 'USDA, USGS The National Map: Orthoimagery.', + sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer', + sourceVersion: '3.3.0', + }, + { + now: () => new Date('2026-07-30T12:00:00.000Z'), + runPmtiles: async (args) => { + if (args[0] === 'convert') { + await copyFile(args[1], args[2]) + await writeFile(args[2], 'pmtiles bytes') + return { stdout: '', stderr: '' } + } + if (args[0] === 'verify') return { stdout: '', stderr: '' } + if (args[0] === 'show') { + return { + stdout: JSON.stringify({ + tile_compression: 'none', + tile_type: 'jpg', + minzoom: 0, + maxzoom: 14, + bounds: [-103, 33, -94, 37], + }), + stderr: '', + } + } + throw new Error(`Unexpected pmtiles command: ${args[0]}`) + }, + } + ) + + assert.equal(result.outputPath, join(directory, 'imagery-ok.pmtiles')) + assert.equal(await readFile(result.outputPath, 'utf8'), 'pmtiles bytes') + assert.deepEqual(result.archive, { + filename: 'imagery-ok.pmtiles', + resourceId: 'aerial-us-ok', + kind: 'raster', + role: 'aerial', + regionId: 'us-ok', + tileFormat: 'jpeg', + tileSize: 256, + minzoom: 0, + maxzoom: 14, + bounds: [-103, 33, -94, 37], + attribution: 'USDA, USGS The National Map: Orthoimagery.', + sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer', + sourceVersion: '3.3.0', + sha256: 'fca0b5920b5ead8723f66020d314eb37ebda06c9e5cb92acf186afa2fc67e789', + installedBytes: 13, + verifiedAt: '2026-07-30T12:00:00.000Z', + }) + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: { [result.archive.filename]: result.archive }, + }) + assert.equal(parsed.status, 'valid') + assert.deepEqual(parsed.archives, [result.archive]) +}) + +test('does not overwrite an archive installed concurrently during conversion', async () => { + const directory = await mkdtemp(join(tmpdir(), 'nomad-raster-conversion-')) + temporaryDirectories.push(directory) + const sourcePath = join(directory, 'provider-output.mbtiles') + const outputPath = join(directory, 'imagery-ok.pmtiles') + await writeFile(sourcePath, 'mbtiles input') + + await assert.rejects( + convertRasterMbtilesToPmtiles( + { + sourceMbtilesPath: sourcePath, + outputDirectory: directory, + filename: 'imagery-ok.pmtiles', + resourceId: 'aerial-us-ok', + role: 'aerial', + regionId: 'us-ok', + tileSize: 256, + attribution: 'USDA, USGS The National Map: Orthoimagery.', + sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer', + sourceVersion: '3.3.0', + }, + { + runPmtiles: async (args) => { + if (args[0] === 'convert') { + await writeFile(args[2], 'converted archive') + return { stdout: '', stderr: '' } + } + if (args[0] === 'verify') return { stdout: '', stderr: '' } + if (args[0] === 'show') { + await writeFile(outputPath, 'concurrent archive') + return { + stdout: JSON.stringify({ + tile_type: 'png', + minzoom: 0, + maxzoom: 14, + bounds: [-103, 33, -94, 37], + }), + stderr: '', + } + } + throw new Error(`Unexpected pmtiles command: ${args[0]}`) + }, + } + ), + /already exists/ + ) + + assert.equal(await readFile(outputPath, 'utf8'), 'concurrent archive') +}) diff --git a/admin/types/maps.ts b/admin/types/maps.ts index 231789b1..c3db34e8 100644 --- a/admin/types/maps.ts +++ b/admin/types/maps.ts @@ -30,20 +30,10 @@ export type MapArchiveKind = Extract export type MapArchiveRole = 'street' | 'aerial' | 'topographic' export type MapTileFormat = 'mvt' | 'jpeg' | 'png' | 'webp' -/** - * Explicit sidecar metadata for an installed PMTiles archive. - * - * Existing archives without an entry remain supported as legacy vector maps. - * Raster archives must have a valid entry before style generation will activate - * them. - */ -export type MapArchiveMetadata = { +type MapArchiveMetadataBase = { filename: string resourceId: string - kind: MapArchiveKind - role?: MapArchiveRole regionId?: string - tileFormat?: MapTileFormat tileSize?: number minzoom?: number maxzoom?: number @@ -56,6 +46,38 @@ export type MapArchiveMetadata = { verifiedAt?: string } +export type VectorMapArchiveMetadata = MapArchiveMetadataBase & { + kind: 'vector' + role: 'street' + tileFormat: 'mvt' +} + +export type RasterMapArchiveMetadata = MapArchiveMetadataBase & { + kind: 'raster' + role: Extract + regionId: string + tileFormat: Extract + tileSize: number + minzoom: number + maxzoom: number + bounds: [number, number, number, number] + attribution: string + sourceUrl: string + sourceVersion: string + sha256: string + installedBytes: number + verifiedAt: string +} + +/** + * Explicit sidecar metadata for an installed PMTiles archive. + * + * Existing archives without an entry remain supported as legacy vector maps. + * Raster archives must have a valid entry before style generation will activate + * them. + */ +export type MapArchiveMetadata = VectorMapArchiveMetadata | RasterMapArchiveMetadata + export type MapArchiveManifest = { schemaVersion: 1 generatedAt?: string @@ -63,6 +85,7 @@ export type MapArchiveManifest = { } export type ParsedMapArchiveManifest = { + status: 'absent' | 'valid' | 'invalid' archives: MapArchiveMetadata[] /** * Files explicitly mentioned by a manifest but rejected during validation.