feat(maps): complete raster PMTiles foundation
Some checks failed
Validate Collection URLs / validate-urls (push) Has been cancelled

This commit is contained in:
Gabe Graves 2026-07-30 19:25:29 -04:00 committed by Chris Sherwood
parent fa6e0c56dd
commit 587a8e6c4d
6 changed files with 489 additions and 32 deletions

View File

@ -44,7 +44,7 @@ import {
composeMapStyle,
DEFAULT_VECTOR_MAP_ATTRIBUTION,
MAP_ARCHIVE_MANIFEST_FILENAME,
parseMapArchiveManifest,
parseMapArchiveManifestJson,
} from '../utils/map_styles.js'
const execFileAsync = promisify(execFile)
@ -413,18 +413,15 @@ export class MapService implements IMapService {
}
const manifest = await getFile(manifestPath, 'string')
if (!manifest) return { archives: [], rejectedFilenames: [] }
if (!manifest) return { status: 'absent', archives: [], rejectedFilenames: [] }
try {
return parseMapArchiveManifest(JSON.parse(manifest.toString()))
} catch (error) {
const parsed = parseMapArchiveManifestJson(manifest.toString())
if (parsed.status === 'invalid') {
logger.warn(
`[MapService] Ignoring malformed ${MAP_ARCHIVE_MANIFEST_FILENAME}: ${
error instanceof Error ? error.message : String(error)
}`
`[MapService] ${MAP_ARCHIVE_MANIFEST_FILENAME} is malformed; blocking unmanifested PMTiles archives`
)
return { archives: [], rejectedFilenames: [] }
}
return parsed
}
async listCuratedCollections(): Promise<CollectionWithStatus[]> {

View File

@ -0,0 +1,220 @@
import { execFile } from 'node:child_process'
import { createHash, randomBytes } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { access, link, mkdir, rm, stat } from 'node:fs/promises'
import { basename, extname, join, resolve, sep } from 'node:path'
import { promisify } from 'node:util'
import { PMTILES_BINARY_PATH } from '../../constants/map_regions.js'
import type { RasterMapArchiveMetadata } from '../../types/maps.js'
const execFileAsync = promisify(execFile)
const SAFE_ARCHIVE_FILENAME = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,199}\.pmtiles$/
const SAFE_RESOURCE_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/
const SAFE_REGION_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/
const HTTPS_URL = /^https:\/\//i
const TILE_FORMATS: Readonly<Record<string, RasterMapArchiveMetadata['tileFormat']>> = {
jpg: 'jpeg',
jpeg: 'jpeg',
png: 'png',
webp: 'webp',
}
export type RasterPmtilesConversionRequest = Pick<
RasterMapArchiveMetadata,
| 'filename'
| 'resourceId'
| 'role'
| 'regionId'
| 'tileSize'
| 'attribution'
| 'sourceUrl'
| 'sourceVersion'
> & {
sourceMbtilesPath: string
outputDirectory: string
}
export type RasterPmtilesCommandResult = {
stdout: string
stderr: string
}
export type RasterPmtilesConversionDependencies = {
runPmtiles?: (args: string[]) => Promise<RasterPmtilesCommandResult>
now?: () => Date
}
export type RasterPmtilesConversionResult = {
outputPath: string
archive: RasterMapArchiveMetadata
}
type PmtilesHeader = Pick<RasterMapArchiveMetadata, 'tileFormat' | 'minzoom' | 'maxzoom' | 'bounds'>
async function defaultPmtilesRunner(args: string[]): Promise<RasterPmtilesCommandResult> {
const result = await execFileAsync(PMTILES_BINARY_PATH, args, {
maxBuffer: 1024 * 1024,
})
return {
stdout: result.stdout.toString(),
stderr: result.stderr.toString(),
}
}
function parseRasterHeader(value: string): PmtilesHeader {
let header: unknown
try {
header = JSON.parse(value)
} catch {
throw new Error('pmtiles show returned malformed header JSON')
}
if (!header || typeof header !== 'object') {
throw new Error('pmtiles show returned an invalid header')
}
const candidate = header as Record<string, unknown>
const tileFormat =
typeof candidate.tile_type === 'string' ? TILE_FORMATS[candidate.tile_type] : undefined
const bounds = candidate.bounds
const minzoom = candidate.minzoom
const maxzoom = candidate.maxzoom
if (!tileFormat) throw new Error('Converted PMTiles archive is not a supported raster tile set')
if (
!Number.isInteger(minzoom) ||
!Number.isInteger(maxzoom) ||
(minzoom as number) < 0 ||
(maxzoom as number) > 30 ||
(minzoom as number) > (maxzoom as number)
) {
throw new Error('Converted PMTiles archive has invalid zoom metadata')
}
if (
!Array.isArray(bounds) ||
bounds.length !== 4 ||
!bounds.every((coordinate) => typeof coordinate === 'number' && Number.isFinite(coordinate))
) {
throw new Error('Converted PMTiles archive has invalid bounds metadata')
}
const [west, south, east, north] = bounds as number[]
if (
west < -180 ||
west > 180 ||
east < -180 ||
east > 180 ||
south < -90 ||
south > 90 ||
north < -90 ||
north > 90 ||
west >= east ||
south >= north
) {
throw new Error('Converted PMTiles archive has invalid bounds metadata')
}
return {
tileFormat,
minzoom: minzoom as number,
maxzoom: maxzoom as number,
bounds: [west, south, east, north],
}
}
async function sha256File(filepath: string): Promise<string> {
const hash = createHash('sha256')
for await (const chunk of createReadStream(filepath)) hash.update(chunk)
return hash.digest('hex')
}
function validateRequest(request: RasterPmtilesConversionRequest): void {
if (
!SAFE_ARCHIVE_FILENAME.test(request.filename) ||
basename(request.filename) !== request.filename
) {
throw new Error('Invalid PMTiles output filename')
}
if (extname(request.sourceMbtilesPath).toLowerCase() !== '.mbtiles') {
throw new Error('Raster conversion input must be an MBTiles archive')
}
if (!SAFE_RESOURCE_ID.test(request.resourceId)) throw new Error('Invalid map resource ID')
if (!SAFE_REGION_ID.test(request.regionId)) throw new Error('Invalid map region ID')
if (request.role !== 'aerial' && request.role !== 'topographic') {
throw new Error('Invalid raster map role')
}
if (![128, 256, 512, 1024].includes(request.tileSize)) {
throw new Error('Invalid raster tile size')
}
if (!request.attribution.trim()) throw new Error('Raster attribution is required')
if (!HTTPS_URL.test(request.sourceUrl)) throw new Error('Raster source URL must use HTTPS')
if (!request.sourceVersion.trim()) throw new Error('Raster source version is required')
}
/**
* Convert a provider-produced raster MBTiles archive into a verified PMTiles
* archive. The converter owns format inspection, checksum generation, and
* atomic activation; provider adapters only supply provenance and staging paths.
*/
export async function convertRasterMbtilesToPmtiles(
request: RasterPmtilesConversionRequest,
dependencies: RasterPmtilesConversionDependencies = {}
): Promise<RasterPmtilesConversionResult> {
validateRequest(request)
const outputDirectory = resolve(request.outputDirectory)
const outputPath = resolve(join(outputDirectory, request.filename))
if (!outputPath.startsWith(outputDirectory + sep)) throw new Error('Invalid PMTiles output path')
await mkdir(outputDirectory, { recursive: true })
try {
await access(outputPath)
throw new Error(`PMTiles output already exists: ${request.filename}`)
} catch (error) {
if (error instanceof Error && !('code' in error && error.code === 'ENOENT')) throw error
}
const stagingPath = join(
outputDirectory,
`.${request.filename}.${randomBytes(8).toString('hex')}.partial`
)
const runPmtiles = dependencies.runPmtiles ?? defaultPmtilesRunner
const now = dependencies.now ?? (() => new Date())
try {
await runPmtiles(['convert', resolve(request.sourceMbtilesPath), stagingPath])
await runPmtiles(['verify', stagingPath])
const inspected = await runPmtiles(['show', stagingPath, '--header-json'])
const header = parseRasterHeader(inspected.stdout)
const [sha256, outputStats] = await Promise.all([sha256File(stagingPath), stat(stagingPath)])
const archive: RasterMapArchiveMetadata = {
filename: request.filename,
resourceId: request.resourceId,
kind: 'raster',
role: request.role,
regionId: request.regionId,
tileFormat: header.tileFormat,
tileSize: request.tileSize,
minzoom: header.minzoom,
maxzoom: header.maxzoom,
bounds: header.bounds,
attribution: request.attribution,
sourceUrl: request.sourceUrl,
sourceVersion: request.sourceVersion,
sha256,
installedBytes: outputStats.size,
verifiedAt: now().toISOString(),
}
try {
await link(stagingPath, outputPath)
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'EEXIST') {
throw new Error(`PMTiles output already exists: ${request.filename}`)
}
throw error
}
await rm(stagingPath)
return { outputPath, archive }
} catch (error) {
await rm(stagingPath, { force: true })
throw error
}
}

View File

@ -136,6 +136,10 @@ function isMapArchiveMetadata(value: unknown): value is MapArchiveMetadata {
}
}
if (archive.kind === 'vector' && (archive.role !== 'street' || archive.tileFormat !== 'mvt')) {
return false
}
if (archive.kind === 'raster') {
if (
(archive.role !== 'aerial' && archive.role !== 'topographic') ||
@ -171,16 +175,23 @@ function candidateFilename(value: unknown): string | null {
* their filenames remain blocked from legacy-vector fallback.
*/
export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifest {
const empty = { archives: [], rejectedFilenames: [] }
if (!value || typeof value !== 'object') return empty
const invalid: ParsedMapArchiveManifest = {
status: 'invalid',
archives: [],
rejectedFilenames: [],
}
if (!value || typeof value !== 'object') return invalid
const manifest = value as { schemaVersion?: unknown; archives?: unknown }
if (manifest.schemaVersion !== 1 || !manifest.archives || typeof manifest.archives !== 'object') {
return empty
if (
manifest.schemaVersion !== 1 ||
!manifest.archives ||
typeof manifest.archives !== 'object' ||
Array.isArray(manifest.archives)
) {
return invalid
}
const entries = Array.isArray(manifest.archives)
? manifest.archives.map((candidate) => [candidateFilename(candidate), candidate] as const)
: Object.entries(manifest.archives)
const entries = Object.entries(manifest.archives)
const archives: MapArchiveMetadata[] = []
const rejectedFilenames = new Set<string>()
const filenames = new Set<string>()
@ -210,7 +221,15 @@ export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifes
archives.push({ ...candidate })
}
return { archives, rejectedFilenames: [...rejectedFilenames] }
return { status: 'valid', archives, rejectedFilenames: [...rejectedFilenames] }
}
export function parseMapArchiveManifestJson(value: string): ParsedMapArchiveManifest {
try {
return parseMapArchiveManifest(JSON.parse(value))
} catch {
return { status: 'invalid', archives: [], rejectedFilenames: [] }
}
}
function compareMapArchiveVersions(a: string | null, b: string | null): number {
@ -249,7 +268,11 @@ export function sourceFromArchive(baseUrl: string, archive: MapArchiveMetadata):
export function buildMapSourceDefinitions(
baseUrl: string,
regions: FileEntry[],
parsedManifest: ParsedMapArchiveManifest = { archives: [], rejectedFilenames: [] }
parsedManifest: ParsedMapArchiveManifest = {
status: 'absent',
archives: [],
rejectedFilenames: [],
}
): MapSourceDefinition[] {
const metadataByFilename = new Map(
parsedManifest.archives.map((archive) => [archive.filename, archive])
@ -275,7 +298,13 @@ export function buildMapSourceDefinitions(
})
continue
}
if (rejectedFilenames.has(region.name) || RESERVED_RASTER_FILENAME.test(region.name)) continue
if (
parsedManifest.status === 'invalid' ||
rejectedFilenames.has(region.name) ||
RESERVED_RASTER_FILENAME.test(region.name)
) {
continue
}
const parsed = parseLegacyMapFilename(region.name)
const regionName = parsed ? parsed.resourceId : region.name.replace('.pmtiles', '')

View File

@ -5,6 +5,7 @@ import {
buildMapSourceDefinitions,
composeMapStyle,
parseMapArchiveManifest,
parseMapArchiveManifestJson,
} from '../../app/utils/map_styles.js'
import type { BaseStylesFile, MapArchiveMetadata, MapSourceDefinition } from '../../types/maps.js'
import type { FileEntry } from '../../types/files.js'
@ -64,6 +65,7 @@ test('uses explicit vector archive metadata instead of inferring the source type
'http://nomad.test/maps/pmtiles',
[file(metadata.filename)],
{
status: 'valid',
archives: [metadata],
rejectedFilenames: [],
}
@ -76,6 +78,31 @@ test('uses explicit vector archive metadata instead of inferring the source type
assert.equal(sources[0].source.maxzoom, 15)
})
test('rejects vector metadata with raster roles or tile formats', () => {
const baseVectorMetadata = {
filename: 'us_20260728_z15.pmtiles',
resourceId: 'street-us',
kind: 'vector',
role: 'street',
tileFormat: 'mvt',
}
for (const metadata of [
{ ...baseVectorMetadata, role: 'aerial' },
{ ...baseVectorMetadata, tileFormat: 'jpeg' },
]) {
const parsed = parseMapArchiveManifest({
schemaVersion: 1,
archives: {
[metadata.filename]: metadata,
},
})
assert.deepEqual(parsed.archives, [])
assert.deepEqual(parsed.rejectedFilenames, [metadata.filename])
}
})
test('parses a filename-keyed manifest for an arbitrary region ID', () => {
const metadata = rasterMetadata('us-ok')
const parsed = parseMapArchiveManifest({
@ -89,6 +116,16 @@ test('parses a filename-keyed manifest for an arbitrary region ID', () => {
assert.deepEqual(parsed.rejectedFilenames, [])
})
test('rejects archive arrays because the manifest schema is filename-keyed', () => {
const parsed = parseMapArchiveManifest({
schemaVersion: 1,
archives: [rasterMetadata('us-ok')],
})
assert.equal(parsed.status, 'invalid')
assert.deepEqual(parsed.archives, [])
})
test('rejects invalid raster metadata and blocks legacy-vector fallback for that file', () => {
const metadata = {
...rasterMetadata('us-ok'),
@ -114,6 +151,18 @@ test('rejects invalid raster metadata and blocks legacy-vector fallback for that
)
})
test('blocks arbitrary PMTiles filenames from vector fallback when the manifest is malformed', () => {
const parsed = parseMapArchiveManifestJson('{not valid json')
const sources = buildMapSourceDefinitions(
'http://nomad.test/maps/pmtiles',
[file('provider-aerial-ok.pmtiles'), file('oklahoma.pmtiles')],
parsed
)
assert.equal(parsed.status, 'invalid')
assert.deepEqual(sources, [])
})
test('requires complete provenance and verification metadata for raster archives', () => {
const requiredFields = [
'role',
@ -152,7 +201,7 @@ test('creates non-colliding raster sources for multiple regions and roles', () =
const sources = buildMapSourceDefinitions(
'http://nomad.test/maps/pmtiles',
metadata.map((archive) => file(archive.filename)),
{ archives: metadata, rejectedFilenames: [] }
{ status: 'valid', archives: metadata, rejectedFilenames: [] }
)
assert.deepEqual(
@ -181,7 +230,7 @@ test('places hidden raster layers below cloned vector layers without mutating th
const sources: MapSourceDefinition[] = buildMapSourceDefinitions(
'http://nomad.test/maps/pmtiles',
[file(vector.filename), file(raster.filename)],
{ archives: [vector, raster], rejectedFilenames: [] }
{ status: 'valid', archives: [vector, raster], rejectedFilenames: [] }
)
const template = vectorTemplate()
const original = structuredClone(template)

View File

@ -0,0 +1,139 @@
import * as assert from 'node:assert/strict'
import { copyFile, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, test } from 'node:test'
import { convertRasterMbtilesToPmtiles } from '../../app/services/raster_pmtiles_converter.js'
import { parseMapArchiveManifest } from '../../app/utils/map_styles.js'
const temporaryDirectories: string[] = []
afterEach(async () => {
await Promise.all(
temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true }))
)
})
test('converts a provider-neutral raster MBTiles archive into verified manifest metadata', async () => {
const directory = await mkdtemp(join(tmpdir(), 'nomad-raster-conversion-'))
temporaryDirectories.push(directory)
const sourcePath = join(directory, 'provider-output.mbtiles')
await writeFile(sourcePath, 'mbtiles input')
const result = await convertRasterMbtilesToPmtiles(
{
sourceMbtilesPath: sourcePath,
outputDirectory: directory,
filename: 'imagery-ok.pmtiles',
resourceId: 'aerial-us-ok',
role: 'aerial',
regionId: 'us-ok',
tileSize: 256,
attribution: 'USDA, USGS The National Map: Orthoimagery.',
sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer',
sourceVersion: '3.3.0',
},
{
now: () => new Date('2026-07-30T12:00:00.000Z'),
runPmtiles: async (args) => {
if (args[0] === 'convert') {
await copyFile(args[1], args[2])
await writeFile(args[2], 'pmtiles bytes')
return { stdout: '', stderr: '' }
}
if (args[0] === 'verify') return { stdout: '', stderr: '' }
if (args[0] === 'show') {
return {
stdout: JSON.stringify({
tile_compression: 'none',
tile_type: 'jpg',
minzoom: 0,
maxzoom: 14,
bounds: [-103, 33, -94, 37],
}),
stderr: '',
}
}
throw new Error(`Unexpected pmtiles command: ${args[0]}`)
},
}
)
assert.equal(result.outputPath, join(directory, 'imagery-ok.pmtiles'))
assert.equal(await readFile(result.outputPath, 'utf8'), 'pmtiles bytes')
assert.deepEqual(result.archive, {
filename: 'imagery-ok.pmtiles',
resourceId: 'aerial-us-ok',
kind: 'raster',
role: 'aerial',
regionId: 'us-ok',
tileFormat: 'jpeg',
tileSize: 256,
minzoom: 0,
maxzoom: 14,
bounds: [-103, 33, -94, 37],
attribution: 'USDA, USGS The National Map: Orthoimagery.',
sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer',
sourceVersion: '3.3.0',
sha256: 'fca0b5920b5ead8723f66020d314eb37ebda06c9e5cb92acf186afa2fc67e789',
installedBytes: 13,
verifiedAt: '2026-07-30T12:00:00.000Z',
})
const parsed = parseMapArchiveManifest({
schemaVersion: 1,
archives: { [result.archive.filename]: result.archive },
})
assert.equal(parsed.status, 'valid')
assert.deepEqual(parsed.archives, [result.archive])
})
test('does not overwrite an archive installed concurrently during conversion', async () => {
const directory = await mkdtemp(join(tmpdir(), 'nomad-raster-conversion-'))
temporaryDirectories.push(directory)
const sourcePath = join(directory, 'provider-output.mbtiles')
const outputPath = join(directory, 'imagery-ok.pmtiles')
await writeFile(sourcePath, 'mbtiles input')
await assert.rejects(
convertRasterMbtilesToPmtiles(
{
sourceMbtilesPath: sourcePath,
outputDirectory: directory,
filename: 'imagery-ok.pmtiles',
resourceId: 'aerial-us-ok',
role: 'aerial',
regionId: 'us-ok',
tileSize: 256,
attribution: 'USDA, USGS The National Map: Orthoimagery.',
sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer',
sourceVersion: '3.3.0',
},
{
runPmtiles: async (args) => {
if (args[0] === 'convert') {
await writeFile(args[2], 'converted archive')
return { stdout: '', stderr: '' }
}
if (args[0] === 'verify') return { stdout: '', stderr: '' }
if (args[0] === 'show') {
await writeFile(outputPath, 'concurrent archive')
return {
stdout: JSON.stringify({
tile_type: 'png',
minzoom: 0,
maxzoom: 14,
bounds: [-103, 33, -94, 37],
}),
stderr: '',
}
}
throw new Error(`Unexpected pmtiles command: ${args[0]}`)
},
}
),
/already exists/
)
assert.equal(await readFile(outputPath, 'utf8'), 'concurrent archive')
})

View File

@ -30,20 +30,10 @@ export type MapArchiveKind = Extract<MapSource['type'], 'vector' | 'raster'>
export type MapArchiveRole = 'street' | 'aerial' | 'topographic'
export type MapTileFormat = 'mvt' | 'jpeg' | 'png' | 'webp'
/**
* Explicit sidecar metadata for an installed PMTiles archive.
*
* Existing archives without an entry remain supported as legacy vector maps.
* Raster archives must have a valid entry before style generation will activate
* them.
*/
export type MapArchiveMetadata = {
type MapArchiveMetadataBase = {
filename: string
resourceId: string
kind: MapArchiveKind
role?: MapArchiveRole
regionId?: string
tileFormat?: MapTileFormat
tileSize?: number
minzoom?: number
maxzoom?: number
@ -56,6 +46,38 @@ export type MapArchiveMetadata = {
verifiedAt?: string
}
export type VectorMapArchiveMetadata = MapArchiveMetadataBase & {
kind: 'vector'
role: 'street'
tileFormat: 'mvt'
}
export type RasterMapArchiveMetadata = MapArchiveMetadataBase & {
kind: 'raster'
role: Extract<MapArchiveRole, 'aerial' | 'topographic'>
regionId: string
tileFormat: Extract<MapTileFormat, 'jpeg' | 'png' | 'webp'>
tileSize: number
minzoom: number
maxzoom: number
bounds: [number, number, number, number]
attribution: string
sourceUrl: string
sourceVersion: string
sha256: string
installedBytes: number
verifiedAt: string
}
/**
* Explicit sidecar metadata for an installed PMTiles archive.
*
* Existing archives without an entry remain supported as legacy vector maps.
* Raster archives must have a valid entry before style generation will activate
* them.
*/
export type MapArchiveMetadata = VectorMapArchiveMetadata | RasterMapArchiveMetadata
export type MapArchiveManifest = {
schemaVersion: 1
generatedAt?: string
@ -63,6 +85,7 @@ export type MapArchiveManifest = {
}
export type ParsedMapArchiveManifest = {
status: 'absent' | 'valid' | 'invalid'
archives: MapArchiveMetadata[]
/**
* Files explicitly mentioned by a manifest but rejected during validation.