From fa6e0c56dd87f00adab1cdf5d0b74bc7cdf6253b Mon Sep 17 00:00:00 2001 From: Gabe Graves <37971265+gabegraves@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:29:37 -0400 Subject: [PATCH] feat(maps): support raster PMTiles manifests --- admin/app/services/map_service.ts | 164 +++----- admin/app/utils/map_styles.ts | 364 ++++++++++++++++++ admin/tests/unit/map_style_generation.spec.ts | 242 ++++++++++++ admin/types/maps.ts | 57 +++ 4 files changed, 717 insertions(+), 110 deletions(-) create mode 100644 admin/app/utils/map_styles.ts create mode 100644 admin/tests/unit/map_style_generation.spec.ts diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index dae68a2f..7b65a3bd 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -1,4 +1,4 @@ -import { BaseStylesFile, MapLayer } from '../../types/maps.js' +import { BaseStylesFile, MapSourceDefinition, ParsedMapArchiveManifest } from '../../types/maps.js' import { DownloadRemoteSuccessCallback, FileEntry, @@ -39,6 +39,13 @@ import { execFile } from 'child_process' import { createHash, randomBytes } from 'crypto' import { tmpdir } from 'os' import { promisify } from 'util' +import { + buildMapSourceDefinitions, + composeMapStyle, + DEFAULT_VECTOR_MAP_ATTRIBUTION, + MAP_ARCHIVE_MANIFEST_FILENAME, + parseMapArchiveManifest, +} from '../utils/map_styles.js' const execFileAsync = promisify(execFile) const DRY_RUN_TIMEOUT_MS = 60_000 @@ -63,8 +70,6 @@ const BASE_ASSETS_MIME_TYPES = [ 'application/octet-stream', ] -const PMTILES_ATTRIBUTION = - 'Protomaps © OpenStreetMap' const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream'] interface IMapService { @@ -358,6 +363,7 @@ export class MapService implements IMapService { const rawStyles = JSON.parse(baseStyle.toString()) as BaseStylesFile const regions = (await this.listRegions()).files + const archiveManifest = await this.loadMapArchiveManifest() /** If we have the host, use it to build public URLs, otherwise we'll fallback to defaults * This is mainly useful because we need to know what host the user is accessing from in order to @@ -365,10 +371,31 @@ export class MapService implements IMapService { * e.g. user is accessing from "example.com", but we would by default generate "localhost:8080/..." so maps would * fail to load. */ - const sources = this.generateSourcesArray(host, regions, protocol) + const pmtilesBaseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol) + const sources = buildMapSourceDefinitions(pmtilesBaseUrl, regions, archiveManifest) + if (this.worldBasemapReady) { + const worldSource: MapSourceDefinition = { + id: WORLD_BASEMAP_SOURCE_NAME, + source: { + type: 'vector', + attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION, + url: `pmtiles://${urlJoin(pmtilesBaseUrl, WORLD_BASEMAP_FILENAME)}`, + }, + archive: { + filename: WORLD_BASEMAP_FILENAME, + resourceId: WORLD_BASEMAP_SOURCE_NAME, + kind: 'vector', + role: 'street', + tileFormat: 'mvt', + attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION, + maxzoom: WORLD_BASEMAP_MAX_ZOOM, + }, + } + sources.unshift(worldSource) + } const baseUrl = this.getPublicFileBaseUrl(host, this.basemapsAssetsDir, protocol) - const styles = await this.generateStylesFile( + const styles = composeMapStyle( rawStyles, sources, urlJoin(baseUrl, 'sprites/v4/light'), @@ -378,6 +405,28 @@ export class MapService implements IMapService { return styles } + private async loadMapArchiveManifest(): Promise { + const manifestPath = resolve(join(this.baseDirPath, 'pmtiles', MAP_ARCHIVE_MANIFEST_FILENAME)) + const pmtilesDirectory = resolve(join(this.baseDirPath, 'pmtiles')) + if (!manifestPath.startsWith(pmtilesDirectory + sep)) { + throw new Error('Invalid map archive manifest path') + } + + const manifest = await getFile(manifestPath, 'string') + if (!manifest) return { archives: [], rejectedFilenames: [] } + + try { + return parseMapArchiveManifest(JSON.parse(manifest.toString())) + } catch (error) { + logger.warn( + `[MapService] Ignoring malformed ${MAP_ARCHIVE_MANIFEST_FILENAME}: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return { archives: [], rejectedFilenames: [] } + } + } + async listCuratedCollections(): Promise { const manifestService = new CollectionManifestService() return manifestService.getMapCollectionsWithStatus() @@ -523,111 +572,6 @@ export class MapService implements IMapService { return await listDirectoryContentsRecursive(this.baseDirPath) } - /** - * Compare two map-file versions (YYYY-MM, or null for an undated legacy file). Returns >0 if - * `a` is newer than `b`, <0 if older, 0 if equal. A dated build is always newer than an undated - * legacy file; two dated builds compare lexicographically (correct for zero-padded YYYY-MM). - */ - private static compareMapVersions(a: string | null, b: string | null): number { - if (a === b) return 0 - if (a === null) return -1 - if (b === null) return 1 - return a < b ? -1 : 1 - } - - private generateSourcesArray(host: string | null, regions: FileEntry[], protocol: string = 'http'): BaseStylesFile['sources'][] { - const sources: BaseStylesFile['sources'][] = [] - const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol) - - // World basemap goes first so its layers render underneath regional extracts. - // Only emitted when ensureWorldBasemap() succeeded — otherwise the style would - // reference a file that doesn't exist and produce 404s on every tile request. - if (this.worldBasemapReady) { - const worldSource: BaseStylesFile['sources'] = {} - worldSource[WORLD_BASEMAP_SOURCE_NAME] = { - type: 'vector', - attribution: PMTILES_ATTRIBUTION, - url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`, - } - sources.push(worldSource) - } - - // Dedupe by region name, keeping only the newest file per region. The source name is the - // date-stripped region (e.g. both "washington.pmtiles" and "washington_2025-12.pmtiles" map - // to "washington"). Emitting both produces duplicate source keys and duplicate layer ids, - // which MapLibre rejects outright — blanking the ENTIRE map, not just that region. Old copies - // linger when a newer curated version installs (#634), so guard against it here so the style - // stays valid even if cleanup hasn't run. A dated build beats an undated legacy file; between - // two dated builds the later YYYY-MM wins (lexicographic compare is correct for that format). - const bestByRegion = new Map() - for (const region of regions) { - if (region.type === 'file' && region.name.endsWith('.pmtiles')) { - const parsed = CollectionManifestService.parseMapFilename(region.name) - const regionName = parsed ? parsed.resource_id : region.name.replace('.pmtiles', '') - const version = parsed?.version ?? null - const existing = bestByRegion.get(regionName) - if (!existing || MapService.compareMapVersions(version, existing.version) > 0) { - if (existing) { - logger.warn( - `[MapService] Duplicate map region "${regionName}": using "${region.name}" over "${existing.region.name}" (keeping newest)` - ) - } - bestByRegion.set(regionName, { region, version }) - } else { - logger.warn( - `[MapService] Duplicate map region "${regionName}": skipping "${region.name}" in favor of "${existing.region.name}" (keeping newest)` - ) - } - } - } - - for (const [regionName, { region }] of bestByRegion) { - const source: BaseStylesFile['sources'] = {} - const sourceUrl = urlJoin(baseUrl, region.name) - - source[regionName] = { - type: 'vector', - attribution: PMTILES_ATTRIBUTION, - url: `pmtiles://${sourceUrl}`, - } - sources.push(source) - } - - return sources - } - - private async generateStylesFile( - template: BaseStylesFile, - sources: BaseStylesFile['sources'][], - sprites: string, - glyphs: string - ): Promise { - const layersTemplates = template.layers.filter((layer) => layer.source) - const withoutSources = template.layers.filter((layer) => !layer.source) - - template.sources = {} // Clear existing sources - template.layers = [...withoutSources] // Start with layers that don't depend on sources - - for (const source of sources) { - for (const layerTemplate of layersTemplates) { - const layer: MapLayer = { - ...layerTemplate, - id: `${layerTemplate.id}-${Object.keys(source)[0]}`, - type: layerTemplate.type, - source: Object.keys(source)[0], - } - template.layers.push(layer) - } - - template.sources = Object.assign(template.sources, source) - } - - template.sprite = sprites - template.glyphs = glyphs - - return template - } - async getGlobalMapInfo(): Promise { const { default: axios } = await import('axios') const response = await axios.get(PROTOMAPS_BUILDS_METADATA_URL, { timeout: 15000 }) diff --git a/admin/app/utils/map_styles.ts b/admin/app/utils/map_styles.ts new file mode 100644 index 00000000..d1dbd264 --- /dev/null +++ b/admin/app/utils/map_styles.ts @@ -0,0 +1,364 @@ +import urlJoin from 'url-join' + +import type { FileEntry } from '../../types/files.js' +import type { + BaseStylesFile, + MapArchiveMetadata, + MapLayer, + MapSource, + MapSourceDefinition, + ParsedMapArchiveManifest, +} from '../../types/maps.js' + +export const DEFAULT_VECTOR_MAP_ATTRIBUTION = + 'Protomaps © OpenStreetMap' +export const MAP_ARCHIVE_MANIFEST_FILENAME = 'map-archive-manifest.json' + +const MAP_ARCHIVE_KINDS = new Set(['vector', 'raster']) +const MAP_ARCHIVE_ROLES = new Set(['street', 'aerial', 'topographic']) +const MAP_TILE_FORMATS = new Set(['mvt', 'jpeg', 'png', 'webp']) +const RASTER_TILE_FORMATS = new Set(['jpeg', 'png', 'webp']) +const SAFE_RESOURCE_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/ +const SAFE_REGION_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/ +const SHA256_DIGEST = /^[a-f0-9]{64}$/i +const HTTPS_URL = /^https:\/\//i +const RESERVED_RASTER_FILENAME = /^raster[-_]/i + +function optionalNonNegativeNumber(value: unknown): value is number | undefined { + return value === undefined || (typeof value === 'number' && Number.isFinite(value) && value >= 0) +} + +function validZoom(value: unknown): value is number | undefined { + return ( + value === undefined || + (typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= 30) + ) +} + +function validBounds(value: unknown): value is [number, number, number, number] | undefined { + if (value === undefined) return true + if (!Array.isArray(value) || value.length !== 4) return false + if (!value.every((item) => typeof item === 'number' && Number.isFinite(item))) return false + const [west, south, east, north] = value + return ( + west >= -180 && + west <= 180 && + east >= -180 && + east <= 180 && + south >= -90 && + south <= 90 && + north >= -90 && + north <= 90 && + west < east && + south < north + ) +} + +function validTimestamp(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && Number.isFinite(Date.parse(value))) +} + +function isMapArchiveMetadata(value: unknown): value is MapArchiveMetadata { + if (!value || typeof value !== 'object') return false + const archive = value as Record + + if ( + typeof archive.filename !== 'string' || + !archive.filename.endsWith('.pmtiles') || + archive.filename.includes('/') || + archive.filename.includes('\\') || + typeof archive.resourceId !== 'string' || + !SAFE_RESOURCE_ID.test(archive.resourceId) || + typeof archive.kind !== 'string' || + !MAP_ARCHIVE_KINDS.has(archive.kind) + ) { + return false + } + + if ( + archive.role !== undefined && + (typeof archive.role !== 'string' || !MAP_ARCHIVE_ROLES.has(archive.role)) + ) { + return false + } + if ( + archive.regionId !== undefined && + (typeof archive.regionId !== 'string' || !SAFE_REGION_ID.test(archive.regionId)) + ) { + return false + } + if ( + archive.tileFormat !== undefined && + (typeof archive.tileFormat !== 'string' || !MAP_TILE_FORMATS.has(archive.tileFormat)) + ) { + return false + } + if ( + archive.tileSize !== undefined && + ![128, 256, 512, 1024].includes(archive.tileSize as number) + ) { + return false + } + if ( + !validZoom(archive.minzoom) || + !validZoom(archive.maxzoom) || + !optionalNonNegativeNumber(archive.installedBytes) || + !validBounds(archive.bounds) + ) { + return false + } + if ( + typeof archive.minzoom === 'number' && + typeof archive.maxzoom === 'number' && + archive.minzoom > archive.maxzoom + ) { + return false + } + if ( + archive.sha256 !== undefined && + (typeof archive.sha256 !== 'string' || !SHA256_DIGEST.test(archive.sha256)) + ) { + return false + } + if (!validTimestamp(archive.verifiedAt)) return false + if ( + archive.sourceUrl !== undefined && + (typeof archive.sourceUrl !== 'string' || !HTTPS_URL.test(archive.sourceUrl)) + ) { + return false + } + for (const field of ['attribution', 'sourceVersion'] as const) { + if ( + archive[field] !== undefined && + (typeof archive[field] !== 'string' || archive[field].trim().length === 0) + ) { + return false + } + } + + if (archive.kind === 'raster') { + if ( + (archive.role !== 'aerial' && archive.role !== 'topographic') || + typeof archive.regionId !== 'string' || + typeof archive.tileFormat !== 'string' || + !RASTER_TILE_FORMATS.has(archive.tileFormat) || + typeof archive.tileSize !== 'number' || + typeof archive.minzoom !== 'number' || + typeof archive.maxzoom !== 'number' || + !archive.bounds || + typeof archive.attribution !== 'string' || + typeof archive.sourceUrl !== 'string' || + typeof archive.sourceVersion !== 'string' || + typeof archive.sha256 !== 'string' || + typeof archive.installedBytes !== 'number' || + typeof archive.verifiedAt !== 'string' + ) { + return false + } + } + + return true +} + +function candidateFilename(value: unknown): string | null { + if (!value || typeof value !== 'object') return null + const filename = (value as { filename?: unknown }).filename + return typeof filename === 'string' && filename.endsWith('.pmtiles') ? filename : null +} + +/** + * Parse the sidecar manifest defensively. Invalid records are isolated, while + * their filenames remain blocked from legacy-vector fallback. + */ +export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifest { + const empty = { archives: [], rejectedFilenames: [] } + if (!value || typeof value !== 'object') return empty + const manifest = value as { schemaVersion?: unknown; archives?: unknown } + if (manifest.schemaVersion !== 1 || !manifest.archives || typeof manifest.archives !== 'object') { + return empty + } + + const entries = Array.isArray(manifest.archives) + ? manifest.archives.map((candidate) => [candidateFilename(candidate), candidate] as const) + : Object.entries(manifest.archives) + const archives: MapArchiveMetadata[] = [] + const rejectedFilenames = new Set() + const filenames = new Set() + const resourceIds = new Set() + + for (const [key, candidate] of entries) { + const keyFilename = typeof key === 'string' && key.endsWith('.pmtiles') ? key : null + const embeddedFilename = candidateFilename(candidate) + const reject = () => { + if (keyFilename) rejectedFilenames.add(keyFilename) + if (embeddedFilename) rejectedFilenames.add(embeddedFilename) + } + + if ( + !keyFilename || + !isMapArchiveMetadata(candidate) || + candidate.filename !== keyFilename || + filenames.has(candidate.filename) || + resourceIds.has(candidate.resourceId) + ) { + reject() + continue + } + + filenames.add(candidate.filename) + resourceIds.add(candidate.resourceId) + archives.push({ ...candidate }) + } + + return { archives, rejectedFilenames: [...rejectedFilenames] } +} + +function compareMapArchiveVersions(a: string | null, b: string | null): number { + if (a === b) return 0 + if (a === null) return -1 + if (b === null) return 1 + return a < b ? -1 : 1 +} + +function parseLegacyMapFilename(filename: string): { resourceId: string; version: string } | null { + const name = filename.replace(/\.pmtiles$/, '') + const match = name.match(/^(.+)_(\d{4}-\d{2})$/) + if (!match) return null + return { resourceId: match[1], version: match[2] } +} + +export function sourceFromArchive(baseUrl: string, archive: MapArchiveMetadata): MapSource { + const source: MapSource = { + type: archive.kind, + url: `pmtiles://${urlJoin(baseUrl, archive.filename)}`, + } + if (archive.attribution) source.attribution = archive.attribution + if (archive.tileSize !== undefined) source.tileSize = archive.tileSize + if (archive.minzoom !== undefined) source.minzoom = archive.minzoom + if (archive.maxzoom !== undefined) source.maxzoom = archive.maxzoom + if (archive.bounds !== undefined) source.bounds = [...archive.bounds] + return source +} + +/** + * Build collision-free style sources from exact archive filenames. + * + * Existing archives without a manifest entry retain legacy vector behavior. + * Files rejected from an explicit manifest never receive that fallback. + */ +export function buildMapSourceDefinitions( + baseUrl: string, + regions: FileEntry[], + parsedManifest: ParsedMapArchiveManifest = { archives: [], rejectedFilenames: [] } +): MapSourceDefinition[] { + const metadataByFilename = new Map( + parsedManifest.archives.map((archive) => [archive.filename, archive]) + ) + const rejectedFilenames = new Set(parsedManifest.rejectedFilenames) + const definitions: MapSourceDefinition[] = [] + const usedSourceIds = new Set() + const bestLegacyByRegion = new Map< + string, + { region: Extract; version: string | null } + >() + + for (const region of regions) { + if (region.type !== 'file' || !region.name.endsWith('.pmtiles')) continue + const archive = metadataByFilename.get(region.name) + if (archive) { + if (usedSourceIds.has(archive.resourceId)) continue + usedSourceIds.add(archive.resourceId) + definitions.push({ + id: archive.resourceId, + source: sourceFromArchive(baseUrl, archive), + archive, + }) + continue + } + if (rejectedFilenames.has(region.name) || RESERVED_RASTER_FILENAME.test(region.name)) continue + + const parsed = parseLegacyMapFilename(region.name) + const regionName = parsed ? parsed.resourceId : region.name.replace('.pmtiles', '') + const version = parsed?.version ?? null + const existing = bestLegacyByRegion.get(regionName) + if (!existing || compareMapArchiveVersions(version, existing.version) > 0) { + bestLegacyByRegion.set(regionName, { region, version }) + } + } + + for (const [resourceId, { region }] of bestLegacyByRegion) { + if (usedSourceIds.has(resourceId)) continue + usedSourceIds.add(resourceId) + const archive: MapArchiveMetadata = { + filename: region.name, + resourceId, + kind: 'vector', + role: 'street', + tileFormat: 'mvt', + attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION, + } + definitions.push({ + id: resourceId, + source: sourceFromArchive(baseUrl, archive), + archive, + }) + } + + return definitions +} + +/** + * Compose raster and vector sources without mutating the base style template. + * Raster layers are emitted below cloned vector layers and hidden until a map + * presentation control explicitly activates them. + */ +export function composeMapStyle( + template: BaseStylesFile, + sources: MapSourceDefinition[], + sprites: string, + glyphs: string +): BaseStylesFile { + const vectorTemplates = template.layers.filter((layer) => layer.source) + const sourceIndependentLayers = template.layers + .filter((layer) => !layer.source) + .map((layer) => ({ ...layer })) + const rasterLayers: MapLayer[] = [] + const vectorLayers: MapLayer[] = [] + const styleSources: BaseStylesFile['sources'] = {} + + for (const definition of sources) { + styleSources[definition.id] = { ...definition.source } + + if (definition.archive.kind === 'raster') { + rasterLayers.push({ + id: `raster-${definition.id}`, + type: 'raster', + source: definition.id, + layout: { visibility: 'none' }, + metadata: { + 'nomad:role': definition.archive.role, + 'nomad:regionId': definition.archive.regionId, + }, + }) + continue + } + if (definition.archive.kind !== 'vector') continue + + for (const layerTemplate of vectorTemplates) { + vectorLayers.push({ + ...layerTemplate, + id: `${layerTemplate.id}-${definition.id}`, + type: layerTemplate.type, + source: definition.id, + }) + } + } + + return { + ...template, + sources: styleSources, + layers: [...sourceIndependentLayers, ...rasterLayers, ...vectorLayers], + sprite: sprites, + glyphs, + } +} diff --git a/admin/tests/unit/map_style_generation.spec.ts b/admin/tests/unit/map_style_generation.spec.ts new file mode 100644 index 00000000..0b3573bc --- /dev/null +++ b/admin/tests/unit/map_style_generation.spec.ts @@ -0,0 +1,242 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + buildMapSourceDefinitions, + composeMapStyle, + parseMapArchiveManifest, +} from '../../app/utils/map_styles.js' +import type { BaseStylesFile, MapArchiveMetadata, MapSourceDefinition } from '../../types/maps.js' +import type { FileEntry } from '../../types/files.js' + +const file = (name: string): FileEntry => ({ + type: 'file', + key: `/storage/maps/pmtiles/${name}`, + name, +}) + +const vectorTemplate = (): BaseStylesFile => ({ + version: 8, + sources: {}, + layers: [ + { id: 'background', type: 'background', paint: { 'background-color': '#eee' } }, + { 'id': 'land', 'type': 'fill', 'source': 'template', 'source-layer': 'land' }, + { 'id': 'roads', 'type': 'line', 'source': 'template', 'source-layer': 'roads' }, + ], + sprite: '', + glyphs: '', +}) + +const rasterMetadata = ( + regionId: string, + role: 'aerial' | 'topographic' = 'aerial' +): MapArchiveMetadata => ({ + filename: `raster-usgs-${role}-${regionId}-2026-07-z14.pmtiles`, + resourceId: `${role}-${regionId}`, + kind: 'raster', + role, + regionId, + tileFormat: 'jpeg', + tileSize: 256, + minzoom: 0, + maxzoom: 14, + bounds: [-103, 33, -94, 37], + attribution: 'USDA, USGS The National Map: Orthoimagery.', + sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer', + sourceVersion: '2026-07', + sha256: 'a'.repeat(64), + installedBytes: 1024, + verifiedAt: '2026-07-30T12:00:00.000Z', +}) + +test('uses explicit vector archive metadata instead of inferring the source type', () => { + const metadata: MapArchiveMetadata = { + filename: 'us_20260728_z15.pmtiles', + resourceId: 'street-us', + kind: 'vector', + role: 'street', + tileFormat: 'mvt', + attribution: 'OpenStreetMap', + maxzoom: 15, + } + + const sources = buildMapSourceDefinitions( + 'http://nomad.test/maps/pmtiles', + [file(metadata.filename)], + { + archives: [metadata], + rejectedFilenames: [], + } + ) + + assert.equal(sources.length, 1) + assert.equal(sources[0].id, 'street-us') + assert.equal(sources[0].source.type, 'vector') + assert.equal(sources[0].source.attribution, 'OpenStreetMap') + assert.equal(sources[0].source.maxzoom, 15) +}) + +test('parses a filename-keyed manifest for an arbitrary region ID', () => { + const metadata = rasterMetadata('us-ok') + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: { + [metadata.filename]: metadata, + }, + }) + + assert.deepEqual(parsed.archives, [metadata]) + assert.deepEqual(parsed.rejectedFilenames, []) +}) + +test('rejects invalid raster metadata and blocks legacy-vector fallback for that file', () => { + const metadata = { + ...rasterMetadata('us-ok'), + sha256: 'not-a-sha256', + } + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: { + [metadata.filename]: metadata, + }, + }) + const sources = buildMapSourceDefinitions( + 'http://nomad.test/maps/pmtiles', + [file('us_20260728_z15.pmtiles'), file(metadata.filename)], + parsed + ) + + assert.deepEqual(parsed.archives, []) + assert.deepEqual(parsed.rejectedFilenames, [metadata.filename]) + assert.deepEqual( + sources.map((source) => source.id), + ['us_20260728_z15'] + ) +}) + +test('requires complete provenance and verification metadata for raster archives', () => { + const requiredFields = [ + 'role', + 'regionId', + 'tileFormat', + 'tileSize', + 'minzoom', + 'maxzoom', + 'bounds', + 'attribution', + 'sourceUrl', + 'sourceVersion', + 'sha256', + 'installedBytes', + 'verifiedAt', + ] as const + + for (const field of requiredFields) { + const metadata = rasterMetadata('us-ok') as Record + delete metadata[field] + const parsed = parseMapArchiveManifest({ + schemaVersion: 1, + archives: { [String(metadata.filename)]: metadata }, + }) + assert.equal(parsed.archives.length, 0, `${field} should be required`) + } +}) + +test('creates non-colliding raster sources for multiple regions and roles', () => { + const metadata = [ + rasterMetadata('us-tx'), + rasterMetadata('us-ok'), + rasterMetadata('us-ga', 'topographic'), + ] + + const sources = buildMapSourceDefinitions( + 'http://nomad.test/maps/pmtiles', + metadata.map((archive) => file(archive.filename)), + { archives: metadata, rejectedFilenames: [] } + ) + + assert.deepEqual( + sources.map((source) => source.id), + ['aerial-us-tx', 'aerial-us-ok', 'topographic-us-ga'] + ) + assert.equal(new Set(sources.map((source) => source.id)).size, 3) + for (const source of sources) { + assert.equal(source.source.type, 'raster') + assert.equal(source.source.tileSize, 256) + assert.equal(source.source.minzoom, 0) + assert.equal(source.source.maxzoom, 14) + assert.deepEqual(source.source.bounds, [-103, 33, -94, 37]) + } +}) + +test('places hidden raster layers below cloned vector layers without mutating the template', () => { + const raster = rasterMetadata('us-ok') + const vector: MapArchiveMetadata = { + filename: 'us_20260728_z15.pmtiles', + resourceId: 'street-us', + kind: 'vector', + role: 'street', + tileFormat: 'mvt', + } + const sources: MapSourceDefinition[] = buildMapSourceDefinitions( + 'http://nomad.test/maps/pmtiles', + [file(vector.filename), file(raster.filename)], + { archives: [vector, raster], rejectedFilenames: [] } + ) + const template = vectorTemplate() + const original = structuredClone(template) + + const style = composeMapStyle( + template, + sources, + 'http://nomad.test/sprites', + 'http://nomad.test/fonts/{fontstack}/{range}.pbf' + ) + + const rasterLayer = style.layers.find((layer) => layer.id === 'raster-aerial-us-ok') + const vectorLayers = style.layers.filter((layer) => layer.source === 'street-us') + + assert.deepEqual(template, original) + assert.ok(rasterLayer) + assert.equal(rasterLayer.type, 'raster') + assert.equal(rasterLayer.source, 'aerial-us-ok') + assert.deepEqual(rasterLayer.layout, { visibility: 'none' }) + assert.deepEqual(rasterLayer.metadata, { + 'nomad:role': 'aerial', + 'nomad:regionId': 'us-ok', + }) + assert.deepEqual( + vectorLayers.map((layer) => layer.id), + ['land-street-us', 'roads-street-us'] + ) + assert.equal( + style.layers.some((layer) => layer.id === 'land-aerial-us-ok'), + false + ) + assert.ok(style.layers.indexOf(rasterLayer) < style.layers.indexOf(vectorLayers[0])) +}) + +test('preserves legacy vector deduplication by keeping the newest dated archive', () => { + const sources = buildMapSourceDefinitions('http://nomad.test/maps/pmtiles', [ + file('washington_2025-11.pmtiles'), + file('washington.pmtiles'), + file('washington_2025-12.pmtiles'), + ]) + + assert.equal(sources.length, 1) + assert.equal(sources[0].id, 'washington') + assert.match(sources[0].source.url, /washington_2025-12\.pmtiles$/) + assert.equal(sources[0].source.type, 'vector') +}) + +test('does not guess reserved unmanifested raster archives as vector maps', () => { + const sources = buildMapSourceDefinitions('http://nomad.test/maps/pmtiles', [ + file('raster-usgs-aerial-us-ok-2026-07-z14.pmtiles'), + file('oklahoma.pmtiles'), + ]) + + assert.deepEqual( + sources.map((source) => source.id), + ['oklahoma'] + ) +}) diff --git a/admin/types/maps.ts b/admin/types/maps.ts index ea44b56a..231789b1 100644 --- a/admin/types/maps.ts +++ b/admin/types/maps.ts @@ -12,6 +12,10 @@ export type MapSource = { type: 'vector' | 'raster' | 'raster-dem' | 'geojson' | 'image' | 'video' attribution?: string url: string + tileSize?: number + minzoom?: number + maxzoom?: number + bounds?: [number, number, number, number] } export type MapLayer = { @@ -22,6 +26,59 @@ export type MapLayer = { [key: string]: any } +export type MapArchiveKind = Extract +export type MapArchiveRole = 'street' | 'aerial' | 'topographic' +export type MapTileFormat = 'mvt' | 'jpeg' | 'png' | 'webp' + +/** + * Explicit sidecar metadata for an installed PMTiles archive. + * + * Existing archives without an entry remain supported as legacy vector maps. + * Raster archives must have a valid entry before style generation will activate + * them. + */ +export type MapArchiveMetadata = { + filename: string + resourceId: string + kind: MapArchiveKind + role?: MapArchiveRole + regionId?: string + tileFormat?: MapTileFormat + tileSize?: number + minzoom?: number + maxzoom?: number + bounds?: [number, number, number, number] + attribution?: string + sourceUrl?: string + sourceVersion?: string + sha256?: string + installedBytes?: number + verifiedAt?: string +} + +export type MapArchiveManifest = { + schemaVersion: 1 + generatedAt?: string + archives: Record +} + +export type ParsedMapArchiveManifest = { + archives: MapArchiveMetadata[] + /** + * Files explicitly mentioned by a manifest but rejected during validation. + * Style generation must not fall back to treating these files as legacy + * vector archives. + */ + rejectedFilenames: string[] +} + +/** Internal style-generation record retaining archive metadata with its source. */ +export type MapSourceDefinition = { + id: string + source: MapSource + archive: MapArchiveMetadata +} + /** ISO 3166-1 alpha-2 country code (e.g. "DE", "FR", "US"). */ export type CountryCode = string