mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-16 21:00:03 +02:00
feat(maps): support raster PMTiles manifests
This commit is contained in:
parent
54ee30a449
commit
fa6e0c56dd
|
|
@ -1,4 +1,4 @@
|
|||
import { BaseStylesFile, MapLayer } from '../../types/maps.js'
|
||||
import { BaseStylesFile, MapSourceDefinition, ParsedMapArchiveManifest } from '../../types/maps.js'
|
||||
import {
|
||||
DownloadRemoteSuccessCallback,
|
||||
FileEntry,
|
||||
|
|
@ -39,6 +39,13 @@ import { execFile } from 'child_process'
|
|||
import { createHash, randomBytes } from 'crypto'
|
||||
import { tmpdir } from 'os'
|
||||
import { promisify } from 'util'
|
||||
import {
|
||||
buildMapSourceDefinitions,
|
||||
composeMapStyle,
|
||||
DEFAULT_VECTOR_MAP_ATTRIBUTION,
|
||||
MAP_ARCHIVE_MANIFEST_FILENAME,
|
||||
parseMapArchiveManifest,
|
||||
} from '../utils/map_styles.js'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const DRY_RUN_TIMEOUT_MS = 60_000
|
||||
|
|
@ -63,8 +70,6 @@ const BASE_ASSETS_MIME_TYPES = [
|
|||
'application/octet-stream',
|
||||
]
|
||||
|
||||
const PMTILES_ATTRIBUTION =
|
||||
'<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
|
||||
const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
|
||||
|
||||
interface IMapService {
|
||||
|
|
@ -358,6 +363,7 @@ export class MapService implements IMapService {
|
|||
const rawStyles = JSON.parse(baseStyle.toString()) as BaseStylesFile
|
||||
|
||||
const regions = (await this.listRegions()).files
|
||||
const archiveManifest = await this.loadMapArchiveManifest()
|
||||
|
||||
/** If we have the host, use it to build public URLs, otherwise we'll fallback to defaults
|
||||
* This is mainly useful because we need to know what host the user is accessing from in order to
|
||||
|
|
@ -365,10 +371,31 @@ export class MapService implements IMapService {
|
|||
* e.g. user is accessing from "example.com", but we would by default generate "localhost:8080/..." so maps would
|
||||
* fail to load.
|
||||
*/
|
||||
const sources = this.generateSourcesArray(host, regions, protocol)
|
||||
const pmtilesBaseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
|
||||
const sources = buildMapSourceDefinitions(pmtilesBaseUrl, regions, archiveManifest)
|
||||
if (this.worldBasemapReady) {
|
||||
const worldSource: MapSourceDefinition = {
|
||||
id: WORLD_BASEMAP_SOURCE_NAME,
|
||||
source: {
|
||||
type: 'vector',
|
||||
attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION,
|
||||
url: `pmtiles://${urlJoin(pmtilesBaseUrl, WORLD_BASEMAP_FILENAME)}`,
|
||||
},
|
||||
archive: {
|
||||
filename: WORLD_BASEMAP_FILENAME,
|
||||
resourceId: WORLD_BASEMAP_SOURCE_NAME,
|
||||
kind: 'vector',
|
||||
role: 'street',
|
||||
tileFormat: 'mvt',
|
||||
attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION,
|
||||
maxzoom: WORLD_BASEMAP_MAX_ZOOM,
|
||||
},
|
||||
}
|
||||
sources.unshift(worldSource)
|
||||
}
|
||||
const baseUrl = this.getPublicFileBaseUrl(host, this.basemapsAssetsDir, protocol)
|
||||
|
||||
const styles = await this.generateStylesFile(
|
||||
const styles = composeMapStyle(
|
||||
rawStyles,
|
||||
sources,
|
||||
urlJoin(baseUrl, 'sprites/v4/light'),
|
||||
|
|
@ -378,6 +405,28 @@ export class MapService implements IMapService {
|
|||
return styles
|
||||
}
|
||||
|
||||
private async loadMapArchiveManifest(): Promise<ParsedMapArchiveManifest> {
|
||||
const manifestPath = resolve(join(this.baseDirPath, 'pmtiles', MAP_ARCHIVE_MANIFEST_FILENAME))
|
||||
const pmtilesDirectory = resolve(join(this.baseDirPath, 'pmtiles'))
|
||||
if (!manifestPath.startsWith(pmtilesDirectory + sep)) {
|
||||
throw new Error('Invalid map archive manifest path')
|
||||
}
|
||||
|
||||
const manifest = await getFile(manifestPath, 'string')
|
||||
if (!manifest) return { archives: [], rejectedFilenames: [] }
|
||||
|
||||
try {
|
||||
return parseMapArchiveManifest(JSON.parse(manifest.toString()))
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`[MapService] Ignoring malformed ${MAP_ARCHIVE_MANIFEST_FILENAME}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
return { archives: [], rejectedFilenames: [] }
|
||||
}
|
||||
}
|
||||
|
||||
async listCuratedCollections(): Promise<CollectionWithStatus[]> {
|
||||
const manifestService = new CollectionManifestService()
|
||||
return manifestService.getMapCollectionsWithStatus()
|
||||
|
|
@ -523,111 +572,6 @@ export class MapService implements IMapService {
|
|||
return await listDirectoryContentsRecursive(this.baseDirPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two map-file versions (YYYY-MM, or null for an undated legacy file). Returns >0 if
|
||||
* `a` is newer than `b`, <0 if older, 0 if equal. A dated build is always newer than an undated
|
||||
* legacy file; two dated builds compare lexicographically (correct for zero-padded YYYY-MM).
|
||||
*/
|
||||
private static compareMapVersions(a: string | null, b: string | null): number {
|
||||
if (a === b) return 0
|
||||
if (a === null) return -1
|
||||
if (b === null) return 1
|
||||
return a < b ? -1 : 1
|
||||
}
|
||||
|
||||
private generateSourcesArray(host: string | null, regions: FileEntry[], protocol: string = 'http'): BaseStylesFile['sources'][] {
|
||||
const sources: BaseStylesFile['sources'][] = []
|
||||
const baseUrl = this.getPublicFileBaseUrl(host, 'pmtiles', protocol)
|
||||
|
||||
// World basemap goes first so its layers render underneath regional extracts.
|
||||
// Only emitted when ensureWorldBasemap() succeeded — otherwise the style would
|
||||
// reference a file that doesn't exist and produce 404s on every tile request.
|
||||
if (this.worldBasemapReady) {
|
||||
const worldSource: BaseStylesFile['sources'] = {}
|
||||
worldSource[WORLD_BASEMAP_SOURCE_NAME] = {
|
||||
type: 'vector',
|
||||
attribution: PMTILES_ATTRIBUTION,
|
||||
url: `pmtiles://${urlJoin(baseUrl, WORLD_BASEMAP_FILENAME)}`,
|
||||
}
|
||||
sources.push(worldSource)
|
||||
}
|
||||
|
||||
// Dedupe by region name, keeping only the newest file per region. The source name is the
|
||||
// date-stripped region (e.g. both "washington.pmtiles" and "washington_2025-12.pmtiles" map
|
||||
// to "washington"). Emitting both produces duplicate source keys and duplicate layer ids,
|
||||
// which MapLibre rejects outright — blanking the ENTIRE map, not just that region. Old copies
|
||||
// linger when a newer curated version installs (#634), so guard against it here so the style
|
||||
// stays valid even if cleanup hasn't run. A dated build beats an undated legacy file; between
|
||||
// two dated builds the later YYYY-MM wins (lexicographic compare is correct for that format).
|
||||
const bestByRegion = new Map<string, { region: FileEntry; version: string | null }>()
|
||||
for (const region of regions) {
|
||||
if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
|
||||
const parsed = CollectionManifestService.parseMapFilename(region.name)
|
||||
const regionName = parsed ? parsed.resource_id : region.name.replace('.pmtiles', '')
|
||||
const version = parsed?.version ?? null
|
||||
const existing = bestByRegion.get(regionName)
|
||||
if (!existing || MapService.compareMapVersions(version, existing.version) > 0) {
|
||||
if (existing) {
|
||||
logger.warn(
|
||||
`[MapService] Duplicate map region "${regionName}": using "${region.name}" over "${existing.region.name}" (keeping newest)`
|
||||
)
|
||||
}
|
||||
bestByRegion.set(regionName, { region, version })
|
||||
} else {
|
||||
logger.warn(
|
||||
`[MapService] Duplicate map region "${regionName}": skipping "${region.name}" in favor of "${existing.region.name}" (keeping newest)`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [regionName, { region }] of bestByRegion) {
|
||||
const source: BaseStylesFile['sources'] = {}
|
||||
const sourceUrl = urlJoin(baseUrl, region.name)
|
||||
|
||||
source[regionName] = {
|
||||
type: 'vector',
|
||||
attribution: PMTILES_ATTRIBUTION,
|
||||
url: `pmtiles://${sourceUrl}`,
|
||||
}
|
||||
sources.push(source)
|
||||
}
|
||||
|
||||
return sources
|
||||
}
|
||||
|
||||
private async generateStylesFile(
|
||||
template: BaseStylesFile,
|
||||
sources: BaseStylesFile['sources'][],
|
||||
sprites: string,
|
||||
glyphs: string
|
||||
): Promise<BaseStylesFile> {
|
||||
const layersTemplates = template.layers.filter((layer) => layer.source)
|
||||
const withoutSources = template.layers.filter((layer) => !layer.source)
|
||||
|
||||
template.sources = {} // Clear existing sources
|
||||
template.layers = [...withoutSources] // Start with layers that don't depend on sources
|
||||
|
||||
for (const source of sources) {
|
||||
for (const layerTemplate of layersTemplates) {
|
||||
const layer: MapLayer = {
|
||||
...layerTemplate,
|
||||
id: `${layerTemplate.id}-${Object.keys(source)[0]}`,
|
||||
type: layerTemplate.type,
|
||||
source: Object.keys(source)[0],
|
||||
}
|
||||
template.layers.push(layer)
|
||||
}
|
||||
|
||||
template.sources = Object.assign(template.sources, source)
|
||||
}
|
||||
|
||||
template.sprite = sprites
|
||||
template.glyphs = glyphs
|
||||
|
||||
return template
|
||||
}
|
||||
|
||||
async getGlobalMapInfo(): Promise<ProtomapsBuildInfo> {
|
||||
const { default: axios } = await import('axios')
|
||||
const response = await axios.get(PROTOMAPS_BUILDS_METADATA_URL, { timeout: 15000 })
|
||||
|
|
|
|||
364
admin/app/utils/map_styles.ts
Normal file
364
admin/app/utils/map_styles.ts
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
import urlJoin from 'url-join'
|
||||
|
||||
import type { FileEntry } from '../../types/files.js'
|
||||
import type {
|
||||
BaseStylesFile,
|
||||
MapArchiveMetadata,
|
||||
MapLayer,
|
||||
MapSource,
|
||||
MapSourceDefinition,
|
||||
ParsedMapArchiveManifest,
|
||||
} from '../../types/maps.js'
|
||||
|
||||
export const DEFAULT_VECTOR_MAP_ATTRIBUTION =
|
||||
'<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
|
||||
export const MAP_ARCHIVE_MANIFEST_FILENAME = 'map-archive-manifest.json'
|
||||
|
||||
const MAP_ARCHIVE_KINDS = new Set(['vector', 'raster'])
|
||||
const MAP_ARCHIVE_ROLES = new Set(['street', 'aerial', 'topographic'])
|
||||
const MAP_TILE_FORMATS = new Set(['mvt', 'jpeg', 'png', 'webp'])
|
||||
const RASTER_TILE_FORMATS = new Set(['jpeg', 'png', 'webp'])
|
||||
const SAFE_RESOURCE_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/
|
||||
const SAFE_REGION_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/
|
||||
const SHA256_DIGEST = /^[a-f0-9]{64}$/i
|
||||
const HTTPS_URL = /^https:\/\//i
|
||||
const RESERVED_RASTER_FILENAME = /^raster[-_]/i
|
||||
|
||||
function optionalNonNegativeNumber(value: unknown): value is number | undefined {
|
||||
return value === undefined || (typeof value === 'number' && Number.isFinite(value) && value >= 0)
|
||||
}
|
||||
|
||||
function validZoom(value: unknown): value is number | undefined {
|
||||
return (
|
||||
value === undefined ||
|
||||
(typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= 30)
|
||||
)
|
||||
}
|
||||
|
||||
function validBounds(value: unknown): value is [number, number, number, number] | undefined {
|
||||
if (value === undefined) return true
|
||||
if (!Array.isArray(value) || value.length !== 4) return false
|
||||
if (!value.every((item) => typeof item === 'number' && Number.isFinite(item))) return false
|
||||
const [west, south, east, north] = value
|
||||
return (
|
||||
west >= -180 &&
|
||||
west <= 180 &&
|
||||
east >= -180 &&
|
||||
east <= 180 &&
|
||||
south >= -90 &&
|
||||
south <= 90 &&
|
||||
north >= -90 &&
|
||||
north <= 90 &&
|
||||
west < east &&
|
||||
south < north
|
||||
)
|
||||
}
|
||||
|
||||
function validTimestamp(value: unknown): value is string | undefined {
|
||||
return value === undefined || (typeof value === 'string' && Number.isFinite(Date.parse(value)))
|
||||
}
|
||||
|
||||
function isMapArchiveMetadata(value: unknown): value is MapArchiveMetadata {
|
||||
if (!value || typeof value !== 'object') return false
|
||||
const archive = value as Record<string, unknown>
|
||||
|
||||
if (
|
||||
typeof archive.filename !== 'string' ||
|
||||
!archive.filename.endsWith('.pmtiles') ||
|
||||
archive.filename.includes('/') ||
|
||||
archive.filename.includes('\\') ||
|
||||
typeof archive.resourceId !== 'string' ||
|
||||
!SAFE_RESOURCE_ID.test(archive.resourceId) ||
|
||||
typeof archive.kind !== 'string' ||
|
||||
!MAP_ARCHIVE_KINDS.has(archive.kind)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (
|
||||
archive.role !== undefined &&
|
||||
(typeof archive.role !== 'string' || !MAP_ARCHIVE_ROLES.has(archive.role))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
archive.regionId !== undefined &&
|
||||
(typeof archive.regionId !== 'string' || !SAFE_REGION_ID.test(archive.regionId))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
archive.tileFormat !== undefined &&
|
||||
(typeof archive.tileFormat !== 'string' || !MAP_TILE_FORMATS.has(archive.tileFormat))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
archive.tileSize !== undefined &&
|
||||
![128, 256, 512, 1024].includes(archive.tileSize as number)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
!validZoom(archive.minzoom) ||
|
||||
!validZoom(archive.maxzoom) ||
|
||||
!optionalNonNegativeNumber(archive.installedBytes) ||
|
||||
!validBounds(archive.bounds)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
typeof archive.minzoom === 'number' &&
|
||||
typeof archive.maxzoom === 'number' &&
|
||||
archive.minzoom > archive.maxzoom
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
archive.sha256 !== undefined &&
|
||||
(typeof archive.sha256 !== 'string' || !SHA256_DIGEST.test(archive.sha256))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (!validTimestamp(archive.verifiedAt)) return false
|
||||
if (
|
||||
archive.sourceUrl !== undefined &&
|
||||
(typeof archive.sourceUrl !== 'string' || !HTTPS_URL.test(archive.sourceUrl))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
for (const field of ['attribution', 'sourceVersion'] as const) {
|
||||
if (
|
||||
archive[field] !== undefined &&
|
||||
(typeof archive[field] !== 'string' || archive[field].trim().length === 0)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if (archive.kind === 'raster') {
|
||||
if (
|
||||
(archive.role !== 'aerial' && archive.role !== 'topographic') ||
|
||||
typeof archive.regionId !== 'string' ||
|
||||
typeof archive.tileFormat !== 'string' ||
|
||||
!RASTER_TILE_FORMATS.has(archive.tileFormat) ||
|
||||
typeof archive.tileSize !== 'number' ||
|
||||
typeof archive.minzoom !== 'number' ||
|
||||
typeof archive.maxzoom !== 'number' ||
|
||||
!archive.bounds ||
|
||||
typeof archive.attribution !== 'string' ||
|
||||
typeof archive.sourceUrl !== 'string' ||
|
||||
typeof archive.sourceVersion !== 'string' ||
|
||||
typeof archive.sha256 !== 'string' ||
|
||||
typeof archive.installedBytes !== 'number' ||
|
||||
typeof archive.verifiedAt !== 'string'
|
||||
) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
function candidateFilename(value: unknown): string | null {
|
||||
if (!value || typeof value !== 'object') return null
|
||||
const filename = (value as { filename?: unknown }).filename
|
||||
return typeof filename === 'string' && filename.endsWith('.pmtiles') ? filename : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the sidecar manifest defensively. Invalid records are isolated, while
|
||||
* their filenames remain blocked from legacy-vector fallback.
|
||||
*/
|
||||
export function parseMapArchiveManifest(value: unknown): ParsedMapArchiveManifest {
|
||||
const empty = { archives: [], rejectedFilenames: [] }
|
||||
if (!value || typeof value !== 'object') return empty
|
||||
const manifest = value as { schemaVersion?: unknown; archives?: unknown }
|
||||
if (manifest.schemaVersion !== 1 || !manifest.archives || typeof manifest.archives !== 'object') {
|
||||
return empty
|
||||
}
|
||||
|
||||
const entries = Array.isArray(manifest.archives)
|
||||
? manifest.archives.map((candidate) => [candidateFilename(candidate), candidate] as const)
|
||||
: Object.entries(manifest.archives)
|
||||
const archives: MapArchiveMetadata[] = []
|
||||
const rejectedFilenames = new Set<string>()
|
||||
const filenames = new Set<string>()
|
||||
const resourceIds = new Set<string>()
|
||||
|
||||
for (const [key, candidate] of entries) {
|
||||
const keyFilename = typeof key === 'string' && key.endsWith('.pmtiles') ? key : null
|
||||
const embeddedFilename = candidateFilename(candidate)
|
||||
const reject = () => {
|
||||
if (keyFilename) rejectedFilenames.add(keyFilename)
|
||||
if (embeddedFilename) rejectedFilenames.add(embeddedFilename)
|
||||
}
|
||||
|
||||
if (
|
||||
!keyFilename ||
|
||||
!isMapArchiveMetadata(candidate) ||
|
||||
candidate.filename !== keyFilename ||
|
||||
filenames.has(candidate.filename) ||
|
||||
resourceIds.has(candidate.resourceId)
|
||||
) {
|
||||
reject()
|
||||
continue
|
||||
}
|
||||
|
||||
filenames.add(candidate.filename)
|
||||
resourceIds.add(candidate.resourceId)
|
||||
archives.push({ ...candidate })
|
||||
}
|
||||
|
||||
return { archives, rejectedFilenames: [...rejectedFilenames] }
|
||||
}
|
||||
|
||||
function compareMapArchiveVersions(a: string | null, b: string | null): number {
|
||||
if (a === b) return 0
|
||||
if (a === null) return -1
|
||||
if (b === null) return 1
|
||||
return a < b ? -1 : 1
|
||||
}
|
||||
|
||||
function parseLegacyMapFilename(filename: string): { resourceId: string; version: string } | null {
|
||||
const name = filename.replace(/\.pmtiles$/, '')
|
||||
const match = name.match(/^(.+)_(\d{4}-\d{2})$/)
|
||||
if (!match) return null
|
||||
return { resourceId: match[1], version: match[2] }
|
||||
}
|
||||
|
||||
export function sourceFromArchive(baseUrl: string, archive: MapArchiveMetadata): MapSource {
|
||||
const source: MapSource = {
|
||||
type: archive.kind,
|
||||
url: `pmtiles://${urlJoin(baseUrl, archive.filename)}`,
|
||||
}
|
||||
if (archive.attribution) source.attribution = archive.attribution
|
||||
if (archive.tileSize !== undefined) source.tileSize = archive.tileSize
|
||||
if (archive.minzoom !== undefined) source.minzoom = archive.minzoom
|
||||
if (archive.maxzoom !== undefined) source.maxzoom = archive.maxzoom
|
||||
if (archive.bounds !== undefined) source.bounds = [...archive.bounds]
|
||||
return source
|
||||
}
|
||||
|
||||
/**
|
||||
* Build collision-free style sources from exact archive filenames.
|
||||
*
|
||||
* Existing archives without a manifest entry retain legacy vector behavior.
|
||||
* Files rejected from an explicit manifest never receive that fallback.
|
||||
*/
|
||||
export function buildMapSourceDefinitions(
|
||||
baseUrl: string,
|
||||
regions: FileEntry[],
|
||||
parsedManifest: ParsedMapArchiveManifest = { archives: [], rejectedFilenames: [] }
|
||||
): MapSourceDefinition[] {
|
||||
const metadataByFilename = new Map(
|
||||
parsedManifest.archives.map((archive) => [archive.filename, archive])
|
||||
)
|
||||
const rejectedFilenames = new Set(parsedManifest.rejectedFilenames)
|
||||
const definitions: MapSourceDefinition[] = []
|
||||
const usedSourceIds = new Set<string>()
|
||||
const bestLegacyByRegion = new Map<
|
||||
string,
|
||||
{ region: Extract<FileEntry, { type: 'file' }>; version: string | null }
|
||||
>()
|
||||
|
||||
for (const region of regions) {
|
||||
if (region.type !== 'file' || !region.name.endsWith('.pmtiles')) continue
|
||||
const archive = metadataByFilename.get(region.name)
|
||||
if (archive) {
|
||||
if (usedSourceIds.has(archive.resourceId)) continue
|
||||
usedSourceIds.add(archive.resourceId)
|
||||
definitions.push({
|
||||
id: archive.resourceId,
|
||||
source: sourceFromArchive(baseUrl, archive),
|
||||
archive,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (rejectedFilenames.has(region.name) || RESERVED_RASTER_FILENAME.test(region.name)) continue
|
||||
|
||||
const parsed = parseLegacyMapFilename(region.name)
|
||||
const regionName = parsed ? parsed.resourceId : region.name.replace('.pmtiles', '')
|
||||
const version = parsed?.version ?? null
|
||||
const existing = bestLegacyByRegion.get(regionName)
|
||||
if (!existing || compareMapArchiveVersions(version, existing.version) > 0) {
|
||||
bestLegacyByRegion.set(regionName, { region, version })
|
||||
}
|
||||
}
|
||||
|
||||
for (const [resourceId, { region }] of bestLegacyByRegion) {
|
||||
if (usedSourceIds.has(resourceId)) continue
|
||||
usedSourceIds.add(resourceId)
|
||||
const archive: MapArchiveMetadata = {
|
||||
filename: region.name,
|
||||
resourceId,
|
||||
kind: 'vector',
|
||||
role: 'street',
|
||||
tileFormat: 'mvt',
|
||||
attribution: DEFAULT_VECTOR_MAP_ATTRIBUTION,
|
||||
}
|
||||
definitions.push({
|
||||
id: resourceId,
|
||||
source: sourceFromArchive(baseUrl, archive),
|
||||
archive,
|
||||
})
|
||||
}
|
||||
|
||||
return definitions
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose raster and vector sources without mutating the base style template.
|
||||
* Raster layers are emitted below cloned vector layers and hidden until a map
|
||||
* presentation control explicitly activates them.
|
||||
*/
|
||||
export function composeMapStyle(
|
||||
template: BaseStylesFile,
|
||||
sources: MapSourceDefinition[],
|
||||
sprites: string,
|
||||
glyphs: string
|
||||
): BaseStylesFile {
|
||||
const vectorTemplates = template.layers.filter((layer) => layer.source)
|
||||
const sourceIndependentLayers = template.layers
|
||||
.filter((layer) => !layer.source)
|
||||
.map((layer) => ({ ...layer }))
|
||||
const rasterLayers: MapLayer[] = []
|
||||
const vectorLayers: MapLayer[] = []
|
||||
const styleSources: BaseStylesFile['sources'] = {}
|
||||
|
||||
for (const definition of sources) {
|
||||
styleSources[definition.id] = { ...definition.source }
|
||||
|
||||
if (definition.archive.kind === 'raster') {
|
||||
rasterLayers.push({
|
||||
id: `raster-${definition.id}`,
|
||||
type: 'raster',
|
||||
source: definition.id,
|
||||
layout: { visibility: 'none' },
|
||||
metadata: {
|
||||
'nomad:role': definition.archive.role,
|
||||
'nomad:regionId': definition.archive.regionId,
|
||||
},
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (definition.archive.kind !== 'vector') continue
|
||||
|
||||
for (const layerTemplate of vectorTemplates) {
|
||||
vectorLayers.push({
|
||||
...layerTemplate,
|
||||
id: `${layerTemplate.id}-${definition.id}`,
|
||||
type: layerTemplate.type,
|
||||
source: definition.id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...template,
|
||||
sources: styleSources,
|
||||
layers: [...sourceIndependentLayers, ...rasterLayers, ...vectorLayers],
|
||||
sprite: sprites,
|
||||
glyphs,
|
||||
}
|
||||
}
|
||||
242
admin/tests/unit/map_style_generation.spec.ts
Normal file
242
admin/tests/unit/map_style_generation.spec.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
import * as assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import {
|
||||
buildMapSourceDefinitions,
|
||||
composeMapStyle,
|
||||
parseMapArchiveManifest,
|
||||
} from '../../app/utils/map_styles.js'
|
||||
import type { BaseStylesFile, MapArchiveMetadata, MapSourceDefinition } from '../../types/maps.js'
|
||||
import type { FileEntry } from '../../types/files.js'
|
||||
|
||||
const file = (name: string): FileEntry => ({
|
||||
type: 'file',
|
||||
key: `/storage/maps/pmtiles/${name}`,
|
||||
name,
|
||||
})
|
||||
|
||||
const vectorTemplate = (): BaseStylesFile => ({
|
||||
version: 8,
|
||||
sources: {},
|
||||
layers: [
|
||||
{ id: 'background', type: 'background', paint: { 'background-color': '#eee' } },
|
||||
{ 'id': 'land', 'type': 'fill', 'source': 'template', 'source-layer': 'land' },
|
||||
{ 'id': 'roads', 'type': 'line', 'source': 'template', 'source-layer': 'roads' },
|
||||
],
|
||||
sprite: '',
|
||||
glyphs: '',
|
||||
})
|
||||
|
||||
const rasterMetadata = (
|
||||
regionId: string,
|
||||
role: 'aerial' | 'topographic' = 'aerial'
|
||||
): MapArchiveMetadata => ({
|
||||
filename: `raster-usgs-${role}-${regionId}-2026-07-z14.pmtiles`,
|
||||
resourceId: `${role}-${regionId}`,
|
||||
kind: 'raster',
|
||||
role,
|
||||
regionId,
|
||||
tileFormat: 'jpeg',
|
||||
tileSize: 256,
|
||||
minzoom: 0,
|
||||
maxzoom: 14,
|
||||
bounds: [-103, 33, -94, 37],
|
||||
attribution: 'USDA, USGS The National Map: Orthoimagery.',
|
||||
sourceUrl: 'https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer',
|
||||
sourceVersion: '2026-07',
|
||||
sha256: 'a'.repeat(64),
|
||||
installedBytes: 1024,
|
||||
verifiedAt: '2026-07-30T12:00:00.000Z',
|
||||
})
|
||||
|
||||
test('uses explicit vector archive metadata instead of inferring the source type', () => {
|
||||
const metadata: MapArchiveMetadata = {
|
||||
filename: 'us_20260728_z15.pmtiles',
|
||||
resourceId: 'street-us',
|
||||
kind: 'vector',
|
||||
role: 'street',
|
||||
tileFormat: 'mvt',
|
||||
attribution: 'OpenStreetMap',
|
||||
maxzoom: 15,
|
||||
}
|
||||
|
||||
const sources = buildMapSourceDefinitions(
|
||||
'http://nomad.test/maps/pmtiles',
|
||||
[file(metadata.filename)],
|
||||
{
|
||||
archives: [metadata],
|
||||
rejectedFilenames: [],
|
||||
}
|
||||
)
|
||||
|
||||
assert.equal(sources.length, 1)
|
||||
assert.equal(sources[0].id, 'street-us')
|
||||
assert.equal(sources[0].source.type, 'vector')
|
||||
assert.equal(sources[0].source.attribution, 'OpenStreetMap')
|
||||
assert.equal(sources[0].source.maxzoom, 15)
|
||||
})
|
||||
|
||||
test('parses a filename-keyed manifest for an arbitrary region ID', () => {
|
||||
const metadata = rasterMetadata('us-ok')
|
||||
const parsed = parseMapArchiveManifest({
|
||||
schemaVersion: 1,
|
||||
archives: {
|
||||
[metadata.filename]: metadata,
|
||||
},
|
||||
})
|
||||
|
||||
assert.deepEqual(parsed.archives, [metadata])
|
||||
assert.deepEqual(parsed.rejectedFilenames, [])
|
||||
})
|
||||
|
||||
test('rejects invalid raster metadata and blocks legacy-vector fallback for that file', () => {
|
||||
const metadata = {
|
||||
...rasterMetadata('us-ok'),
|
||||
sha256: 'not-a-sha256',
|
||||
}
|
||||
const parsed = parseMapArchiveManifest({
|
||||
schemaVersion: 1,
|
||||
archives: {
|
||||
[metadata.filename]: metadata,
|
||||
},
|
||||
})
|
||||
const sources = buildMapSourceDefinitions(
|
||||
'http://nomad.test/maps/pmtiles',
|
||||
[file('us_20260728_z15.pmtiles'), file(metadata.filename)],
|
||||
parsed
|
||||
)
|
||||
|
||||
assert.deepEqual(parsed.archives, [])
|
||||
assert.deepEqual(parsed.rejectedFilenames, [metadata.filename])
|
||||
assert.deepEqual(
|
||||
sources.map((source) => source.id),
|
||||
['us_20260728_z15']
|
||||
)
|
||||
})
|
||||
|
||||
test('requires complete provenance and verification metadata for raster archives', () => {
|
||||
const requiredFields = [
|
||||
'role',
|
||||
'regionId',
|
||||
'tileFormat',
|
||||
'tileSize',
|
||||
'minzoom',
|
||||
'maxzoom',
|
||||
'bounds',
|
||||
'attribution',
|
||||
'sourceUrl',
|
||||
'sourceVersion',
|
||||
'sha256',
|
||||
'installedBytes',
|
||||
'verifiedAt',
|
||||
] as const
|
||||
|
||||
for (const field of requiredFields) {
|
||||
const metadata = rasterMetadata('us-ok') as Record<string, unknown>
|
||||
delete metadata[field]
|
||||
const parsed = parseMapArchiveManifest({
|
||||
schemaVersion: 1,
|
||||
archives: { [String(metadata.filename)]: metadata },
|
||||
})
|
||||
assert.equal(parsed.archives.length, 0, `${field} should be required`)
|
||||
}
|
||||
})
|
||||
|
||||
test('creates non-colliding raster sources for multiple regions and roles', () => {
|
||||
const metadata = [
|
||||
rasterMetadata('us-tx'),
|
||||
rasterMetadata('us-ok'),
|
||||
rasterMetadata('us-ga', 'topographic'),
|
||||
]
|
||||
|
||||
const sources = buildMapSourceDefinitions(
|
||||
'http://nomad.test/maps/pmtiles',
|
||||
metadata.map((archive) => file(archive.filename)),
|
||||
{ archives: metadata, rejectedFilenames: [] }
|
||||
)
|
||||
|
||||
assert.deepEqual(
|
||||
sources.map((source) => source.id),
|
||||
['aerial-us-tx', 'aerial-us-ok', 'topographic-us-ga']
|
||||
)
|
||||
assert.equal(new Set(sources.map((source) => source.id)).size, 3)
|
||||
for (const source of sources) {
|
||||
assert.equal(source.source.type, 'raster')
|
||||
assert.equal(source.source.tileSize, 256)
|
||||
assert.equal(source.source.minzoom, 0)
|
||||
assert.equal(source.source.maxzoom, 14)
|
||||
assert.deepEqual(source.source.bounds, [-103, 33, -94, 37])
|
||||
}
|
||||
})
|
||||
|
||||
test('places hidden raster layers below cloned vector layers without mutating the template', () => {
|
||||
const raster = rasterMetadata('us-ok')
|
||||
const vector: MapArchiveMetadata = {
|
||||
filename: 'us_20260728_z15.pmtiles',
|
||||
resourceId: 'street-us',
|
||||
kind: 'vector',
|
||||
role: 'street',
|
||||
tileFormat: 'mvt',
|
||||
}
|
||||
const sources: MapSourceDefinition[] = buildMapSourceDefinitions(
|
||||
'http://nomad.test/maps/pmtiles',
|
||||
[file(vector.filename), file(raster.filename)],
|
||||
{ archives: [vector, raster], rejectedFilenames: [] }
|
||||
)
|
||||
const template = vectorTemplate()
|
||||
const original = structuredClone(template)
|
||||
|
||||
const style = composeMapStyle(
|
||||
template,
|
||||
sources,
|
||||
'http://nomad.test/sprites',
|
||||
'http://nomad.test/fonts/{fontstack}/{range}.pbf'
|
||||
)
|
||||
|
||||
const rasterLayer = style.layers.find((layer) => layer.id === 'raster-aerial-us-ok')
|
||||
const vectorLayers = style.layers.filter((layer) => layer.source === 'street-us')
|
||||
|
||||
assert.deepEqual(template, original)
|
||||
assert.ok(rasterLayer)
|
||||
assert.equal(rasterLayer.type, 'raster')
|
||||
assert.equal(rasterLayer.source, 'aerial-us-ok')
|
||||
assert.deepEqual(rasterLayer.layout, { visibility: 'none' })
|
||||
assert.deepEqual(rasterLayer.metadata, {
|
||||
'nomad:role': 'aerial',
|
||||
'nomad:regionId': 'us-ok',
|
||||
})
|
||||
assert.deepEqual(
|
||||
vectorLayers.map((layer) => layer.id),
|
||||
['land-street-us', 'roads-street-us']
|
||||
)
|
||||
assert.equal(
|
||||
style.layers.some((layer) => layer.id === 'land-aerial-us-ok'),
|
||||
false
|
||||
)
|
||||
assert.ok(style.layers.indexOf(rasterLayer) < style.layers.indexOf(vectorLayers[0]))
|
||||
})
|
||||
|
||||
test('preserves legacy vector deduplication by keeping the newest dated archive', () => {
|
||||
const sources = buildMapSourceDefinitions('http://nomad.test/maps/pmtiles', [
|
||||
file('washington_2025-11.pmtiles'),
|
||||
file('washington.pmtiles'),
|
||||
file('washington_2025-12.pmtiles'),
|
||||
])
|
||||
|
||||
assert.equal(sources.length, 1)
|
||||
assert.equal(sources[0].id, 'washington')
|
||||
assert.match(sources[0].source.url, /washington_2025-12\.pmtiles$/)
|
||||
assert.equal(sources[0].source.type, 'vector')
|
||||
})
|
||||
|
||||
test('does not guess reserved unmanifested raster archives as vector maps', () => {
|
||||
const sources = buildMapSourceDefinitions('http://nomad.test/maps/pmtiles', [
|
||||
file('raster-usgs-aerial-us-ok-2026-07-z14.pmtiles'),
|
||||
file('oklahoma.pmtiles'),
|
||||
])
|
||||
|
||||
assert.deepEqual(
|
||||
sources.map((source) => source.id),
|
||||
['oklahoma']
|
||||
)
|
||||
})
|
||||
|
|
@ -12,6 +12,10 @@ export type MapSource = {
|
|||
type: 'vector' | 'raster' | 'raster-dem' | 'geojson' | 'image' | 'video'
|
||||
attribution?: string
|
||||
url: string
|
||||
tileSize?: number
|
||||
minzoom?: number
|
||||
maxzoom?: number
|
||||
bounds?: [number, number, number, number]
|
||||
}
|
||||
|
||||
export type MapLayer = {
|
||||
|
|
@ -22,6 +26,59 @@ export type MapLayer = {
|
|||
[key: string]: any
|
||||
}
|
||||
|
||||
export type MapArchiveKind = Extract<MapSource['type'], 'vector' | 'raster'>
|
||||
export type MapArchiveRole = 'street' | 'aerial' | 'topographic'
|
||||
export type MapTileFormat = 'mvt' | 'jpeg' | 'png' | 'webp'
|
||||
|
||||
/**
|
||||
* Explicit sidecar metadata for an installed PMTiles archive.
|
||||
*
|
||||
* Existing archives without an entry remain supported as legacy vector maps.
|
||||
* Raster archives must have a valid entry before style generation will activate
|
||||
* them.
|
||||
*/
|
||||
export type MapArchiveMetadata = {
|
||||
filename: string
|
||||
resourceId: string
|
||||
kind: MapArchiveKind
|
||||
role?: MapArchiveRole
|
||||
regionId?: string
|
||||
tileFormat?: MapTileFormat
|
||||
tileSize?: number
|
||||
minzoom?: number
|
||||
maxzoom?: number
|
||||
bounds?: [number, number, number, number]
|
||||
attribution?: string
|
||||
sourceUrl?: string
|
||||
sourceVersion?: string
|
||||
sha256?: string
|
||||
installedBytes?: number
|
||||
verifiedAt?: string
|
||||
}
|
||||
|
||||
export type MapArchiveManifest = {
|
||||
schemaVersion: 1
|
||||
generatedAt?: string
|
||||
archives: Record<string, MapArchiveMetadata>
|
||||
}
|
||||
|
||||
export type ParsedMapArchiveManifest = {
|
||||
archives: MapArchiveMetadata[]
|
||||
/**
|
||||
* Files explicitly mentioned by a manifest but rejected during validation.
|
||||
* Style generation must not fall back to treating these files as legacy
|
||||
* vector archives.
|
||||
*/
|
||||
rejectedFilenames: string[]
|
||||
}
|
||||
|
||||
/** Internal style-generation record retaining archive metadata with its source. */
|
||||
export type MapSourceDefinition = {
|
||||
id: string
|
||||
source: MapSource
|
||||
archive: MapArchiveMetadata
|
||||
}
|
||||
|
||||
/** ISO 3166-1 alpha-2 country code (e.g. "DE", "FR", "US"). */
|
||||
export type CountryCode = string
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user