mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-28 03:29:25 +01:00
feat: [wip] new maps system
This commit is contained in:
parent
bff5136564
commit
12a6f2230d
66
admin/app/controllers/maps_controller.ts
Normal file
66
admin/app/controllers/maps_controller.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import { MapService } from '#services/map_service'
|
||||||
|
import {
|
||||||
|
filenameValidator,
|
||||||
|
remoteDownloadValidator,
|
||||||
|
remoteDownloadValidatorOptional,
|
||||||
|
} from '#validators/common'
|
||||||
|
import { inject } from '@adonisjs/core'
|
||||||
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
|
|
||||||
|
@inject()
|
||||||
|
export default class MapsController {
|
||||||
|
constructor(private mapService: MapService) {}
|
||||||
|
|
||||||
|
async index({ inertia }: HttpContext) {
|
||||||
|
return inertia.render('maps')
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkBaseAssets({}: HttpContext) {
|
||||||
|
const exists = await this.mapService.checkBaseAssetsExist()
|
||||||
|
return { exists }
|
||||||
|
}
|
||||||
|
|
||||||
|
async downloadBaseAssets({ request }: HttpContext) {
|
||||||
|
const payload = await request.validateUsing(remoteDownloadValidatorOptional)
|
||||||
|
await this.mapService.downloadBaseAssets(payload.url)
|
||||||
|
return { success: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
async downloadRemote({ request }: HttpContext) {
|
||||||
|
const payload = await request.validateUsing(remoteDownloadValidator)
|
||||||
|
const filename = await this.mapService.downloadRemote(payload.url)
|
||||||
|
return {
|
||||||
|
message: 'Download started successfully',
|
||||||
|
filename,
|
||||||
|
url: payload.url,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async listRegions({}: HttpContext) {
|
||||||
|
return await this.mapService.listRegions()
|
||||||
|
}
|
||||||
|
|
||||||
|
async styles({ response }: HttpContext) {
|
||||||
|
const styles = await this.mapService.generateStylesJSON()
|
||||||
|
return response.json(styles)
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete({ request, response }: HttpContext) {
|
||||||
|
const payload = await request.validateUsing(filenameValidator)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.mapService.delete(payload.filename)
|
||||||
|
} catch (error) {
|
||||||
|
if (error.message === 'not_found') {
|
||||||
|
return response.status(404).send({
|
||||||
|
message: `Map file with key ${payload.filename} not found`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
throw error // Re-throw any other errors and let the global error handler catch
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
message: 'Map file deleted successfully',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { MapService } from '#services/map_service';
|
||||||
import { SystemService } from '#services/system_service';
|
import { SystemService } from '#services/system_service';
|
||||||
import { inject } from '@adonisjs/core';
|
import { inject } from '@adonisjs/core';
|
||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
|
|
@ -6,6 +7,7 @@ import type { HttpContext } from '@adonisjs/core/http'
|
||||||
export default class SettingsController {
|
export default class SettingsController {
|
||||||
constructor(
|
constructor(
|
||||||
private systemService: SystemService,
|
private systemService: SystemService,
|
||||||
|
private mapService: MapService
|
||||||
) { }
|
) { }
|
||||||
|
|
||||||
async system({ inertia }: HttpContext) {
|
async system({ inertia }: HttpContext) {
|
||||||
|
|
@ -25,11 +27,22 @@ export default class SettingsController {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async legal({ inertia }: HttpContext) {
|
async legal({ inertia }: HttpContext) {
|
||||||
return inertia.render('settings/legal');
|
return inertia.render('settings/legal');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async maps({ inertia }: HttpContext) {
|
||||||
|
const baseAssetsCheck = await this.mapService.checkBaseAssetsExist();
|
||||||
|
const regionFiles = await this.mapService.listRegions();
|
||||||
|
return inertia.render('settings/maps', {
|
||||||
|
maps: {
|
||||||
|
baseAssetsExist: baseAssetsCheck,
|
||||||
|
regionFiles: regionFiles.files
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async zim({ inertia }: HttpContext) {
|
async zim({ inertia }: HttpContext) {
|
||||||
return inertia.render('settings/zim/index')
|
return inertia.render('settings/zim/index')
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,49 +1,48 @@
|
||||||
import { ZimService } from '#services/zim_service';
|
import { ZimService } from '#services/zim_service'
|
||||||
import { inject } from '@adonisjs/core';
|
import { filenameValidator, remoteDownloadValidator } from '#validators/common'
|
||||||
|
import { inject } from '@adonisjs/core'
|
||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
|
|
||||||
@inject()
|
@inject()
|
||||||
export default class ZimController {
|
export default class ZimController {
|
||||||
constructor(
|
constructor(private zimService: ZimService) {}
|
||||||
private zimService: ZimService
|
|
||||||
) { }
|
|
||||||
|
|
||||||
async list({ }: HttpContext) {
|
async list({}: HttpContext) {
|
||||||
return await this.zimService.list();
|
return await this.zimService.list()
|
||||||
|
}
|
||||||
|
|
||||||
|
async listRemote({ request }: HttpContext) {
|
||||||
|
const { start = 0, count = 12 } = request.qs()
|
||||||
|
return await this.zimService.listRemote({ start, count })
|
||||||
|
}
|
||||||
|
|
||||||
|
async downloadRemote({ request }: HttpContext) {
|
||||||
|
const payload = await request.validateUsing(remoteDownloadValidator)
|
||||||
|
const filename = await this.zimService.downloadRemote(payload.url)
|
||||||
|
|
||||||
|
return {
|
||||||
|
message: 'Download started successfully',
|
||||||
|
filename,
|
||||||
|
url: payload.url,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete({ request, response }: HttpContext) {
|
||||||
|
const payload = await request.validateUsing(filenameValidator)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.zimService.delete(payload.filename)
|
||||||
|
} catch (error) {
|
||||||
|
if (error.message === 'not_found') {
|
||||||
|
return response.status(404).send({
|
||||||
|
message: `ZIM file with key ${payload.filename} not found`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
throw error // Re-throw any other errors and let the global error handler catch
|
||||||
}
|
}
|
||||||
|
|
||||||
async listRemote({ request }: HttpContext) {
|
return {
|
||||||
const { start = 0, count = 12 } = request.qs();
|
message: 'ZIM file deleted successfully',
|
||||||
return await this.zimService.listRemote({ start, count });
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
async downloadRemote({ request, response }: HttpContext) {
|
}
|
||||||
const { url } = request.body()
|
|
||||||
const filename = await this.zimService.downloadRemote(url);
|
|
||||||
|
|
||||||
response.status(200).send({
|
|
||||||
message: 'Download started successfully',
|
|
||||||
filename,
|
|
||||||
url
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async delete({ request, response }: HttpContext) {
|
|
||||||
const { key } = request.params();
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.zimService.delete(key);
|
|
||||||
} catch (error) {
|
|
||||||
if (error.message === 'not_found') {
|
|
||||||
return response.status(404).send({
|
|
||||||
message: `ZIM file with key ${key} not found`
|
|
||||||
});
|
|
||||||
}
|
|
||||||
throw error; // Re-throw any other errors and let the global error handler catch
|
|
||||||
}
|
|
||||||
|
|
||||||
response.status(200).send({
|
|
||||||
message: 'ZIM file deleted successfully'
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
235
admin/app/services/map_service.ts
Normal file
235
admin/app/services/map_service.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
||||||
|
import { BaseStylesFile, MapLayer } from '../../types/maps.js'
|
||||||
|
import { FileEntry } from '../../types/files.js'
|
||||||
|
import { doBackgroundDownload, doResumableDownloadWithRetry } from '../utils/downloads.js'
|
||||||
|
import { extract } from 'tar'
|
||||||
|
import env from '#start/env'
|
||||||
|
import {
|
||||||
|
listDirectoryContentsRecursive,
|
||||||
|
listDirectoryContents,
|
||||||
|
getFileStatsIfExists,
|
||||||
|
deleteFileIfExists,
|
||||||
|
getFile,
|
||||||
|
} from '../utils/fs.js'
|
||||||
|
import { join } from 'path'
|
||||||
|
import urlJoin from 'url-join'
|
||||||
|
|
||||||
|
const BASE_ASSETS_MIME_TYPES = [
|
||||||
|
'application/gzip',
|
||||||
|
'application/x-gzip',
|
||||||
|
'application/octet-stream',
|
||||||
|
]
|
||||||
|
const BROADCAST_CHANNEL = 'map-downloads'
|
||||||
|
|
||||||
|
const PMTILES_ATTRIBUTION =
|
||||||
|
'<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
|
||||||
|
const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
|
||||||
|
|
||||||
|
export class MapService {
|
||||||
|
private readonly mapStoragePath = '/storage/maps'
|
||||||
|
private readonly baseStylesFile = 'nomad-base-styles.json'
|
||||||
|
private readonly basemapsAssetsDir = 'basemaps-assets'
|
||||||
|
private readonly baseAssetsTarFile = 'base-assets.tar.gz'
|
||||||
|
private activeDownloads = new Map<string, AbortController>()
|
||||||
|
|
||||||
|
async listRegions() {
|
||||||
|
const files = (await this.listAllMapStorageItems()).filter(
|
||||||
|
(item) => item.type === 'file' && item.name.endsWith('.pmtiles')
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
files,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async downloadBaseAssets(url?: string) {
|
||||||
|
const tempTarPath = join(process.cwd(), this.mapStoragePath, this.baseAssetsTarFile)
|
||||||
|
|
||||||
|
const defaultTarFileURL = new URL(
|
||||||
|
this.baseAssetsTarFile,
|
||||||
|
'https://github.com/Crosstalk-Solutions/project-nomad-maps/blob/master'
|
||||||
|
)
|
||||||
|
defaultTarFileURL.searchParams.append('raw', 'true')
|
||||||
|
|
||||||
|
const resolvedURL = url ? new URL(url) : defaultTarFileURL
|
||||||
|
await doResumableDownloadWithRetry({
|
||||||
|
url: resolvedURL.toString(),
|
||||||
|
path: tempTarPath,
|
||||||
|
timeout: 30000,
|
||||||
|
max_retries: 2,
|
||||||
|
allowedMimeTypes: BASE_ASSETS_MIME_TYPES,
|
||||||
|
onProgress(progress) {
|
||||||
|
console.log(
|
||||||
|
`Downloading: ${progress.downloadedBytes.toFixed(2)}b / ${progress.totalBytes.toFixed(2)}b`
|
||||||
|
)
|
||||||
|
},
|
||||||
|
onAttemptError(error, attempt) {
|
||||||
|
console.error(`Attempt ${attempt} to download tar file failed: ${error.message}`)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const tarFileBuffer = await getFileStatsIfExists(tempTarPath)
|
||||||
|
if (!tarFileBuffer) {
|
||||||
|
throw new Error(`Failed to download tar file`)
|
||||||
|
}
|
||||||
|
|
||||||
|
await extract({
|
||||||
|
cwd: join(process.cwd(), this.mapStoragePath),
|
||||||
|
file: tempTarPath,
|
||||||
|
strip: 1,
|
||||||
|
})
|
||||||
|
|
||||||
|
await deleteFileIfExists(tempTarPath)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
async downloadRemote(url: string): Promise<string> {
|
||||||
|
const parsed = new URL(url)
|
||||||
|
if (!parsed.pathname.endsWith('.pmtiles')) {
|
||||||
|
throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = this.activeDownloads.get(url)
|
||||||
|
if (existing) {
|
||||||
|
throw new Error(`Download already in progress for URL ${url}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const filename = url.split('/').pop()
|
||||||
|
if (!filename) {
|
||||||
|
throw new Error('Could not determine filename from URL')
|
||||||
|
}
|
||||||
|
|
||||||
|
const path = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
|
||||||
|
|
||||||
|
// Don't await the download, run it in the background
|
||||||
|
doBackgroundDownload({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
timeout: 30000,
|
||||||
|
allowedMimeTypes: PMTILES_MIME_TYPES,
|
||||||
|
forceNew: true,
|
||||||
|
channel: BROADCAST_CHANNEL,
|
||||||
|
activeDownloads: this.activeDownloads,
|
||||||
|
})
|
||||||
|
|
||||||
|
return filename
|
||||||
|
}
|
||||||
|
|
||||||
|
async generateStylesJSON() {
|
||||||
|
if (!(await this.checkBaseAssetsExist())) {
|
||||||
|
throw new Error('Base map assets are missing from storage/maps')
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseStylePath = join(process.cwd(), this.mapStoragePath, this.baseStylesFile)
|
||||||
|
const baseStyle = await getFile(baseStylePath, 'string')
|
||||||
|
if (!baseStyle) {
|
||||||
|
throw new Error('Base styles file not found in storage/maps')
|
||||||
|
}
|
||||||
|
|
||||||
|
const localUrl = env.get('URL')
|
||||||
|
const rawStyles = JSON.parse(baseStyle.toString()) as BaseStylesFile
|
||||||
|
|
||||||
|
const regions = (await this.listRegions()).files
|
||||||
|
const sources = this.generateSourcesArray(regions)
|
||||||
|
|
||||||
|
const baseUrl = urlJoin(localUrl, this.mapStoragePath, this.basemapsAssetsDir)
|
||||||
|
|
||||||
|
const styles = await this.generateStylesFile(
|
||||||
|
rawStyles,
|
||||||
|
sources,
|
||||||
|
urlJoin(baseUrl, 'sprites/v4/light'),
|
||||||
|
urlJoin(baseUrl, 'fonts/{fontstack}/{range}.pbf')
|
||||||
|
)
|
||||||
|
|
||||||
|
return styles
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkBaseAssetsExist() {
|
||||||
|
const storageContents = await this.listMapStorageItems()
|
||||||
|
const baseStyleItem = storageContents.find(
|
||||||
|
(item) => item.type === 'file' && item.name === this.baseStylesFile
|
||||||
|
)
|
||||||
|
const basemapsAssetsItem = storageContents.find(
|
||||||
|
(item) => item.type === 'directory' && item.name === this.basemapsAssetsDir
|
||||||
|
)
|
||||||
|
|
||||||
|
return !!baseStyleItem && !!basemapsAssetsItem
|
||||||
|
}
|
||||||
|
|
||||||
|
private async listMapStorageItems(): Promise<FileEntry[]> {
|
||||||
|
const dirPath = join(process.cwd(), this.mapStoragePath)
|
||||||
|
return await listDirectoryContents(dirPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
private async listAllMapStorageItems(): Promise<FileEntry[]> {
|
||||||
|
const dirPath = join(process.cwd(), this.mapStoragePath)
|
||||||
|
return await listDirectoryContentsRecursive(dirPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
private generateSourcesArray(regions: FileEntry[]): BaseStylesFile['sources'][] {
|
||||||
|
const localUrl = env.get('URL')
|
||||||
|
const sources: BaseStylesFile['sources'][] = []
|
||||||
|
|
||||||
|
for (const region of regions) {
|
||||||
|
if (region.type === 'file' && region.name.endsWith('.pmtiles')) {
|
||||||
|
const regionName = region.name.replace('.pmtiles', '')
|
||||||
|
const source: BaseStylesFile['sources'] = {}
|
||||||
|
source[regionName] = {
|
||||||
|
type: 'vector',
|
||||||
|
attribution: PMTILES_ATTRIBUTION,
|
||||||
|
url: `pmtiles://http://${urlJoin(localUrl, this.mapStoragePath, 'pmtiles', region.name)}`,
|
||||||
|
}
|
||||||
|
sources.push(source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sources
|
||||||
|
}
|
||||||
|
|
||||||
|
private async generateStylesFile(
|
||||||
|
template: BaseStylesFile,
|
||||||
|
sources: BaseStylesFile['sources'][],
|
||||||
|
sprites: string,
|
||||||
|
glyphs: string
|
||||||
|
): Promise<BaseStylesFile> {
|
||||||
|
const layersTemplates = template.layers.filter((layer) => layer.source)
|
||||||
|
const withoutSources = template.layers.filter((layer) => !layer.source)
|
||||||
|
|
||||||
|
template.sources = {} // Clear existing sources
|
||||||
|
template.layers = [...withoutSources] // Start with layers that don't depend on sources
|
||||||
|
|
||||||
|
for (const source of sources) {
|
||||||
|
for (const layerTemplate of layersTemplates) {
|
||||||
|
const layer: MapLayer = {
|
||||||
|
...layerTemplate,
|
||||||
|
id: `${layerTemplate.id}-${Object.keys(source)[0]}`,
|
||||||
|
type: layerTemplate.type,
|
||||||
|
source: Object.keys(source)[0],
|
||||||
|
}
|
||||||
|
template.layers.push(layer)
|
||||||
|
}
|
||||||
|
|
||||||
|
template.sources = Object.assign(template.sources, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
template.sprite = sprites
|
||||||
|
template.glyphs = glyphs
|
||||||
|
|
||||||
|
return template
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(file: string): Promise<void> {
|
||||||
|
let fileName = file
|
||||||
|
if (!fileName.endsWith('.zim')) {
|
||||||
|
fileName += '.zim'
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullPath = join(process.cwd(), this.mapStoragePath, fileName)
|
||||||
|
|
||||||
|
const exists = await getFileStatsIfExists(fullPath)
|
||||||
|
if (!exists) {
|
||||||
|
throw new Error('not_found')
|
||||||
|
}
|
||||||
|
|
||||||
|
await deleteFileIfExists(fullPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,90 +1,100 @@
|
||||||
import drive from "@adonisjs/drive/services/main";
|
import {
|
||||||
import { DownloadOptions, DownloadProgress, ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, ZimFilesEntry } from "../../types/zim.js";
|
ListRemoteZimFilesResponse,
|
||||||
import axios from "axios";
|
RawRemoteZimFileEntry,
|
||||||
|
RemoteZimFileEntry,
|
||||||
|
} from '../../types/zim.js'
|
||||||
|
import axios from 'axios'
|
||||||
import { XMLParser } from 'fast-xml-parser'
|
import { XMLParser } from 'fast-xml-parser'
|
||||||
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from "../../util/zim.js";
|
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
|
||||||
import transmit from "@adonisjs/transmit/services/main";
|
import transmit from '@adonisjs/transmit/services/main'
|
||||||
import { Transform } from "stream";
|
import logger from '@adonisjs/core/services/logger'
|
||||||
import logger from "@adonisjs/core/services/logger";
|
import { DockerService } from './docker_service.js'
|
||||||
import { DockerService } from "./docker_service.js";
|
import { inject } from '@adonisjs/core'
|
||||||
import { inject } from "@adonisjs/core";
|
import { doBackgroundDownload } from '../utils/downloads.js'
|
||||||
|
import {
|
||||||
|
deleteFileIfExists,
|
||||||
|
ensureDirectoryExists,
|
||||||
|
getFileStatsIfExists,
|
||||||
|
listDirectoryContents,
|
||||||
|
} from '../utils/fs.js'
|
||||||
|
import { join } from 'path'
|
||||||
|
|
||||||
|
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
|
||||||
|
const BROADCAST_CHANNEL = 'zim-downloads'
|
||||||
|
|
||||||
@inject()
|
@inject()
|
||||||
export class ZimService {
|
export class ZimService {
|
||||||
private activeDownloads = new Map<string, AbortController>();
|
private zimStoragePath = '/storage/zim'
|
||||||
|
private activeDownloads = new Map<string, AbortController>()
|
||||||
|
|
||||||
constructor(
|
constructor(private dockerService: DockerService) {}
|
||||||
private dockerService: DockerService
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async list() {
|
async list() {
|
||||||
const disk = drive.use('fs');
|
const dirPath = join(process.cwd(), this.zimStoragePath)
|
||||||
const contents = await disk.listAll('/zim')
|
|
||||||
|
|
||||||
const files: ZimFilesEntry[] = []
|
await ensureDirectoryExists(dirPath)
|
||||||
for (let item of contents.objects) {
|
|
||||||
if (item.isFile) {
|
const files = await listDirectoryContents(dirPath)
|
||||||
files.push({
|
|
||||||
type: 'file',
|
|
||||||
key: item.key,
|
|
||||||
name: item.name
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
files.push({
|
|
||||||
type: 'directory',
|
|
||||||
prefix: item.prefix,
|
|
||||||
name: item.name
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
files,
|
files,
|
||||||
next: contents.paginationToken
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async listRemote({ start, count }: { start: number, count: number }): Promise<ListRemoteZimFilesResponse> {
|
async listRemote({
|
||||||
|
start,
|
||||||
|
count,
|
||||||
|
}: {
|
||||||
|
start: number
|
||||||
|
count: number
|
||||||
|
}): Promise<ListRemoteZimFilesResponse> {
|
||||||
const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
|
const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
|
||||||
|
|
||||||
const res = await axios.get(LIBRARY_BASE_URL, {
|
const res = await axios.get(LIBRARY_BASE_URL, {
|
||||||
params: {
|
params: {
|
||||||
start: start,
|
start: start,
|
||||||
count: count,
|
count: count,
|
||||||
lang: 'eng'
|
lang: 'eng',
|
||||||
},
|
},
|
||||||
responseType: 'text'
|
responseType: 'text',
|
||||||
});
|
})
|
||||||
|
|
||||||
const data = res.data;
|
const data = res.data
|
||||||
const parser = new XMLParser({
|
const parser = new XMLParser({
|
||||||
ignoreAttributes: false,
|
ignoreAttributes: false,
|
||||||
attributeNamePrefix: '',
|
attributeNamePrefix: '',
|
||||||
textNodeName: '#text',
|
textNodeName: '#text',
|
||||||
});
|
})
|
||||||
const result = parser.parse(data);
|
const result = parser.parse(data)
|
||||||
|
|
||||||
if (!isRawListRemoteZimFilesResponse(result)) {
|
if (!isRawListRemoteZimFilesResponse(result)) {
|
||||||
throw new Error('Invalid response format from remote library');
|
throw new Error('Invalid response format from remote library')
|
||||||
}
|
}
|
||||||
|
|
||||||
const filtered = result.feed.entry.filter((entry: any) => {
|
const filtered = result.feed.entry.filter((entry: any) => {
|
||||||
return isRawRemoteZimFileEntry(entry);
|
return isRawRemoteZimFileEntry(entry)
|
||||||
})
|
})
|
||||||
|
|
||||||
const mapped: (RemoteZimFileEntry | null)[] = filtered.map((entry: RawRemoteZimFileEntry) => {
|
const mapped: (RemoteZimFileEntry | null)[] = filtered.map((entry: RawRemoteZimFileEntry) => {
|
||||||
const downloadLink = entry.link.find((link: any) => {
|
const downloadLink = entry.link.find((link: any) => {
|
||||||
return typeof link === 'object' && 'rel' in link && 'length' in link && 'href' in link && 'type' in link && link.type === 'application/x-zim'
|
return (
|
||||||
});
|
typeof link === 'object' &&
|
||||||
|
'rel' in link &&
|
||||||
|
'length' in link &&
|
||||||
|
'href' in link &&
|
||||||
|
'type' in link &&
|
||||||
|
link.type === 'application/x-zim'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
if (!downloadLink) {
|
if (!downloadLink) {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
// downloadLink['href'] will end with .meta4, we need to remove that to get the actual download URL
|
// downloadLink['href'] will end with .meta4, we need to remove that to get the actual download URL
|
||||||
const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6);
|
const download_url = downloadLink['href'].substring(0, downloadLink['href'].length - 6)
|
||||||
const file_name = download_url.split('/').pop() || `${entry.title}.zim`;
|
const file_name = download_url.split('/').pop() || `${entry.title}.zim`
|
||||||
const sizeBytes = parseInt(downloadLink['length'], 10);
|
const sizeBytes = parseInt(downloadLink['length'], 10)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id: entry.id,
|
id: entry.id,
|
||||||
|
|
@ -94,281 +104,96 @@ export class ZimService {
|
||||||
size_bytes: sizeBytes || 0,
|
size_bytes: sizeBytes || 0,
|
||||||
download_url: download_url,
|
download_url: download_url,
|
||||||
author: entry.author.name,
|
author: entry.author.name,
|
||||||
file_name: file_name
|
file_name: file_name,
|
||||||
}
|
}
|
||||||
});
|
})
|
||||||
|
|
||||||
// Filter out any null entries (those without a valid download link)
|
// Filter out any null entries (those without a valid download link)
|
||||||
// or files that already exist in the local storage
|
// or files that already exist in the local storage
|
||||||
const existing = await this.list();
|
const existing = await this.list()
|
||||||
const existingKeys = new Set(existing.files.map(file => file.name));
|
const existingKeys = new Set(existing.files.map((file) => file.name))
|
||||||
const withoutExisting = mapped.filter((entry): entry is RemoteZimFileEntry => entry !== null && !existingKeys.has(entry.file_name));
|
const withoutExisting = mapped.filter(
|
||||||
|
(entry): entry is RemoteZimFileEntry => entry !== null && !existingKeys.has(entry.file_name)
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
items: withoutExisting,
|
items: withoutExisting,
|
||||||
has_more: result.feed.totalResults > start,
|
has_more: result.feed.totalResults > start,
|
||||||
total_count: result.feed.totalResults,
|
total_count: result.feed.totalResults,
|
||||||
};
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadRemote(url: string, opts: DownloadOptions = {}): Promise<string> {
|
async downloadRemote(url: string): Promise<string> {
|
||||||
if (!url.endsWith('.zim')) {
|
const parsed = new URL(url)
|
||||||
throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`);
|
if (!parsed.pathname.endsWith('.zim')) {
|
||||||
|
throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`)
|
||||||
}
|
}
|
||||||
|
|
||||||
const existing = this.activeDownloads.get(url);
|
const existing = this.activeDownloads.get(url)
|
||||||
if (existing) {
|
if (existing) {
|
||||||
throw new Error(`Download already in progress for URL ${url}`);
|
throw new Error(`Download already in progress for URL ${url}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await ensureDirectoryExists(join(process.cwd(), this.zimStoragePath))
|
||||||
|
|
||||||
// Extract the filename from the URL
|
// Extract the filename from the URL
|
||||||
const filename = url.split('/').pop() || `downloaded-${Date.now()}.zim`;
|
const filename = url.split('/').pop()
|
||||||
const path = `/zim/${filename}`;
|
if (!filename) {
|
||||||
|
throw new Error('Could not determine filename from URL')
|
||||||
|
}
|
||||||
|
|
||||||
this._runDownload(url, path, opts); // Don't await - let run in background
|
const path = join(process.cwd(), this.zimStoragePath, filename)
|
||||||
|
|
||||||
return filename;
|
// Don't await the download, run it in the background
|
||||||
|
doBackgroundDownload({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
channel: BROADCAST_CHANNEL,
|
||||||
|
activeDownloads: this.activeDownloads,
|
||||||
|
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||||
|
timeout: 30000,
|
||||||
|
forceNew: true,
|
||||||
|
onComplete: async () => {
|
||||||
|
// Restart KIWIX container to pick up new ZIM file
|
||||||
|
await this.dockerService
|
||||||
|
.affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
|
||||||
|
.catch((error) => {
|
||||||
|
logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
|
||||||
|
})
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return filename
|
||||||
}
|
}
|
||||||
|
|
||||||
getActiveDownloads(): string[] {
|
getActiveDownloads(): string[] {
|
||||||
return Array.from(this.activeDownloads.keys());
|
return Array.from(this.activeDownloads.keys())
|
||||||
}
|
}
|
||||||
|
|
||||||
cancelDownload(url: string): boolean {
|
cancelDownload(url: string): boolean {
|
||||||
const entry = this.activeDownloads.get(url);
|
const entry = this.activeDownloads.get(url)
|
||||||
if (entry) {
|
if (entry) {
|
||||||
entry.abort();
|
entry.abort()
|
||||||
this.activeDownloads.delete(url);
|
this.activeDownloads.delete(url)
|
||||||
transmit.broadcast(`zim-downloads`, { url, status: 'cancelled' });
|
transmit.broadcast(BROADCAST_CHANNEL, { url, status: 'cancelled' })
|
||||||
return true;
|
return true
|
||||||
}
|
}
|
||||||
return false;
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
async delete(key: string): Promise<void> {
|
async delete(file: string): Promise<void> {
|
||||||
let fileName = key;
|
let fileName = file
|
||||||
if (!fileName.endsWith('.zim')) {
|
if (!fileName.endsWith('.zim')) {
|
||||||
fileName += '.zim';
|
fileName += '.zim'
|
||||||
}
|
}
|
||||||
|
|
||||||
const disk = drive.use('fs');
|
const fullPath = join(process.cwd(), this.zimStoragePath, fileName)
|
||||||
const exists = await disk.exists(fileName);
|
|
||||||
|
|
||||||
|
const exists = await getFileStatsIfExists(fullPath)
|
||||||
if (!exists) {
|
if (!exists) {
|
||||||
throw new Error('not_found');
|
throw new Error('not_found')
|
||||||
}
|
}
|
||||||
|
|
||||||
await disk.delete(fileName);
|
await deleteFileIfExists(fullPath)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
private async _runDownload(url: string, path: string, opts: DownloadOptions = {}): Promise<string> {
|
|
||||||
try {
|
|
||||||
const {
|
|
||||||
max_retries = 3,
|
|
||||||
retry_delay = 2000,
|
|
||||||
timeout = 30000,
|
|
||||||
onError,
|
|
||||||
}: DownloadOptions = opts;
|
|
||||||
|
|
||||||
let attempt = 0;
|
|
||||||
while (attempt < max_retries) {
|
|
||||||
try {
|
|
||||||
const abortController = new AbortController();
|
|
||||||
this.activeDownloads.set(url, abortController);
|
|
||||||
|
|
||||||
await this._attemptDownload(
|
|
||||||
url,
|
|
||||||
path,
|
|
||||||
abortController.signal,
|
|
||||||
timeout,
|
|
||||||
);
|
|
||||||
|
|
||||||
transmit.broadcast('zim-downloads', { url, path, status: 'completed', progress: { downloaded_bytes: 0, total_bytes: 0, percentage: 100, speed: '0 B/s', time_remaining: 0 } });
|
|
||||||
await this.dockerService.affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart').catch((error) => {
|
|
||||||
logger.error(`Failed to restart KIWIX container:`, error); // Don't stop the download completion, just log the error.
|
|
||||||
});
|
|
||||||
|
|
||||||
break; // Exit loop on success
|
|
||||||
} catch (error) {
|
|
||||||
attempt++;
|
|
||||||
|
|
||||||
const isAborted = error.name === 'AbortError' || error.code === 'ABORT_ERR';
|
|
||||||
const isNetworkError = error.code === 'ECONNRESET' ||
|
|
||||||
error.code === 'ENOTFOUND' ||
|
|
||||||
error.code === 'ETIMEDOUT';
|
|
||||||
|
|
||||||
onError?.(error);
|
|
||||||
if (isAborted) {
|
|
||||||
throw new Error(`Download aborted for URL: ${url}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attempt < max_retries && isNetworkError) {
|
|
||||||
await this.delay(retry_delay);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
logger.error(`Failed to download ${url}:`, error);
|
|
||||||
transmit.broadcast('zim-downloads', { url, error: error.message, status: 'failed' });
|
|
||||||
} finally {
|
|
||||||
this.activeDownloads.delete(url);
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async _attemptDownload(
|
|
||||||
url: string,
|
|
||||||
path: string,
|
|
||||||
signal: AbortSignal,
|
|
||||||
timeout: number,
|
|
||||||
): Promise<string> {
|
|
||||||
const disk = drive.use('fs');
|
|
||||||
|
|
||||||
// Check if partial file exists for resume
|
|
||||||
let startByte = 0;
|
|
||||||
let appendMode = false;
|
|
||||||
|
|
||||||
if (await disk.exists(path)) {
|
|
||||||
const stats = await disk.getMetaData(path);
|
|
||||||
startByte = stats.contentLength;
|
|
||||||
appendMode = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get file info with HEAD request first
|
|
||||||
const headResponse = await axios.head(url, {
|
|
||||||
signal,
|
|
||||||
timeout
|
|
||||||
});
|
|
||||||
|
|
||||||
const totalBytes = parseInt(headResponse.headers['content-length'] || '0');
|
|
||||||
const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes';
|
|
||||||
|
|
||||||
// If file is already complete
|
|
||||||
if (startByte === totalBytes && totalBytes > 0) {
|
|
||||||
logger.info(`File ${path} is already complete`);
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If server doesn't support range requests and we have a partial file, delete it
|
|
||||||
if (!supportsRangeRequests && startByte > 0) {
|
|
||||||
await disk.delete(path);
|
|
||||||
startByte = 0;
|
|
||||||
appendMode = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const headers: Record<string, string> = {};
|
|
||||||
if (supportsRangeRequests && startByte > 0) {
|
|
||||||
headers.Range = `bytes=${startByte}-`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await axios.get(url, {
|
|
||||||
responseType: 'stream',
|
|
||||||
headers,
|
|
||||||
signal,
|
|
||||||
timeout
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status !== 200 && response.status !== 206) {
|
|
||||||
throw new Error(`Failed to download: HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
let downloadedBytes = startByte;
|
|
||||||
let lastProgressTime = Date.now();
|
|
||||||
let lastDownloadedBytes = startByte;
|
|
||||||
|
|
||||||
// Progress tracking stream to monitor data flow
|
|
||||||
const progressStream = new Transform({
|
|
||||||
transform(chunk: Buffer, _: any, callback: Function) {
|
|
||||||
downloadedBytes += chunk.length;
|
|
||||||
this.push(chunk);
|
|
||||||
callback();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Update progress every 500ms
|
|
||||||
const progressInterval = setInterval(() => {
|
|
||||||
this.updateProgress({
|
|
||||||
downloadedBytes,
|
|
||||||
totalBytes,
|
|
||||||
lastProgressTime,
|
|
||||||
lastDownloadedBytes,
|
|
||||||
url
|
|
||||||
});
|
|
||||||
}, 500);
|
|
||||||
|
|
||||||
// Handle errors and cleanup
|
|
||||||
const cleanup = (error?: Error) => {
|
|
||||||
clearInterval(progressInterval);
|
|
||||||
progressStream.destroy();
|
|
||||||
response.data.destroy();
|
|
||||||
if (error) {
|
|
||||||
reject(error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
response.data.on('error', cleanup);
|
|
||||||
progressStream.on('error', cleanup);
|
|
||||||
|
|
||||||
signal.addEventListener('abort', () => {
|
|
||||||
cleanup(new Error('Download aborted'));
|
|
||||||
});
|
|
||||||
|
|
||||||
// Pipe through progress stream and then to disk
|
|
||||||
const sourceStream = response.data.pipe(progressStream);
|
|
||||||
|
|
||||||
// Use disk.putStream with append mode for resumable downloads
|
|
||||||
disk.putStream(path, sourceStream, { append: appendMode })
|
|
||||||
.then(() => {
|
|
||||||
clearInterval(progressInterval);
|
|
||||||
resolve(path);
|
|
||||||
})
|
|
||||||
.catch(cleanup);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateProgress({
|
|
||||||
downloadedBytes,
|
|
||||||
totalBytes,
|
|
||||||
lastProgressTime,
|
|
||||||
lastDownloadedBytes,
|
|
||||||
url
|
|
||||||
}: {
|
|
||||||
downloadedBytes: number;
|
|
||||||
totalBytes: number;
|
|
||||||
lastProgressTime: number;
|
|
||||||
lastDownloadedBytes: number;
|
|
||||||
url: string;
|
|
||||||
}) {
|
|
||||||
const now = Date.now();
|
|
||||||
const timeDiff = (now - lastProgressTime) / 1000;
|
|
||||||
const bytesDiff = downloadedBytes - lastDownloadedBytes;
|
|
||||||
const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0;
|
|
||||||
const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0;
|
|
||||||
const speed = this.formatSpeed(rawSpeed);
|
|
||||||
|
|
||||||
const progress: DownloadProgress = {
|
|
||||||
downloaded_bytes: downloadedBytes,
|
|
||||||
total_bytes: totalBytes,
|
|
||||||
percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
|
|
||||||
speed,
|
|
||||||
time_remaining: timeRemaining
|
|
||||||
};
|
|
||||||
|
|
||||||
transmit.broadcast('zim-downloads', { url, progress, status: "in_progress" });
|
|
||||||
|
|
||||||
lastProgressTime = now;
|
|
||||||
lastDownloadedBytes = downloadedBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
private delay(ms: number): Promise<void> {
|
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
private formatSpeed(bytesPerSecond: number): string {
|
|
||||||
if (bytesPerSecond < 1024) return `${bytesPerSecond.toFixed(0)} B/s`;
|
|
||||||
if (bytesPerSecond < 1024 * 1024) return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`;
|
|
||||||
return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
289
admin/app/utils/downloads.ts
Normal file
289
admin/app/utils/downloads.ts
Normal file
|
|
@ -0,0 +1,289 @@
|
||||||
|
import {
|
||||||
|
DoBackgroundDownloadParams,
|
||||||
|
DoResumableDownloadParams,
|
||||||
|
DoResumableDownloadProgress,
|
||||||
|
DoResumableDownloadWithRetryParams,
|
||||||
|
} from '../../types/downloads.js'
|
||||||
|
import axios from 'axios'
|
||||||
|
import { Transform } from 'stream'
|
||||||
|
import { deleteFileIfExists, getFileStatsIfExists } from './fs.js'
|
||||||
|
import { createWriteStream } from 'fs'
|
||||||
|
import { formatSpeed } from './misc.js'
|
||||||
|
import { DownloadProgress } from '../../types/files.js'
|
||||||
|
import transmit from '@adonisjs/transmit/services/main'
|
||||||
|
import logger from '@adonisjs/core/services/logger'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a resumable download with progress tracking
|
||||||
|
* @param param0 - Download parameters. Leave allowedMimeTypes empty to skip mime type checking.
|
||||||
|
* Otherwise, mime types should be in the format "application/pdf", "image/png", etc.
|
||||||
|
* @returns Path to the downloaded file
|
||||||
|
*/
|
||||||
|
export async function doResumableDownload({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
timeout = 30000,
|
||||||
|
signal,
|
||||||
|
onProgress,
|
||||||
|
forceNew = false,
|
||||||
|
allowedMimeTypes,
|
||||||
|
}: DoResumableDownloadParams): Promise<string> {
|
||||||
|
// Check if partial file exists for resume
|
||||||
|
let startByte = 0
|
||||||
|
let appendMode = false
|
||||||
|
|
||||||
|
console.log(`Starting download from ${url} to ${path}`)
|
||||||
|
console.log('Checking for existing file to resume...')
|
||||||
|
const existingStats = await getFileStatsIfExists(path)
|
||||||
|
if (existingStats && !forceNew) {
|
||||||
|
startByte = existingStats.size
|
||||||
|
appendMode = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get file info with HEAD request first
|
||||||
|
const headResponse = await axios.head(url, {
|
||||||
|
signal,
|
||||||
|
timeout,
|
||||||
|
})
|
||||||
|
|
||||||
|
const contentType = headResponse.headers['content-type'] || ''
|
||||||
|
const totalBytes = parseInt(headResponse.headers['content-length'] || '0')
|
||||||
|
const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes'
|
||||||
|
|
||||||
|
// If allowedMimeTypes is provided, check content type
|
||||||
|
if (allowedMimeTypes && allowedMimeTypes.length > 0) {
|
||||||
|
const isMimeTypeAllowed = allowedMimeTypes.some((mimeType) => contentType.includes(mimeType))
|
||||||
|
if (!isMimeTypeAllowed) {
|
||||||
|
throw new Error(`MIME type ${contentType} is not allowed`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If file is already complete and not forcing overwrite just return path
|
||||||
|
if (startByte === totalBytes && totalBytes > 0 && !forceNew) {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
// If server doesn't support range requests and we have a partial file, delete it
|
||||||
|
if (!supportsRangeRequests && startByte > 0) {
|
||||||
|
await deleteFileIfExists(path)
|
||||||
|
startByte = 0
|
||||||
|
appendMode = false
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers: Record<string, string> = {}
|
||||||
|
if (supportsRangeRequests && startByte > 0) {
|
||||||
|
headers.Range = `bytes=${startByte}-`
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await axios.get(url, {
|
||||||
|
responseType: 'stream',
|
||||||
|
headers,
|
||||||
|
signal,
|
||||||
|
timeout,
|
||||||
|
})
|
||||||
|
|
||||||
|
if (response.status !== 200 && response.status !== 206) {
|
||||||
|
throw new Error(`Failed to download: HTTP ${response.status}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let downloadedBytes = startByte
|
||||||
|
let lastProgressTime = Date.now()
|
||||||
|
let lastDownloadedBytes = startByte
|
||||||
|
|
||||||
|
// Progress tracking stream to monitor data flow
|
||||||
|
const progressStream = new Transform({
|
||||||
|
transform(chunk: Buffer, _: any, callback: Function) {
|
||||||
|
downloadedBytes += chunk.length
|
||||||
|
|
||||||
|
// Update progress tracking
|
||||||
|
const now = Date.now()
|
||||||
|
if (onProgress && now - lastProgressTime >= 500) {
|
||||||
|
lastProgressTime = now
|
||||||
|
lastDownloadedBytes = downloadedBytes
|
||||||
|
onProgress({
|
||||||
|
downloadedBytes,
|
||||||
|
totalBytes,
|
||||||
|
lastProgressTime,
|
||||||
|
lastDownloadedBytes,
|
||||||
|
url,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
this.push(chunk)
|
||||||
|
callback()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const writeStream = createWriteStream(path, {
|
||||||
|
flags: appendMode ? 'a' : 'w',
|
||||||
|
})
|
||||||
|
|
||||||
|
// Handle errors and cleanup
|
||||||
|
const cleanup = (error?: Error) => {
|
||||||
|
progressStream.destroy()
|
||||||
|
response.data.destroy()
|
||||||
|
writeStream.destroy()
|
||||||
|
if (error) {
|
||||||
|
reject(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
response.data.on('error', cleanup)
|
||||||
|
progressStream.on('error', cleanup)
|
||||||
|
writeStream.on('error', cleanup)
|
||||||
|
writeStream.on('error', cleanup)
|
||||||
|
|
||||||
|
signal?.addEventListener('abort', () => {
|
||||||
|
cleanup(new Error('Download aborted'))
|
||||||
|
})
|
||||||
|
|
||||||
|
writeStream.on('finish', () => {
|
||||||
|
if (onProgress) {
|
||||||
|
onProgress({
|
||||||
|
downloadedBytes,
|
||||||
|
totalBytes,
|
||||||
|
lastProgressTime: Date.now(),
|
||||||
|
lastDownloadedBytes: downloadedBytes,
|
||||||
|
url,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
resolve(path)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Pipe: response -> progressStream -> writeStream
|
||||||
|
response.data.pipe(progressStream).pipe(writeStream)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function doResumableDownloadWithRetry({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
signal,
|
||||||
|
timeout = 30000,
|
||||||
|
onProgress,
|
||||||
|
max_retries = 3,
|
||||||
|
retry_delay = 2000,
|
||||||
|
onAttemptError,
|
||||||
|
allowedMimeTypes,
|
||||||
|
}: DoResumableDownloadWithRetryParams): Promise<string> {
|
||||||
|
let attempt = 0
|
||||||
|
let lastError: Error | null = null
|
||||||
|
|
||||||
|
while (attempt < max_retries) {
|
||||||
|
try {
|
||||||
|
const result = await doResumableDownload({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
signal,
|
||||||
|
timeout,
|
||||||
|
allowedMimeTypes,
|
||||||
|
onProgress,
|
||||||
|
})
|
||||||
|
|
||||||
|
return result // return on success
|
||||||
|
} catch (error) {
|
||||||
|
attempt++
|
||||||
|
lastError = error as Error
|
||||||
|
|
||||||
|
const isAborted = error.name === 'AbortError' || error.code === 'ABORT_ERR'
|
||||||
|
const isNetworkError =
|
||||||
|
error.code === 'ECONNRESET' || error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT'
|
||||||
|
|
||||||
|
onAttemptError?.(error, attempt)
|
||||||
|
if (isAborted) {
|
||||||
|
throw new Error(`Download aborted for URL: ${url}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < max_retries && isNetworkError) {
|
||||||
|
await delay(retry_delay)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// If max retries reached or non-retriable error, throw
|
||||||
|
if (attempt >= max_retries || !isNetworkError) {
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// should not reach here, but TypeScript needs a return
|
||||||
|
throw lastError || new Error('Unknown error during download')
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise<void> {
|
||||||
|
const { url, path, channel, activeDownloads, onComplete, ...restParams } = params
|
||||||
|
|
||||||
|
try {
|
||||||
|
const abortController = new AbortController()
|
||||||
|
activeDownloads.set(url, abortController)
|
||||||
|
|
||||||
|
await doResumableDownloadWithRetry({
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
signal: abortController.signal,
|
||||||
|
...restParams,
|
||||||
|
onProgress: (progressData) => {
|
||||||
|
sendProgressBroadcast(channel, progressData)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
sendCompletedBroadcast(channel, url, path)
|
||||||
|
|
||||||
|
if (onComplete) {
|
||||||
|
await onComplete(url, path)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(`Background download failed for ${url}: ${error.message}`)
|
||||||
|
sendErrorBroadcast(channel, url, error.message)
|
||||||
|
} finally {
|
||||||
|
activeDownloads.delete(url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sendProgressBroadcast(
|
||||||
|
channel: string,
|
||||||
|
progressData: DoResumableDownloadProgress,
|
||||||
|
status = 'in_progress'
|
||||||
|
) {
|
||||||
|
const { downloadedBytes, totalBytes, lastProgressTime, lastDownloadedBytes, url } = progressData
|
||||||
|
const now = Date.now()
|
||||||
|
const timeDiff = (now - lastProgressTime) / 1000
|
||||||
|
const bytesDiff = downloadedBytes - lastDownloadedBytes
|
||||||
|
const rawSpeed = timeDiff > 0 ? bytesDiff / timeDiff : 0
|
||||||
|
const timeRemaining = rawSpeed > 0 ? (totalBytes - downloadedBytes) / rawSpeed : 0
|
||||||
|
const speed = formatSpeed(rawSpeed)
|
||||||
|
|
||||||
|
const progress: DownloadProgress = {
|
||||||
|
downloaded_bytes: downloadedBytes,
|
||||||
|
total_bytes: totalBytes,
|
||||||
|
percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
|
||||||
|
speed,
|
||||||
|
time_remaining: timeRemaining,
|
||||||
|
}
|
||||||
|
|
||||||
|
transmit.broadcast(channel, { url, progress, status })
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sendCompletedBroadcast(channel: string, url: string, path: string) {
|
||||||
|
transmit.broadcast(channel, {
|
||||||
|
url,
|
||||||
|
path,
|
||||||
|
status: 'completed',
|
||||||
|
progress: {
|
||||||
|
downloaded_bytes: 0,
|
||||||
|
total_bytes: 0,
|
||||||
|
percentage: 100,
|
||||||
|
speed: '0 B/s',
|
||||||
|
time_remaining: 0,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sendErrorBroadcast(channel: string, url: string, errorMessage: string) {
|
||||||
|
transmit.broadcast(channel, { url, error: errorMessage, status: 'failed' })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||||
|
}
|
||||||
115
admin/app/utils/fs.ts
Normal file
115
admin/app/utils/fs.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
import { mkdir, readdir, readFile, stat, unlink } from 'fs/promises'
|
||||||
|
import { join } from 'path'
|
||||||
|
import { DriveDisks } from '@adonisjs/drive/types'
|
||||||
|
import driveConfig from '#config/drive'
|
||||||
|
import app from '@adonisjs/core/services/app'
|
||||||
|
import { FileEntry } from '../../types/files.js'
|
||||||
|
|
||||||
|
export async function listDirectoryContents(path: string): Promise<FileEntry[]> {
|
||||||
|
const entries = await readdir(path, { withFileTypes: true })
|
||||||
|
const results: FileEntry[] = []
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (entry.isFile()) {
|
||||||
|
results.push({
|
||||||
|
type: 'file',
|
||||||
|
key: join(path, entry.name),
|
||||||
|
name: entry.name,
|
||||||
|
})
|
||||||
|
} else if (entry.isDirectory()) {
|
||||||
|
results.push({
|
||||||
|
type: 'directory',
|
||||||
|
prefix: join(path, entry.name),
|
||||||
|
name: entry.name,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listDirectoryContentsRecursive(path: string): Promise<FileEntry[]> {
|
||||||
|
let results: FileEntry[] = []
|
||||||
|
const entries = await readdir(path, { withFileTypes: true })
|
||||||
|
for (const entry of entries) {
|
||||||
|
const fullPath = join(path, entry.name)
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
const subdirectoryContents = await listDirectoryContentsRecursive(fullPath)
|
||||||
|
results = results.concat(subdirectoryContents)
|
||||||
|
} else {
|
||||||
|
results.push({
|
||||||
|
type: 'file',
|
||||||
|
key: fullPath,
|
||||||
|
name: entry.name,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function ensureDirectoryExists(path: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await stat(path)
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') {
|
||||||
|
await mkdir(path, { recursive: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getFile(path: string, returnType: 'buffer'): Promise<Buffer | null>
|
||||||
|
export async function getFile(path: string, returnType: 'string'): Promise<string | null>
|
||||||
|
export async function getFile(path: string, returnType: 'buffer' | 'string' = 'buffer'): Promise<Buffer | string | null> {
|
||||||
|
try {
|
||||||
|
if (returnType === 'buffer') {
|
||||||
|
return await readFile(path)
|
||||||
|
} else {
|
||||||
|
return await readFile(path, 'utf-8')
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getFileStatsIfExists(
|
||||||
|
path: string
|
||||||
|
): Promise<{ size: number; modifiedTime: Date } | null> {
|
||||||
|
try {
|
||||||
|
const stats = await stat(path)
|
||||||
|
return {
|
||||||
|
size: stats.size,
|
||||||
|
modifiedTime: stats.mtime,
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteFileIfExists(path: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await unlink(path)
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code !== 'ENOENT') {
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getFullDrivePath(diskName: keyof DriveDisks): Promise<string> {
|
||||||
|
const config = await driveConfig.resolver(app)
|
||||||
|
const serviceConfig = config.config.services[diskName]
|
||||||
|
const resolved = serviceConfig()
|
||||||
|
if (!resolved) {
|
||||||
|
throw new Error(`Disk ${diskName} not configured`)
|
||||||
|
}
|
||||||
|
|
||||||
|
let path = resolved.options.location
|
||||||
|
if (path instanceof URL) {
|
||||||
|
return path.pathname
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
5
admin/app/utils/misc.ts
Normal file
5
admin/app/utils/misc.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
export function formatSpeed(bytesPerSecond: number): string {
|
||||||
|
if (bytesPerSecond < 1024) return `${bytesPerSecond.toFixed(0)} B/s`
|
||||||
|
if (bytesPerSecond < 1024 * 1024) return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`
|
||||||
|
return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`
|
||||||
|
}
|
||||||
0
admin/app/utils/url.ts
Normal file
0
admin/app/utils/url.ts
Normal file
19
admin/app/validators/common.ts
Normal file
19
admin/app/validators/common.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
import vine from '@vinejs/vine'
|
||||||
|
|
||||||
|
export const remoteDownloadValidator = vine.compile(
|
||||||
|
vine.object({
|
||||||
|
url: vine.string().url().trim(),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
export const remoteDownloadValidatorOptional = vine.compile(
|
||||||
|
vine.object({
|
||||||
|
url: vine.string().url().trim().optional(),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
export const filenameValidator = vine.compile(
|
||||||
|
vine.object({
|
||||||
|
filename: vine.string().trim().minLength(1).maxLength(4096),
|
||||||
|
})
|
||||||
|
)
|
||||||
0
admin/app/validators/zim.ts
Normal file
0
admin/app/validators/zim.ts
Normal file
|
|
@ -2,13 +2,14 @@ import { ExclamationTriangleIcon, XCircleIcon } from '@heroicons/react/24/solid'
|
||||||
import { IconCircleCheck } from '@tabler/icons-react'
|
import { IconCircleCheck } from '@tabler/icons-react'
|
||||||
import classNames from '~/lib/classNames'
|
import classNames from '~/lib/classNames'
|
||||||
|
|
||||||
interface AlertProps extends React.HTMLAttributes<HTMLDivElement> {
|
export type AlertProps = React.HTMLAttributes<HTMLDivElement> & {
|
||||||
title: string
|
title: string
|
||||||
message?: string
|
message?: string
|
||||||
type: 'warning' | 'error' | 'success'
|
type: 'warning' | 'error' | 'success'
|
||||||
|
children?: React.ReactNode
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function Alert({ title, message, type, ...props }: AlertProps) {
|
export default function Alert({ title, message, type, children, ...props }: AlertProps) {
|
||||||
const getIcon = () => {
|
const getIcon = () => {
|
||||||
const Icon =
|
const Icon =
|
||||||
type === 'warning'
|
type === 'warning'
|
||||||
|
|
@ -43,16 +44,19 @@ export default function Alert({ title, message, type, ...props }: AlertProps) {
|
||||||
'border border-gray-200 rounded-md p-3 shadow-xs'
|
'border border-gray-200 rounded-md p-3 shadow-xs'
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
<div className="flex">
|
<div className="flex flex-row justify-between items-center">
|
||||||
<div className="shrink-0">{getIcon()}</div>
|
<div className="flex flex-row">
|
||||||
<div className="ml-3">
|
<div className="shrink-0">{getIcon()}</div>
|
||||||
<h3 className={`text-sm font-medium ${getTextColor()}`}>{title}</h3>
|
<div className="ml-3">
|
||||||
{message && (
|
<h3 className={`text-sm font-medium ${getTextColor()}`}>{title}</h3>
|
||||||
<div className={`mt-2 text-sm ${getTextColor()}`}>
|
{message && (
|
||||||
<p>{message}</p>
|
<div className={`mt-2 text-sm ${getTextColor()}`}>
|
||||||
</div>
|
<p>{message}</p>
|
||||||
)}
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{children}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
|
|
|
||||||
16
admin/inertia/components/AlertWithButton.tsx
Normal file
16
admin/inertia/components/AlertWithButton.tsx
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
import Alert, { AlertProps } from './Alert'
|
||||||
|
import StyledButton, { StyledButtonProps } from './StyledButton'
|
||||||
|
|
||||||
|
export type AlertWithButtonProps = {
|
||||||
|
buttonProps: StyledButtonProps
|
||||||
|
} & AlertProps
|
||||||
|
|
||||||
|
const AlertWithButton = ({ buttonProps, ...alertProps }: AlertWithButtonProps) => {
|
||||||
|
return (
|
||||||
|
<Alert {...alertProps}>
|
||||||
|
<StyledButton {...buttonProps} />
|
||||||
|
</Alert>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default AlertWithButton
|
||||||
22
admin/inertia/components/layout/BackToHomeHeader.tsx
Normal file
22
admin/inertia/components/layout/BackToHomeHeader.tsx
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
import { Link } from '@inertiajs/react'
|
||||||
|
import { IconArrowLeft } from '@tabler/icons-react'
|
||||||
|
import classNames from '~/lib/classNames'
|
||||||
|
|
||||||
|
interface BackToHomeHeaderProps {
|
||||||
|
className?: string
|
||||||
|
children?: React.ReactNode
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function BackToHomeHeader({ className, children }: BackToHomeHeaderProps) {
|
||||||
|
return (
|
||||||
|
<div className={classNames('flex border-b border-gray-900/10 p-4', className)}>
|
||||||
|
<div className="justify-self-start">
|
||||||
|
<Link href="/home" className="flex items-center">
|
||||||
|
<IconArrowLeft className="mr-2" size={24} />
|
||||||
|
<p className="text-lg text-gray-600">Back to Home</p>
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
<div className="flex-grow flex flex-col justify-center">{children}</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
29
admin/inertia/components/layout/MissingBaseAssetsAlert.tsx
Normal file
29
admin/inertia/components/layout/MissingBaseAssetsAlert.tsx
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
import AlertWithButton from "../AlertWithButton"
|
||||||
|
|
||||||
|
export type MissingBaseAssetsAlertProps = {
|
||||||
|
onClickDownload?: () => Promise<void>
|
||||||
|
loading?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
const MissingBaseAssetsAlert = (props: MissingBaseAssetsAlertProps) => {
|
||||||
|
return (
|
||||||
|
<AlertWithButton
|
||||||
|
title="The base map assets have not been installed. Please download them first to enable map functionality."
|
||||||
|
type="warning"
|
||||||
|
className="!mt-6"
|
||||||
|
buttonProps={{
|
||||||
|
variant: 'secondary',
|
||||||
|
children: 'Download Base Assets',
|
||||||
|
icon: 'ArrowDownTrayIcon',
|
||||||
|
loading: props.loading || false,
|
||||||
|
onClick: () => {
|
||||||
|
if (props.onClickDownload) {
|
||||||
|
return props.onClickDownload()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default MissingBaseAssetsAlert
|
||||||
44
admin/inertia/components/maps/MapComponent.tsx
Normal file
44
admin/inertia/components/maps/MapComponent.tsx
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
import Map, { FullscreenControl, NavigationControl, MapProvider } from 'react-map-gl/maplibre'
|
||||||
|
import maplibregl from 'maplibre-gl'
|
||||||
|
import 'maplibre-gl/dist/maplibre-gl.css'
|
||||||
|
import { Protocol } from 'pmtiles'
|
||||||
|
import { useEffect } from 'react'
|
||||||
|
import { useWindowSize } from 'usehooks-ts'
|
||||||
|
|
||||||
|
export default function MapComponent() {
|
||||||
|
const { width = 0, height = 0 } = useWindowSize()
|
||||||
|
|
||||||
|
// Add the PMTiles protocol to maplibre-gl
|
||||||
|
useEffect(() => {
|
||||||
|
let protocol = new Protocol()
|
||||||
|
maplibregl.addProtocol('pmtiles', protocol.tile)
|
||||||
|
return () => {
|
||||||
|
maplibregl.removeProtocol('pmtiles')
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
return (
|
||||||
|
<MapProvider>
|
||||||
|
<Map
|
||||||
|
reuseMaps
|
||||||
|
style={{
|
||||||
|
width: width,
|
||||||
|
height: height - 175,
|
||||||
|
borderRadius: '5px',
|
||||||
|
boxShadow: '0 0 4px rgba(0,0,0,0.3)',
|
||||||
|
backgroundColor: '#fff',
|
||||||
|
}}
|
||||||
|
mapStyle={`http://${window.location.hostname}:${window.location.port}/api/maps/styles`}
|
||||||
|
mapLib={maplibregl}
|
||||||
|
initialViewState={{
|
||||||
|
longitude: -101,
|
||||||
|
latitude: 40,
|
||||||
|
zoom: 3.5,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<NavigationControl />
|
||||||
|
<FullscreenControl />
|
||||||
|
</Map>
|
||||||
|
</MapProvider>
|
||||||
|
)
|
||||||
|
}
|
||||||
13
admin/inertia/hooks/useMapRegionFiles.ts
Normal file
13
admin/inertia/hooks/useMapRegionFiles.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
import { useQuery } from '@tanstack/react-query'
|
||||||
|
import { FileEntry } from '../../types/files'
|
||||||
|
import api from '~/lib/api'
|
||||||
|
|
||||||
|
const useMapRegionFiles = () => {
|
||||||
|
return useQuery<FileEntry[]>({
|
||||||
|
queryKey: ['map-region-files'],
|
||||||
|
queryFn: () => api.listMapRegionFiles(),
|
||||||
|
staleTime: 5 * 60 * 1000, // 5 minutes
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export default useMapRegionFiles
|
||||||
|
|
@ -3,7 +3,7 @@ import Footer from "~/components/Footer";
|
||||||
export default function AppLayout({ children }: { children: React.ReactNode }) {
|
export default function AppLayout({ children }: { children: React.ReactNode }) {
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen flex flex-col">
|
<div className="min-h-screen flex flex-col">
|
||||||
<div className="p-2 flex gap-2 flex-col items-center justify-center">
|
<div className="p-2 flex gap-2 flex-col items-center justify-center cursor-pointer" onClick={() => window.location.href = '/home'}>
|
||||||
<img src="/project_nomad_logo.png" alt="Project Nomad Logo" className="h-40 w-40" />
|
<img src="/project_nomad_logo.png" alt="Project Nomad Logo" className="h-40 w-40" />
|
||||||
<h1 className="text-5xl font-bold text-desert-green">Command Center</h1>
|
<h1 className="text-5xl font-bold text-desert-green">Command Center</h1>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
10
admin/inertia/layouts/MapsLayout.tsx
Normal file
10
admin/inertia/layouts/MapsLayout.tsx
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
import Footer from '~/components/Footer'
|
||||||
|
|
||||||
|
export default function MapsLayout({ children }: { children: React.ReactNode }) {
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen flex flex-col">
|
||||||
|
<div className="flex-1 w-full bg-desert">{children}</div>
|
||||||
|
<Footer />
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -4,13 +4,14 @@ import {
|
||||||
FolderIcon,
|
FolderIcon,
|
||||||
MagnifyingGlassIcon,
|
MagnifyingGlassIcon,
|
||||||
} from '@heroicons/react/24/outline'
|
} from '@heroicons/react/24/outline'
|
||||||
import { IconDashboard, IconGavel } from '@tabler/icons-react'
|
import { IconDashboard, IconGavel, IconMapRoute } from '@tabler/icons-react'
|
||||||
import StyledSidebar from '~/components/StyledSidebar'
|
import StyledSidebar from '~/components/StyledSidebar'
|
||||||
import { getServiceLink } from '~/lib/navigation'
|
import { getServiceLink } from '~/lib/navigation'
|
||||||
|
|
||||||
const navigation = [
|
const navigation = [
|
||||||
{ name: 'Apps', href: '/settings/apps', icon: CommandLineIcon, current: false },
|
{ name: 'Apps', href: '/settings/apps', icon: CommandLineIcon, current: false },
|
||||||
{ name: 'Legal Notices', href: '/settings/legal', icon: IconGavel, current: false },
|
{ name: 'Legal Notices', href: '/settings/legal', icon: IconGavel, current: false },
|
||||||
|
{ name: 'Maps Manager', href: '/settings/maps', icon: IconMapRoute, current: false },
|
||||||
{
|
{
|
||||||
name: 'Service Logs & Metrics',
|
name: 'Service Logs & Metrics',
|
||||||
href: getServiceLink('9999'),
|
href: getServiceLink('9999'),
|
||||||
|
|
|
||||||
|
|
@ -1,96 +1,122 @@
|
||||||
import axios from "axios";
|
import axios from 'axios'
|
||||||
import { ListRemoteZimFilesResponse, ListZimFilesResponse } from "../../types/zim";
|
import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zim'
|
||||||
import { ServiceSlim } from "../../types/services";
|
import { ServiceSlim } from '../../types/services'
|
||||||
|
import { FileEntry } from '../../types/files'
|
||||||
|
|
||||||
class API {
|
class API {
|
||||||
private client;
|
private client
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.client = axios.create({
|
this.client = axios.create({
|
||||||
baseURL: "/api",
|
baseURL: '/api',
|
||||||
headers: {
|
headers: {
|
||||||
"Content-Type": "application/json",
|
'Content-Type': 'application/json',
|
||||||
},
|
},
|
||||||
});
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async listDocs() {
|
||||||
|
try {
|
||||||
|
const response = await this.client.get<Array<{ title: string; slug: string }>>('/docs/list')
|
||||||
|
return response.data
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error listing docs:', error)
|
||||||
|
throw error
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async listDocs() {
|
async listMapRegionFiles() {
|
||||||
try {
|
try {
|
||||||
const response = await this.client.get<Array<{ title: string; slug: string }>>("/docs/list");
|
const response = await this.client.get<{ files: FileEntry[] }>('/maps/regions')
|
||||||
return response.data;
|
return response.data.files
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error listing docs:", error);
|
console.error('Error listing map region files:', error)
|
||||||
throw error;
|
throw error
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async listServices() {
|
async downloadBaseMapAssets() {
|
||||||
try {
|
try {
|
||||||
const response = await this.client.get<Array<ServiceSlim>>("/system/services");
|
const response = await this.client.post<{ success: boolean }>('/maps/download-base-assets')
|
||||||
return response.data;
|
return response.data
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error listing services:", error);
|
console.error('Error downloading base map assets:', error)
|
||||||
throw error;
|
throw error
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async listServices() {
|
||||||
async installService(service_name: string) {
|
try {
|
||||||
try {
|
const response = await this.client.get<Array<ServiceSlim>>('/system/services')
|
||||||
const response = await this.client.post<{ success: boolean; message: string }>("/system/services/install", { service_name });
|
return response.data
|
||||||
return response.data;
|
} catch (error) {
|
||||||
} catch (error) {
|
console.error('Error listing services:', error)
|
||||||
console.error("Error installing service:", error);
|
throw error
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async affectService(service_name: string, action: "start" | "stop" | "restart") {
|
async installService(service_name: string) {
|
||||||
try {
|
try {
|
||||||
const response = await this.client.post<{ success: boolean; message: string }>("/system/services/affect", { service_name, action });
|
const response = await this.client.post<{ success: boolean; message: string }>(
|
||||||
return response.data;
|
'/system/services/install',
|
||||||
} catch (error) {
|
{ service_name }
|
||||||
console.error("Error affecting service:", error);
|
)
|
||||||
throw error;
|
return response.data
|
||||||
}
|
} catch (error) {
|
||||||
|
console.error('Error installing service:', error)
|
||||||
|
throw error
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async listZimFiles() {
|
async affectService(service_name: string, action: 'start' | 'stop' | 'restart') {
|
||||||
return await this.client.get<ListZimFilesResponse>("/zim/list");
|
try {
|
||||||
|
const response = await this.client.post<{ success: boolean; message: string }>(
|
||||||
|
'/system/services/affect',
|
||||||
|
{ service_name, action }
|
||||||
|
)
|
||||||
|
return response.data
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error affecting service:', error)
|
||||||
|
throw error
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async listRemoteZimFiles({ start = 0, count = 12 }: { start?: number; count?: number }) {
|
async listZimFiles() {
|
||||||
return await this.client.get<ListRemoteZimFilesResponse>("/zim/list-remote", {
|
return await this.client.get<ListZimFilesResponse>('/zim/list')
|
||||||
params: {
|
}
|
||||||
start,
|
|
||||||
count
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async downloadRemoteZimFile(url: string): Promise<{
|
async listRemoteZimFiles({ start = 0, count = 12 }: { start?: number; count?: number }) {
|
||||||
message: string;
|
return await this.client.get<ListRemoteZimFilesResponse>('/zim/list-remote', {
|
||||||
filename: string;
|
params: {
|
||||||
url: string;
|
start,
|
||||||
}> {
|
count,
|
||||||
try {
|
},
|
||||||
const response = await this.client.post("/zim/download-remote", { url });
|
})
|
||||||
return response.data;
|
}
|
||||||
} catch (error) {
|
|
||||||
console.error("Error downloading remote ZIM file:", error);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async deleteZimFile(key: string) {
|
async downloadRemoteZimFile(url: string): Promise<{
|
||||||
try {
|
message: string
|
||||||
const response = await this.client.delete(`/zim/${key}`);
|
filename: string
|
||||||
return response.data;
|
url: string
|
||||||
} catch (error) {
|
}> {
|
||||||
console.error("Error deleting ZIM file:", error);
|
try {
|
||||||
throw error;
|
const response = await this.client.post('/zim/download-remote', { url })
|
||||||
}
|
return response.data
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error downloading remote ZIM file:', error)
|
||||||
|
throw error
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteZimFile(key: string) {
|
||||||
|
try {
|
||||||
|
const response = await this.client.delete(`/zim/${key}`)
|
||||||
|
return response.data
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error deleting ZIM file:', error)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default new API();
|
export default new API()
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import DocsLayout from '~/layouts/DocsLayout'
|
||||||
export default function Show({ content }: { content: any; }) {
|
export default function Show({ content }: { content: any; }) {
|
||||||
return (
|
return (
|
||||||
<DocsLayout>
|
<DocsLayout>
|
||||||
<Head title={'Documentation | Project N.O.M.A.D.'} />
|
<Head title={'Documentation'} />
|
||||||
<div className="xl:pl-80 py-6">
|
<div className="xl:pl-80 py-6">
|
||||||
<MarkdocRenderer content={content} />
|
<MarkdocRenderer content={content} />
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { IconHelp, IconPlus, IconSettings, IconWifiOff } from '@tabler/icons-react'
|
import { IconHelp, IconMapRoute, IconPlus, IconSettings, IconWifiOff } from '@tabler/icons-react'
|
||||||
import { Head } from '@inertiajs/react'
|
import { Head } from '@inertiajs/react'
|
||||||
import BouncingLogo from '~/components/BouncingLogo'
|
import BouncingLogo from '~/components/BouncingLogo'
|
||||||
import AppLayout from '~/layouts/AppLayout'
|
import AppLayout from '~/layouts/AppLayout'
|
||||||
|
|
@ -21,6 +21,14 @@ const STATIC_ITEMS = [
|
||||||
icon: <IconHelp size={48} />,
|
icon: <IconHelp size={48} />,
|
||||||
installed: true,
|
installed: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
label: 'Maps',
|
||||||
|
to: '/maps',
|
||||||
|
target: '',
|
||||||
|
description: 'View offline maps',
|
||||||
|
icon: <IconMapRoute size={48} />,
|
||||||
|
installed: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
label: 'Settings',
|
label: 'Settings',
|
||||||
to: '/settings/system',
|
to: '/settings/system',
|
||||||
|
|
@ -52,7 +60,7 @@ export default function Home(props: {
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<AppLayout>
|
<AppLayout>
|
||||||
<Head title="Project N.O.M.A.D Command Center" />
|
<Head title="Command Center" />
|
||||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4 p-4">
|
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4 p-4">
|
||||||
{items.map((item) => (
|
{items.map((item) => (
|
||||||
<a key={item.label} href={item.to} target={item.target}>
|
<a key={item.label} href={item.to} target={item.target}>
|
||||||
|
|
|
||||||
27
admin/inertia/pages/maps.tsx
Normal file
27
admin/inertia/pages/maps.tsx
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
import MapsLayout from '~/layouts/MapsLayout'
|
||||||
|
import { Head, Link } from '@inertiajs/react'
|
||||||
|
import MapComponent from '~/components/maps/MapComponent'
|
||||||
|
import StyledButton from '~/components/StyledButton'
|
||||||
|
import { IconArrowLeft } from '@tabler/icons-react'
|
||||||
|
|
||||||
|
export default function Maps() {
|
||||||
|
return (
|
||||||
|
<MapsLayout>
|
||||||
|
<Head title="Maps" />
|
||||||
|
<div className="flex border-b border-gray-900/10 p-4 justify-between">
|
||||||
|
<Link href="/home" className="flex items-center">
|
||||||
|
<IconArrowLeft className="mr-2" size={24} />
|
||||||
|
<p className="text-lg text-gray-600">Back to Home</p>
|
||||||
|
</Link>
|
||||||
|
<Link href="/settings/maps">
|
||||||
|
<StyledButton variant="primary" icon="Cog6ToothIcon">
|
||||||
|
Manage Map Regions
|
||||||
|
</StyledButton>
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
<div className="w-full h-full flex p-4 justify-center items-center">
|
||||||
|
<MapComponent />
|
||||||
|
</div>
|
||||||
|
</MapsLayout>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -209,7 +209,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<SettingsLayout>
|
<SettingsLayout>
|
||||||
<Head title="App Settings | Project N.O.M.A.D." />
|
<Head title="App Settings" />
|
||||||
<div className="xl:pl-72 w-full">
|
<div className="xl:pl-72 w-full">
|
||||||
<main className="px-12 py-6">
|
<main className="px-12 py-6">
|
||||||
<h1 className="text-4xl font-semibold mb-4">Apps</h1>
|
<h1 className="text-4xl font-semibold mb-4">Apps</h1>
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import SettingsLayout from '~/layouts/SettingsLayout'
|
||||||
export default function SettingsPage() {
|
export default function SettingsPage() {
|
||||||
return (
|
return (
|
||||||
<SettingsLayout>
|
<SettingsLayout>
|
||||||
<Head title="Legal Notices | Project N.O.M.A.D." />
|
<Head title="Legal Notices" />
|
||||||
<div className="xl:pl-72 w-full">
|
<div className="xl:pl-72 w-full">
|
||||||
<main className="px-12 py-6">
|
<main className="px-12 py-6">
|
||||||
<h1 className="text-4xl font-semibold mb-6">Legal Notices</h1>
|
<h1 className="text-4xl font-semibold mb-6">Legal Notices</h1>
|
||||||
|
|
|
||||||
117
admin/inertia/pages/settings/maps.tsx
Normal file
117
admin/inertia/pages/settings/maps.tsx
Normal file
|
|
@ -0,0 +1,117 @@
|
||||||
|
import { Head, router } from '@inertiajs/react'
|
||||||
|
import StyledTable from '~/components/StyledTable'
|
||||||
|
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||||
|
import StyledButton from '~/components/StyledButton'
|
||||||
|
import { useModals } from '~/context/ModalContext'
|
||||||
|
import StyledModal from '~/components/StyledModal'
|
||||||
|
import { FileEntry } from '../../../types/files'
|
||||||
|
import MissingBaseAssetsAlert from '~/components/layout/MissingBaseAssetsAlert'
|
||||||
|
import { useNotifications } from '~/context/NotificationContext'
|
||||||
|
import { useState } from 'react'
|
||||||
|
import api from '~/lib/api'
|
||||||
|
|
||||||
|
export default function MapsManager(props: {
|
||||||
|
maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] }
|
||||||
|
}) {
|
||||||
|
const { openModal, closeAllModals } = useModals()
|
||||||
|
const { addNotification } = useNotifications()
|
||||||
|
const [downloading, setDownloading] = useState(false)
|
||||||
|
|
||||||
|
async function downloadBaseAssets() {
|
||||||
|
try {
|
||||||
|
setDownloading(true)
|
||||||
|
|
||||||
|
const res = await api.downloadBaseMapAssets()
|
||||||
|
if (res.success) {
|
||||||
|
addNotification({
|
||||||
|
type: 'success',
|
||||||
|
message: 'Base map assets downloaded successfully.',
|
||||||
|
})
|
||||||
|
router.reload()
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error downloading base assets:', error)
|
||||||
|
addNotification({
|
||||||
|
type: 'error',
|
||||||
|
message: 'An error occurred while downloading the base map assets. Please try again.',
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
setDownloading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function confirmDeleteFile(file: FileEntry) {
|
||||||
|
openModal(
|
||||||
|
<StyledModal
|
||||||
|
title="Confirm Delete?"
|
||||||
|
onConfirm={() => {
|
||||||
|
closeAllModals()
|
||||||
|
}}
|
||||||
|
onCancel={closeAllModals}
|
||||||
|
open={true}
|
||||||
|
confirmText="Delete"
|
||||||
|
cancelText="Cancel"
|
||||||
|
confirmVariant="danger"
|
||||||
|
>
|
||||||
|
<p className="text-gray-700">
|
||||||
|
Are you sure you want to delete {file.name}? This action cannot be undone.
|
||||||
|
</p>
|
||||||
|
</StyledModal>,
|
||||||
|
'confirm-delete-file-modal'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<SettingsLayout>
|
||||||
|
<Head title="Maps Manager" />
|
||||||
|
<div className="xl:pl-72 w-full">
|
||||||
|
<main className="px-12 py-6">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex flex-col">
|
||||||
|
<h1 className="text-4xl font-semibold mb-2">Maps Manager</h1>
|
||||||
|
<p className="text-gray-500">Manage your stored map data files.</p>
|
||||||
|
</div>
|
||||||
|
<StyledButton
|
||||||
|
variant="primary"
|
||||||
|
onClick={downloadBaseAssets}
|
||||||
|
loading={downloading}
|
||||||
|
icon='CloudArrowDownIcon'
|
||||||
|
>
|
||||||
|
Download New Map File
|
||||||
|
</StyledButton>
|
||||||
|
</div>
|
||||||
|
{!props.maps.baseAssetsExist && (
|
||||||
|
<MissingBaseAssetsAlert loading={downloading} onClickDownload={downloadBaseAssets} />
|
||||||
|
)}
|
||||||
|
<StyledTable<FileEntry & { actions?: any }>
|
||||||
|
className="font-semibold mt-4"
|
||||||
|
rowLines={true}
|
||||||
|
loading={false}
|
||||||
|
compact
|
||||||
|
columns={[
|
||||||
|
{ accessor: 'name', title: 'Name' },
|
||||||
|
{
|
||||||
|
accessor: 'actions',
|
||||||
|
title: 'Actions',
|
||||||
|
render: (record) => (
|
||||||
|
<div className="flex space-x-2">
|
||||||
|
<StyledButton
|
||||||
|
variant="danger"
|
||||||
|
icon={'TrashIcon'}
|
||||||
|
onClick={() => {
|
||||||
|
confirmDeleteFile(record)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Delete
|
||||||
|
</StyledButton>
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
data={props.maps.regionFiles || []}
|
||||||
|
/>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
</SettingsLayout>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -28,7 +28,7 @@ export default function SettingsPage(props: {
|
||||||
}) {
|
}) {
|
||||||
return (
|
return (
|
||||||
<SettingsLayout>
|
<SettingsLayout>
|
||||||
<Head title="Settings | Project N.O.M.A.D." />
|
<Head title="Settings" />
|
||||||
<div className="xl:pl-72 w-full">
|
<div className="xl:pl-72 w-full">
|
||||||
<main className="px-12 py-6">
|
<main className="px-12 py-6">
|
||||||
<h1 className="text-4xl font-semibold mb-6">System Information</h1>
|
<h1 className="text-4xl font-semibold mb-6">System Information</h1>
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,20 @@
|
||||||
import { Head, Link } from '@inertiajs/react'
|
import { Head } from '@inertiajs/react'
|
||||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
import StyledTable from '~/components/StyledTable'
|
import StyledTable from '~/components/StyledTable'
|
||||||
import SettingsLayout from '~/layouts/SettingsLayout'
|
import SettingsLayout from '~/layouts/SettingsLayout'
|
||||||
import { ZimFilesEntry } from '../../../../types/zim'
|
|
||||||
import api from '~/lib/api'
|
import api from '~/lib/api'
|
||||||
import StyledButton from '~/components/StyledButton'
|
import StyledButton from '~/components/StyledButton'
|
||||||
import { useModals } from '~/context/ModalContext'
|
import { useModals } from '~/context/ModalContext'
|
||||||
import StyledModal from '~/components/StyledModal'
|
import StyledModal from '~/components/StyledModal'
|
||||||
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||||
import Alert from '~/components/Alert'
|
import Alert from '~/components/Alert'
|
||||||
|
import { FileEntry } from '../../../../types/files'
|
||||||
|
|
||||||
export default function ZimPage() {
|
export default function ZimPage() {
|
||||||
const queryClient = useQueryClient()
|
const queryClient = useQueryClient()
|
||||||
const { openModal, closeAllModals } = useModals()
|
const { openModal, closeAllModals } = useModals()
|
||||||
const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve')
|
const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve')
|
||||||
const { data, isLoading } = useQuery<ZimFilesEntry[]>({
|
const { data, isLoading } = useQuery<FileEntry[]>({
|
||||||
queryKey: ['zim-files'],
|
queryKey: ['zim-files'],
|
||||||
queryFn: getFiles,
|
queryFn: getFiles,
|
||||||
})
|
})
|
||||||
|
|
@ -24,7 +24,7 @@ export default function ZimPage() {
|
||||||
return res.data.files
|
return res.data.files
|
||||||
}
|
}
|
||||||
|
|
||||||
async function confirmDeleteFile(file: ZimFilesEntry) {
|
async function confirmDeleteFile(file: FileEntry) {
|
||||||
openModal(
|
openModal(
|
||||||
<StyledModal
|
<StyledModal
|
||||||
title="Confirm Delete?"
|
title="Confirm Delete?"
|
||||||
|
|
@ -47,7 +47,7 @@ export default function ZimPage() {
|
||||||
}
|
}
|
||||||
|
|
||||||
const deleteFileMutation = useMutation({
|
const deleteFileMutation = useMutation({
|
||||||
mutationFn: async (file: ZimFilesEntry) => api.deleteZimFile(file.name.replace('.zim', '')),
|
mutationFn: async (file: FileEntry) => api.deleteZimFile(file.name.replace('.zim', '')),
|
||||||
onSuccess: () => {
|
onSuccess: () => {
|
||||||
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
|
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
|
||||||
},
|
},
|
||||||
|
|
@ -73,7 +73,7 @@ export default function ZimPage() {
|
||||||
className="!mt-6"
|
className="!mt-6"
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
<StyledTable<ZimFilesEntry & { actions?: any }>
|
<StyledTable<FileEntry & { actions?: any }>
|
||||||
className="font-semibold mt-4"
|
className="font-semibold mt-4"
|
||||||
rowLines={true}
|
rowLines={true}
|
||||||
loading={isLoading}
|
loading={isLoading}
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ const NotificationsProvider = ({ children }: { children: React.ReactNode }) => {
|
||||||
<Icon type={notification.type} />
|
<Icon type={notification.type} />
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<p className="">{notification.message}</p>
|
<p className="break-all">{notification.message}</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
709
admin/package-lock.json
generated
709
admin/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
|
|
@ -71,6 +71,7 @@
|
||||||
"@heroicons/react": "^2.2.0",
|
"@heroicons/react": "^2.2.0",
|
||||||
"@inertiajs/react": "^2.0.13",
|
"@inertiajs/react": "^2.0.13",
|
||||||
"@markdoc/markdoc": "^0.5.2",
|
"@markdoc/markdoc": "^0.5.2",
|
||||||
|
"@protomaps/basemaps": "^5.7.0",
|
||||||
"@tabler/icons-react": "^3.34.0",
|
"@tabler/icons-react": "^3.34.0",
|
||||||
"@tailwindcss/vite": "^4.1.10",
|
"@tailwindcss/vite": "^4.1.10",
|
||||||
"@tanstack/react-query": "^5.81.5",
|
"@tanstack/react-query": "^5.81.5",
|
||||||
|
|
@ -85,15 +86,21 @@
|
||||||
"edge.js": "^6.2.1",
|
"edge.js": "^6.2.1",
|
||||||
"fast-xml-parser": "^5.2.5",
|
"fast-xml-parser": "^5.2.5",
|
||||||
"luxon": "^3.6.1",
|
"luxon": "^3.6.1",
|
||||||
|
"maplibre-gl": "^4.7.1",
|
||||||
"mysql2": "^3.14.1",
|
"mysql2": "^3.14.1",
|
||||||
"pino-pretty": "^13.0.0",
|
"pino-pretty": "^13.0.0",
|
||||||
|
"pmtiles": "^4.3.0",
|
||||||
"postcss": "^8.5.6",
|
"postcss": "^8.5.6",
|
||||||
"react": "^19.1.0",
|
"react": "^19.1.0",
|
||||||
"react-adonis-transmit": "^1.0.1",
|
"react-adonis-transmit": "^1.0.1",
|
||||||
"react-dom": "^19.1.0",
|
"react-dom": "^19.1.0",
|
||||||
|
"react-map-gl": "^8.1.0",
|
||||||
"reflect-metadata": "^0.2.2",
|
"reflect-metadata": "^0.2.2",
|
||||||
"systeminformation": "^5.27.7",
|
"systeminformation": "^5.27.7",
|
||||||
"tailwindcss": "^4.1.10",
|
"tailwindcss": "^4.1.10",
|
||||||
|
"tar": "^7.5.2",
|
||||||
|
"url-join": "^5.0.0",
|
||||||
|
"usehooks-ts": "^3.1.1",
|
||||||
"yaml": "^2.8.0"
|
"yaml": "^2.8.0"
|
||||||
},
|
},
|
||||||
"hotHook": {
|
"hotHook": {
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ export default await Env.create(new URL('../', import.meta.url), {
|
||||||
PORT: Env.schema.number(),
|
PORT: Env.schema.number(),
|
||||||
APP_KEY: Env.schema.string(),
|
APP_KEY: Env.schema.string(),
|
||||||
HOST: Env.schema.string({ format: 'host' }),
|
HOST: Env.schema.string({ format: 'host' }),
|
||||||
|
URL: Env.schema.string(),
|
||||||
LOG_LEVEL: Env.schema.string(),
|
LOG_LEVEL: Env.schema.string(),
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
||||||
|
|
@ -8,50 +8,75 @@
|
||||||
*/
|
*/
|
||||||
import DocsController from '#controllers/docs_controller'
|
import DocsController from '#controllers/docs_controller'
|
||||||
import HomeController from '#controllers/home_controller'
|
import HomeController from '#controllers/home_controller'
|
||||||
|
import MapsController from '#controllers/maps_controller'
|
||||||
import SettingsController from '#controllers/settings_controller'
|
import SettingsController from '#controllers/settings_controller'
|
||||||
import SystemController from '#controllers/system_controller'
|
import SystemController from '#controllers/system_controller'
|
||||||
import ZimController from '#controllers/zim_controller'
|
import ZimController from '#controllers/zim_controller'
|
||||||
import router from '@adonisjs/core/services/router'
|
import router from '@adonisjs/core/services/router'
|
||||||
import transmit from '@adonisjs/transmit/services/main'
|
import transmit from '@adonisjs/transmit/services/main'
|
||||||
|
|
||||||
transmit.registerRoutes();
|
transmit.registerRoutes()
|
||||||
|
|
||||||
router.get('/', [HomeController, 'index']);
|
router.get('/', [HomeController, 'index'])
|
||||||
router.get('/home', [HomeController, 'home']);
|
router.get('/home', [HomeController, 'home'])
|
||||||
router.on('/about').renderInertia('about')
|
router.on('/about').renderInertia('about')
|
||||||
|
|
||||||
router.group(() => {
|
router
|
||||||
|
.group(() => {
|
||||||
router.get('/system', [SettingsController, 'system'])
|
router.get('/system', [SettingsController, 'system'])
|
||||||
router.get('/apps', [SettingsController, 'apps'])
|
router.get('/apps', [SettingsController, 'apps'])
|
||||||
router.get('/legal', [SettingsController, 'legal'])
|
router.get('/legal', [SettingsController, 'legal'])
|
||||||
|
router.get('/maps', [SettingsController, 'maps'])
|
||||||
router.get('/zim', [SettingsController, 'zim'])
|
router.get('/zim', [SettingsController, 'zim'])
|
||||||
router.get('/zim/remote-explorer', [SettingsController, 'zimRemote'])
|
router.get('/zim/remote-explorer', [SettingsController, 'zimRemote'])
|
||||||
}).prefix('/settings')
|
})
|
||||||
|
.prefix('/settings')
|
||||||
|
|
||||||
router.group(() => {
|
router
|
||||||
|
.group(() => {
|
||||||
router.get('/:slug', [DocsController, 'show'])
|
router.get('/:slug', [DocsController, 'show'])
|
||||||
router.get('/', ({ inertia }) => {
|
router.get('/', ({ inertia }) => {
|
||||||
return inertia.render('Docs/Index', {
|
return inertia.render('Docs/Index', {
|
||||||
title: "Documentation",
|
title: 'Documentation',
|
||||||
content: "Welcome to the documentation!"
|
content: 'Welcome to the documentation!',
|
||||||
});
|
})
|
||||||
});
|
})
|
||||||
}).prefix('/docs')
|
})
|
||||||
|
.prefix('/docs')
|
||||||
|
|
||||||
router.group(() => {
|
router.get('/maps', [MapsController, 'index'])
|
||||||
|
|
||||||
|
router
|
||||||
|
.group(() => {
|
||||||
|
router.get('/regions', [MapsController, 'listRegions'])
|
||||||
|
router.get('/styles', [MapsController, 'styles'])
|
||||||
|
router.get('/preflight', [MapsController, 'checkBaseAssets'])
|
||||||
|
router.post('/download-base-assets', [MapsController, 'downloadBaseAssets'])
|
||||||
|
router.post('/download-remote', [MapsController, 'downloadRemote'])
|
||||||
|
router.delete('/:filename', [MapsController, 'delete'])
|
||||||
|
})
|
||||||
|
.prefix('/api/maps')
|
||||||
|
|
||||||
|
router
|
||||||
|
.group(() => {
|
||||||
router.get('/list', [DocsController, 'list'])
|
router.get('/list', [DocsController, 'list'])
|
||||||
}).prefix('/api/docs')
|
})
|
||||||
|
.prefix('/api/docs')
|
||||||
|
|
||||||
router.group(() => {
|
router
|
||||||
|
.group(() => {
|
||||||
router.get('/info', [SystemController, 'getSystemInfo'])
|
router.get('/info', [SystemController, 'getSystemInfo'])
|
||||||
router.get('/services', [SystemController, 'getServices'])
|
router.get('/services', [SystemController, 'getServices'])
|
||||||
router.post('/services/affect', [SystemController, 'affectService'])
|
router.post('/services/affect', [SystemController, 'affectService'])
|
||||||
router.post('/services/install', [SystemController, 'installService'])
|
router.post('/services/install', [SystemController, 'installService'])
|
||||||
}).prefix('/api/system')
|
})
|
||||||
|
.prefix('/api/system')
|
||||||
|
|
||||||
router.group(() => {
|
router
|
||||||
|
.group(() => {
|
||||||
router.get('/list', [ZimController, 'list'])
|
router.get('/list', [ZimController, 'list'])
|
||||||
router.get('/list-remote', [ZimController, 'listRemote'])
|
router.get('/list-remote', [ZimController, 'listRemote'])
|
||||||
router.post('/download-remote', [ZimController, 'downloadRemote'])
|
router.post('/download-remote', [ZimController, 'downloadRemote'])
|
||||||
router.delete('/:key', [ZimController, 'delete'])
|
router.delete('/:filename', [ZimController, 'delete'])
|
||||||
}).prefix('/api/zim')
|
})
|
||||||
|
.prefix('/api/zim')
|
||||||
|
|
|
||||||
32
admin/types/downloads.ts
Normal file
32
admin/types/downloads.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
export type DoResumableDownloadParams = {
|
||||||
|
url: string
|
||||||
|
path: string
|
||||||
|
timeout: number
|
||||||
|
allowedMimeTypes: string[]
|
||||||
|
signal?: AbortSignal
|
||||||
|
onProgress?: (progress: DoResumableDownloadProgress) => void
|
||||||
|
forceNew?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & {
|
||||||
|
max_retries?: number
|
||||||
|
retry_delay?: number
|
||||||
|
onAttemptError?: (error: Error, attempt: number) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DoResumableDownloadProgress = {
|
||||||
|
downloadedBytes: number
|
||||||
|
totalBytes: number
|
||||||
|
lastProgressTime: number
|
||||||
|
lastDownloadedBytes: number
|
||||||
|
url: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DoBackgroundDownloadParams = Omit<
|
||||||
|
DoResumableDownloadWithRetryParams,
|
||||||
|
'onProgress' | 'onAttemptError' | 'signal'
|
||||||
|
> & {
|
||||||
|
channel: string
|
||||||
|
activeDownloads: Map<string, AbortController>
|
||||||
|
onComplete?: (url: string, path: string) => void | Promise<void>
|
||||||
|
}
|
||||||
30
admin/types/files.ts
Normal file
30
admin/types/files.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
/* General file transfer/download utility types */
|
||||||
|
|
||||||
|
export type FileEntry =
|
||||||
|
| {
|
||||||
|
type: 'file'
|
||||||
|
key: string
|
||||||
|
name: string
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: 'directory'
|
||||||
|
prefix: string
|
||||||
|
name: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DownloadProgress = {
|
||||||
|
downloaded_bytes: number
|
||||||
|
total_bytes: number
|
||||||
|
percentage: number
|
||||||
|
speed: string
|
||||||
|
time_remaining: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DownloadOptions = {
|
||||||
|
max_retries?: number
|
||||||
|
retry_delay?: number
|
||||||
|
chunk_size?: number
|
||||||
|
timeout?: number
|
||||||
|
onError?: (error: Error) => void
|
||||||
|
onComplete?: (filepath: string) => void
|
||||||
|
}
|
||||||
23
admin/types/maps.ts
Normal file
23
admin/types/maps.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
export type BaseStylesFile = {
|
||||||
|
version: number
|
||||||
|
sources: {
|
||||||
|
[key: string]: MapSource
|
||||||
|
}
|
||||||
|
layers: MapLayer[]
|
||||||
|
sprite: string
|
||||||
|
glyphs: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MapSource = {
|
||||||
|
type: 'vector' | 'raster' | 'raster-dem' | 'geojson' | 'image' | 'video'
|
||||||
|
attribution?: string
|
||||||
|
url: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MapLayer = {
|
||||||
|
'id': string
|
||||||
|
'type': string
|
||||||
|
'source'?: string
|
||||||
|
'source-layer'?: string
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
|
@ -1,16 +1,7 @@
|
||||||
export type ZimFilesEntry =
|
import { FileEntry } from './files.js'
|
||||||
{
|
|
||||||
type: 'file'
|
|
||||||
key: string;
|
|
||||||
name: string;
|
|
||||||
} | {
|
|
||||||
type: 'directory';
|
|
||||||
prefix: string;
|
|
||||||
name: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type ListZimFilesResponse = {
|
export type ListZimFilesResponse = {
|
||||||
files: ZimFilesEntry[]
|
files: FileEntry[]
|
||||||
next?: string
|
next?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -65,21 +56,4 @@ export type RemoteZimFileEntry = {
|
||||||
download_url: string;
|
download_url: string;
|
||||||
author: string;
|
author: string;
|
||||||
file_name: string;
|
file_name: string;
|
||||||
}
|
|
||||||
|
|
||||||
export type DownloadProgress = {
|
|
||||||
downloaded_bytes: number;
|
|
||||||
total_bytes: number;
|
|
||||||
percentage: number;
|
|
||||||
speed: string;
|
|
||||||
time_remaining: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type DownloadOptions = {
|
|
||||||
max_retries?: number;
|
|
||||||
retry_delay?: number;
|
|
||||||
chunk_size?: number;
|
|
||||||
timeout?: number;
|
|
||||||
onError?: (error: Error) => void;
|
|
||||||
onComplete?: (filepath: string) => void;
|
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user