diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index f33b549..72f65b1 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -36,6 +36,14 @@ export default class MapsController { } } + // For providing a "preflight" check in the UI before actually starting a background download + async downloadRemotePreflight({ request }: HttpContext) { + const payload = await request.validateUsing(remoteDownloadValidator) + console.log(payload) + const info = await this.mapService.downloadRemotePreflight(payload.url) + return info + } + async listRegions({}: HttpContext) { return await this.mapService.listRegions() } diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 380d0de..e52c676 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -1,5 +1,6 @@ import { ZimService } from '#services/zim_service' import { filenameValidator, remoteDownloadValidator } from '#validators/common' +import { listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -12,8 +13,9 @@ export default class ZimController { } async listRemote({ request }: HttpContext) { - const { start = 0, count = 12 } = request.qs() - return await this.zimService.listRemote({ start, count }) + const payload = await request.validateUsing(listRemoteZimValidator) + const { start = 0, count = 12, query } = payload + return await this.zimService.listRemote({ start, count, query }) } async downloadRemote({ request }: HttpContext) { @@ -27,6 +29,10 @@ export default class ZimController { } } + async listActiveDownloads({}: HttpContext) { + return this.zimService.listActiveDownloads() + } + async delete({ request, response }: HttpContext) { const payload = await request.validateUsing(filenameValidator) diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index aa96541..6fd614c 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -12,6 +12,7 @@ import { } from '../utils/fs.js' import { join } from 'path' import urlJoin from 'url-join' +import axios from 'axios' const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', @@ -114,6 +115,36 @@ export class MapService { return filename } + async downloadRemotePreflight( + url: string + ): Promise<{ filename: string; size: number } | { message: string }> { + try { + const parsed = new URL(url) + if (!parsed.pathname.endsWith('.pmtiles')) { + throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`) + } + + const filename = url.split('/').pop() + if (!filename) { + throw new Error('Could not determine filename from URL') + } + + // Perform a HEAD request to get the content length + const response = await axios.head(url) + + if (response.status !== 200) { + throw new Error(`Failed to fetch file info: ${response.status} ${response.statusText}`) + } + + const contentLength = response.headers['content-length'] + const size = contentLength ? parseInt(contentLength, 10) : 0 + + return { filename, size } + } catch (error) { + return { message: `Preflight check failed: ${error.message}` } + } + } + async generateStylesJSON() { if (!(await this.checkBaseAssetsExist())) { throw new Error('Base map assets are missing from storage/maps') diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index dfa66fa..656fb0b 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -44,9 +44,11 @@ export class ZimService { async listRemote({ start, count, + query, }: { start: number count: number + query?: string }): Promise { const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' @@ -55,6 +57,7 @@ export class ZimService { start: start, count: count, lang: 'eng', + ...(query ? { q: query } : {}), }, responseType: 'text', }) @@ -71,7 +74,8 @@ export class ZimService { throw new Error('Invalid response format from remote library') } - const filtered = result.feed.entry.filter((entry: any) => { + const entries = Array.isArray(result.feed.entry) ? result.feed.entry : [result.feed.entry] + const filtered = entries.filter((entry: any) => { return isRawRemoteZimFileEntry(entry) }) @@ -166,7 +170,7 @@ export class ZimService { return filename } - getActiveDownloads(): string[] { + listActiveDownloads(): string[] { return Array.from(this.activeDownloads.keys()) } diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index d2cd235..56f431e 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -2,13 +2,17 @@ import vine from '@vinejs/vine' export const remoteDownloadValidator = vine.compile( vine.object({ - url: vine.string().url().trim(), + url: vine.string().url({ + require_tld: false, // Allow local URLs + }).trim(), }) ) export const remoteDownloadValidatorOptional = vine.compile( vine.object({ - url: vine.string().url().trim().optional(), + url: vine.string().url({ + require_tld: false, // Allow local URLs + }).trim().optional(), }) ) diff --git a/admin/app/validators/zim.ts b/admin/app/validators/zim.ts index e69de29..2c18271 100644 --- a/admin/app/validators/zim.ts +++ b/admin/app/validators/zim.ts @@ -0,0 +1,9 @@ +import vine from '@vinejs/vine' + +export const listRemoteZimValidator = vine.compile( + vine.object({ + start: vine.number().min(0).optional(), + count: vine.number().min(1).max(100).optional(), + query: vine.string().optional(), + }) +) diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index b6fc3c4..ed30993 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -24,27 +24,6 @@ export default class ServiceSeeder extends BaseSeeder { is_dependency_service: false, depends_on: null, }, - { - service_name: DockerService.OPENSTREETMAP_SERVICE_NAME, - friendly_name: 'OpenStreetMap Tile Server', - description: 'Self-hosted OpenStreetMap tile server', - container_image: 'overv/openstreetmap-tile-server', - container_command: 'run', - container_config: JSON.stringify({ - HostConfig: { - RestartPolicy: { Name: 'unless-stopped' }, - Binds: [ - `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/db:/data/database:rw`, - `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/tiles:/data/tiles:rw` - ], - PortBindings: { '80/tcp': [{ HostPort: '9000' }] } - } - }), - ui_location: '9000', - installed: false, - is_dependency_service: false, - depends_on: null, - }, { service_name: DockerService.OLLAMA_SERVICE_NAME, friendly_name: 'Ollama', diff --git a/admin/inertia/components/DownloadURLModal.tsx b/admin/inertia/components/DownloadURLModal.tsx new file mode 100644 index 0000000..4bba80b --- /dev/null +++ b/admin/inertia/components/DownloadURLModal.tsx @@ -0,0 +1,83 @@ +import { useState } from 'react' +import StyledModal, { StyledModalProps } from './StyledModal' +import Input from './inputs/Input' +import api from '~/lib/api' + +export type DownloadURLModalProps = Omit< + StyledModalProps, + 'onConfirm' | 'open' | 'confirmText' | 'cancelText' | 'confirmVariant' | 'children' +> & { + suggestedURL?: string + onPreflightSuccess?: (url: string) => void +} + +const DownloadURLModal: React.FC = ({ + suggestedURL, + onPreflightSuccess, + ...modalProps +}) => { + const [url, setUrl] = useState(suggestedURL || '') + const [messages, setMessages] = useState([]) + const [passedPreflight, setPassedPreflight] = useState(false) + + async function runPreflightCheck(downloadUrl: string) { + try { + setMessages([`Running preflight check for URL: ${downloadUrl}`]) + const res = await api.downloadRemoteMapRegionPreflight(downloadUrl) + if ('message' in res) { + throw new Error(res.message) + } + + setMessages((prev) => [ + ...prev, + `Preflight check passed. Filename: ${res.filename}, Size: ${(res.size / (1024 * 1024)).toFixed(2)} MB`, + ]) + setPassedPreflight(true) + } catch (error) { + console.error('Preflight check failed:', error) + setMessages((prev) => [...prev, `Preflight check failed: ${error.message}`]) + setPassedPreflight(false) + } + } + + return ( + runPreflightCheck(url)} + open={true} + confirmText="Download" + confirmIcon="ArrowDownTrayIcon" + cancelText="Cancel" + confirmVariant="primary" + large + > +
+

+ Enter the URL of the map region file you wish to download. The URL must be publicly + reachable and end with .pmtiles. A preflight check will be run to verify the file's + availability, type, and approximate size. +

+ setUrl(e.target.value)} + /> +
+ {messages.map((message, idx) => ( +

+ {message} +

+ ))} +
+
+ + ) +} + +export default DownloadURLModal diff --git a/admin/inertia/components/StyledModal.tsx b/admin/inertia/components/StyledModal.tsx index 704adb6..dd4ad6e 100644 --- a/admin/inertia/components/StyledModal.tsx +++ b/admin/inertia/components/StyledModal.tsx @@ -3,11 +3,13 @@ import StyledButton, { StyledButtonProps } from './StyledButton' import React from 'react' import classNames from '~/lib/classNames' -interface StyledModalProps { +export type StyledModalProps = { onClose?: () => void title: string cancelText?: string + cancelIcon?: StyledButtonProps['icon'] confirmText?: string + confirmIcon?: StyledButtonProps['icon'] confirmVariant?: StyledButtonProps['variant'] open: boolean onCancel?: () => void @@ -23,7 +25,9 @@ const StyledModal: React.FC = ({ open, onClose, cancelText = 'Cancel', + cancelIcon, confirmText = 'Confirm', + confirmIcon, confirmVariant = 'action', onCancel, onConfirm, @@ -68,10 +72,11 @@ const StyledModal: React.FC = ({
{cancelText && onCancel && ( { if (onCancel) onCancel() }} + icon={cancelIcon} > {cancelText} @@ -82,6 +87,7 @@ const StyledModal: React.FC = ({ onClick={() => { if (onConfirm) onConfirm() }} + icon={confirmIcon} > {confirmText} diff --git a/admin/inertia/components/StyledTable.tsx b/admin/inertia/components/StyledTable.tsx index e24c661..daa477d 100644 --- a/admin/inertia/components/StyledTable.tsx +++ b/admin/inertia/components/StyledTable.tsx @@ -48,7 +48,7 @@ function StyledTable({ return (
({ ))} {!loading && data.length === 0 && ( - + {noDataText} diff --git a/admin/inertia/components/inputs/Input.tsx b/admin/inertia/components/inputs/Input.tsx new file mode 100644 index 0000000..d6b1c81 --- /dev/null +++ b/admin/inertia/components/inputs/Input.tsx @@ -0,0 +1,59 @@ +import classNames from "classnames"; +import { InputHTMLAttributes } from "react"; + +export interface InputProps extends InputHTMLAttributes { + name: string; + label: string; + className?: string; + labelClassName?: string; + inputClassName?: string; + leftIcon?: React.ReactNode; + error?: boolean; + required?: boolean; +} + +const Input: React.FC = ({ + className, + label, + name, + labelClassName, + inputClassName, + leftIcon, + error, + required, + ...props +}) => { + return ( +
+ +
+
+ {leftIcon && ( +
+ {leftIcon} +
+ )} + +
+
+
+ ); +}; + +export default Input; diff --git a/admin/inertia/hooks/useDebounce.ts b/admin/inertia/hooks/useDebounce.ts new file mode 100644 index 0000000..a7cd555 --- /dev/null +++ b/admin/inertia/hooks/useDebounce.ts @@ -0,0 +1,23 @@ +import { useRef, useEffect } from "react"; + +const useDebounce = () => { + const timeout = useRef(400); + + const debounce = + (func: Function, wait: number = 0) => + (...args: any[]) => { + clearTimeout(timeout.current); + timeout.current = window.setTimeout(() => func(...args), wait); + }; + + useEffect(() => { + return () => { + if (!timeout.current) return; + clearTimeout(timeout.current); + }; + }, []); + + return { debounce }; +}; + +export default useDebounce; diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 72034e8..38323bf 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -46,6 +46,31 @@ class API { } } + async downloadRemoteMapRegion(url: string) { + try { + const response = await this.client.post<{ message: string; filename: string; url: string }>( + '/maps/download-remote', + { url } + ) + return response.data + } catch (error) { + console.error('Error downloading remote map region:', error) + throw error + } + } + + async downloadRemoteMapRegionPreflight(url: string) { + try { + const response = await this.client.post< + { filename: string; size: number } | { message: string } + >('/maps/download-remote-preflight', { url }) + return response.data + } catch (error) { + console.error('Error preflighting remote map region download:', error) + throw error + } + } + async listServices() { try { const response = await this.client.get>('/system/services') @@ -86,15 +111,34 @@ class API { return await this.client.get('/zim/list') } - async listRemoteZimFiles({ start = 0, count = 12 }: { start?: number; count?: number }) { + async listRemoteZimFiles({ + start = 0, + count = 12, + query, + }: { + start?: number + count?: number + query?: string + }) { return await this.client.get('/zim/list-remote', { params: { start, count, + query, }, }) } + async listActiveZimDownloads(): Promise { + try { + const response = await this.client.get('/zim/active-downloads') + return response.data + } catch (error) { + console.error('Error listing active ZIM downloads:', error) + throw error + } + } + async downloadRemoteZimFile(url: string): Promise<{ message: string filename: string diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 6030439..343b7d5 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -9,6 +9,7 @@ import MissingBaseAssetsAlert from '~/components/layout/MissingBaseAssetsAlert' import { useNotifications } from '~/context/NotificationContext' import { useState } from 'react' import api from '~/lib/api' +import DownloadURLModal from '~/components/DownloadURLModal' export default function MapsManager(props: { maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] } @@ -61,6 +62,17 @@ export default function MapsManager(props: { ) } + async function openDownloadModal() { + openModal( + closeAllModals()} + />, + 'download-map-file-modal' + ) + } + return ( @@ -73,9 +85,9 @@ export default function MapsManager(props: {
Download New Map File diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index f4fbe43..f48f273 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -16,6 +16,9 @@ import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' import Alert from '~/components/Alert' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' +import Input from '~/components/inputs/Input' +import { IconSearch } from '@tabler/icons-react' +import useDebounce from '~/hooks/useDebounce' export default function ZimRemoteExplorer() { const tableParentRef = useRef(null) @@ -24,17 +27,26 @@ export default function ZimRemoteExplorer() { const { addNotification } = useNotifications() const { isOnline } = useInternetStatus() const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve') + const { debounce } = useDebounce() + + const [query, setQuery] = useState('') + const [queryUI, setQueryUI] = useState('') + const [activeDownloads, setActiveDownloads] = useState< Map >(new Map()) + const debouncedSetQuery = debounce((val: string) => { + setQuery(val) + }, 400) + const { data, fetchNextPage, isFetching, isLoading } = useInfiniteQuery({ - queryKey: ['remote-zim-files'], + queryKey: ['remote-zim-files', query], queryFn: async ({ pageParam = 0 }) => { const pageParsed = parseInt((pageParam as number).toString(), 10) const start = isNaN(pageParsed) ? 0 : pageParsed * 12 - const res = await api.listRemoteZimFiles({ start, count: 12 }) + const res = await api.listRemoteZimFiles({ start, count: 12, query: query || undefined }) return res.data }, initialPageParam: 0, @@ -166,6 +178,20 @@ export default function ZimRemoteExplorer() { className="!mt-6" /> )} +
+ { + setQueryUI(e.target.value) + debouncedSetQuery(e.target.value) + }} + className="w-1/3" + leftIcon={} + /> +
data={flatData.map((i, idx) => { const row = virtualizer.getVirtualItems().find((v) => v.index === idx) diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 0cbcfb2..19f8767 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -53,6 +53,7 @@ router router.get('/preflight', [MapsController, 'checkBaseAssets']) router.post('/download-base-assets', [MapsController, 'downloadBaseAssets']) router.post('/download-remote', [MapsController, 'downloadRemote']) + router.post('/download-remote-preflight', [MapsController, 'downloadRemotePreflight']) router.delete('/:filename', [MapsController, 'delete']) }) .prefix('/api/maps') @@ -76,6 +77,7 @@ router .group(() => { router.get('/list', [ZimController, 'list']) router.get('/list-remote', [ZimController, 'listRemote']) + router.get('/active-downloads', [ZimController, 'listActiveDownloads']) router.post('/download-remote', [ZimController, 'downloadRemote']) router.delete('/:filename', [ZimController, 'delete']) }) diff --git a/admin/types/zim.ts b/admin/types/zim.ts index 33be262..600a46f 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -1,59 +1,59 @@ import { FileEntry } from './files.js' export type ListZimFilesResponse = { - files: FileEntry[] - next?: string + files: FileEntry[] + next?: string } export type ListRemoteZimFilesResponse = { - items: RemoteZimFileEntry[]; - has_more: boolean; - total_count: number; + items: RemoteZimFileEntry[] + has_more: boolean + total_count: number } export type RawRemoteZimFileEntry = { - id: string; - title: string; - updated: string; - summary: string; - language: string; - name: string; - flavour: string; - category: string; - tags: string; - articleCount: number; - mediaCount: number; - link: Record[]; - author: { - name: string - }; - publisher: { - name: string; - }; - 'dc:issued': string; + 'id': string + 'title': string + 'updated': string + 'summary': string + 'language': string + 'name': string + 'flavour': string + 'category': string + 'tags': string + 'articleCount': number + 'mediaCount': number + 'link': Record[] + 'author': { + name: string + } + 'publisher': { + name: string + } + 'dc:issued': string } export type RawListRemoteZimFilesResponse = { - '?xml': string; - feed: { - id: string; - link: string[]; - title: string; - updated: string; - totalResults: number; - startIndex: number; - itemsPerPage: number; - entry: any[]; - } + '?xml': string + 'feed': { + id: string + link: string[] + title: string + updated: string + totalResults: number + startIndex: number + itemsPerPage: number + entry: RawRemoteZimFileEntry | RawRemoteZimFileEntry[] + } } export type RemoteZimFileEntry = { - id: string; - title: string; - updated: string; - summary: string; - size_bytes: number; - download_url: string; - author: string; - file_name: string; -} \ No newline at end of file + id: string + title: string + updated: string + summary: string + size_bytes: number + download_url: string + author: string + file_name: string +} diff --git a/admin/util/zim.ts b/admin/util/zim.ts index 34b6205..ce9221e 100644 --- a/admin/util/zim.ts +++ b/admin/util/zim.ts @@ -1,10 +1,15 @@ -import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from "../types/zim.js"; - +import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from '../types/zim.js' export function isRawListRemoteZimFilesResponse(obj: any): obj is RawListRemoteZimFilesResponse { - return obj && typeof obj === 'object' && 'feed' in obj && 'entry' in obj.feed && Array.isArray(obj.feed.entry); + return ( + obj && + typeof obj === 'object' && + 'feed' in obj && + 'entry' in obj.feed && + typeof obj.feed.entry === 'object' // could be array or single object but typeof array is technically 'object' + ) } export function isRawRemoteZimFileEntry(obj: any): obj is RawRemoteZimFileEntry { - return obj && typeof obj === 'object' && 'id' in obj && 'title' in obj && 'summary' in obj; -} \ No newline at end of file + return obj && typeof obj === 'object' && 'id' in obj && 'title' in obj && 'summary' in obj +}