feat: [wip] custom map and zim downloads

admin/app/controllers/maps_controller.ts

@@ -36,6 +36,14 @@ export default class MapsController {
     }
   }
 
+  // For providing a "preflight" check in the UI before actually starting a background download
+  async downloadRemotePreflight({ request }: HttpContext) {
+    const payload = await request.validateUsing(remoteDownloadValidator)
+    console.log(payload)
+    const info = await this.mapService.downloadRemotePreflight(payload.url)
+    return info
+  }
+
   async listRegions({}: HttpContext) {
     return await this.mapService.listRegions()
   }

admin/app/controllers/zim_controller.ts

@@ -1,5 +1,6 @@
 import { ZimService } from '#services/zim_service'
 import { filenameValidator, remoteDownloadValidator } from '#validators/common'
+import { listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 
@@ -12,8 +13,9 @@ export default class ZimController {
   }
 
   async listRemote({ request }: HttpContext) {
-    const { start = 0, count = 12 } = request.qs()
+    const payload = await request.validateUsing(listRemoteZimValidator)
-    return await this.zimService.listRemote({ start, count })
+    const { start = 0, count = 12, query } = payload
+    return await this.zimService.listRemote({ start, count, query })
   }
 
   async downloadRemote({ request }: HttpContext) {
@@ -27,6 +29,10 @@ export default class ZimController {
     }
   }
 
+  async listActiveDownloads({}: HttpContext) {
+    return this.zimService.listActiveDownloads()
+  }
+
   async delete({ request, response }: HttpContext) {
     const payload = await request.validateUsing(filenameValidator)
 

admin/app/services/map_service.ts

@@ -12,6 +12,7 @@ import {
 } from '../utils/fs.js'
 import { join } from 'path'
 import urlJoin from 'url-join'
+import axios from 'axios'
 
 const BASE_ASSETS_MIME_TYPES = [
   'application/gzip',
@@ -114,6 +115,36 @@ export class MapService {
     return filename
   }
 
+  async downloadRemotePreflight(
+    url: string
+  ): Promise<{ filename: string; size: number } | { message: string }> {
+    try {
+      const parsed = new URL(url)
+      if (!parsed.pathname.endsWith('.pmtiles')) {
+        throw new Error(`Invalid PMTiles file URL: ${url}. URL must end with .pmtiles`)
+      }
+
+      const filename = url.split('/').pop()
+      if (!filename) {
+        throw new Error('Could not determine filename from URL')
+      }
+
+      // Perform a HEAD request to get the content length
+      const response = await axios.head(url)
+
+      if (response.status !== 200) {
+        throw new Error(`Failed to fetch file info: ${response.status} ${response.statusText}`)
+      }
+
+      const contentLength = response.headers['content-length']
+      const size = contentLength ? parseInt(contentLength, 10) : 0
+
+      return { filename, size }
+    } catch (error) {
+      return { message: `Preflight check failed: ${error.message}` }
+    }
+  }
+
   async generateStylesJSON() {
     if (!(await this.checkBaseAssetsExist())) {
       throw new Error('Base map assets are missing from storage/maps')

admin/app/services/zim_service.ts

@@ -44,9 +44,11 @@ export class ZimService {
   async listRemote({
     start,
     count,
+    query,
   }: {
     start: number
     count: number
+    query?: string
   }): Promise<ListRemoteZimFilesResponse> {
     const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries'
 
@@ -55,6 +57,7 @@ export class ZimService {
         start: start,
         count: count,
         lang: 'eng',
+        ...(query ? { q: query } : {}),
       },
       responseType: 'text',
     })
@@ -71,7 +74,8 @@ export class ZimService {
       throw new Error('Invalid response format from remote library')
     }
 
-    const filtered = result.feed.entry.filter((entry: any) => {
+    const entries = Array.isArray(result.feed.entry) ? result.feed.entry : [result.feed.entry]
+    const filtered = entries.filter((entry: any) => {
       return isRawRemoteZimFileEntry(entry)
     })
 
@@ -166,7 +170,7 @@ export class ZimService {
     return filename
   }
 
-  getActiveDownloads(): string[] {
+  listActiveDownloads(): string[] {
     return Array.from(this.activeDownloads.keys())
   }
 

admin/app/validators/common.ts

@@ -2,13 +2,17 @@ import vine from '@vinejs/vine'
 
 export const remoteDownloadValidator = vine.compile(
   vine.object({
-    url: vine.string().url().trim(),
+    url: vine.string().url({
+      require_tld: false, // Allow local URLs
+    }).trim(),
   })
 )
 
 export const remoteDownloadValidatorOptional = vine.compile(
   vine.object({
-    url: vine.string().url().trim().optional(),
+    url: vine.string().url({
+      require_tld: false, // Allow local URLs
+    }).trim().optional(),
   })
 )
 

admin/app/validators/zim.ts

@@ -0,0 +1,9 @@
+import vine from '@vinejs/vine'
+
+export const listRemoteZimValidator = vine.compile(
+  vine.object({
+    start: vine.number().min(0).optional(),
+    count: vine.number().min(1).max(100).optional(),
+    query: vine.string().optional(),
+  })
+)

admin/database/seeders/service_seeder.ts

@@ -24,27 +24,6 @@ export default class ServiceSeeder extends BaseSeeder {
       is_dependency_service: false,
       depends_on: null,
     },
-    {
-      service_name: DockerService.OPENSTREETMAP_SERVICE_NAME,
-      friendly_name: 'OpenStreetMap Tile Server',
-      description: 'Self-hosted OpenStreetMap tile server',
-      container_image: 'overv/openstreetmap-tile-server',
-      container_command: 'run',
-      container_config: JSON.stringify({
-        HostConfig: {
-          RestartPolicy: { Name: 'unless-stopped' },
-          Binds: [
-            `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/db:/data/database:rw`,
-            `${DockerService.NOMAD_STORAGE_ABS_PATH}/osm/tiles:/data/tiles:rw`
-          ],
-          PortBindings: { '80/tcp': [{ HostPort: '9000' }] }
-        }
-      }),
-      ui_location: '9000',
-      installed: false,
-      is_dependency_service: false,
-      depends_on: null,
-    },
     {
       service_name: DockerService.OLLAMA_SERVICE_NAME,
       friendly_name: 'Ollama',

admin/inertia/components/DownloadURLModal.tsx

@@ -0,0 +1,83 @@
+import { useState } from 'react'
+import StyledModal, { StyledModalProps } from './StyledModal'
+import Input from './inputs/Input'
+import api from '~/lib/api'
+
+export type DownloadURLModalProps = Omit<
+  StyledModalProps,
+  'onConfirm' | 'open' | 'confirmText' | 'cancelText' | 'confirmVariant' | 'children'
+> & {
+  suggestedURL?: string
+  onPreflightSuccess?: (url: string) => void
+}
+
+const DownloadURLModal: React.FC<DownloadURLModalProps> = ({
+  suggestedURL,
+  onPreflightSuccess,
+  ...modalProps
+}) => {
+  const [url, setUrl] = useState<string>(suggestedURL || '')
+  const [messages, setMessages] = useState<string[]>([])
+  const [passedPreflight, setPassedPreflight] = useState<boolean>(false)
+
+  async function runPreflightCheck(downloadUrl: string) {
+    try {
+      setMessages([`Running preflight check for URL: ${downloadUrl}`])
+      const res = await api.downloadRemoteMapRegionPreflight(downloadUrl)
+      if ('message' in res) {
+        throw new Error(res.message)
+      }
+
+      setMessages((prev) => [
+        ...prev,
+        `Preflight check passed. Filename: ${res.filename}, Size: ${(res.size / (1024 * 1024)).toFixed(2)} MB`,
+      ])
+      setPassedPreflight(true)
+    } catch (error) {
+      console.error('Preflight check failed:', error)
+      setMessages((prev) => [...prev, `Preflight check failed: ${error.message}`])
+      setPassedPreflight(false)
+    }
+  }
+
+  return (
+    <StyledModal
+      {...modalProps}
+      onConfirm={() => runPreflightCheck(url)}
+      open={true}
+      confirmText="Download"
+      confirmIcon="ArrowDownTrayIcon"
+      cancelText="Cancel"
+      confirmVariant="primary"
+      large
+    >
+      <div className="flex flex-col pb-4">
+        <p className="text-gray-700 mb-8">
+          Enter the URL of the map region file you wish to download. The URL must be publicly
+          reachable and end with .pmtiles. A preflight check will be run to verify the file's
+          availability, type, and approximate size.
+        </p>
+        <Input
+          name="download-url"
+          label=""
+          placeholder={suggestedURL || 'Enter download URL...'}
+          className="mb-4"
+          value={url}
+          onChange={(e) => setUrl(e.target.value)}
+        />
+        <div className="min-h-24 max-h-96 overflow-y-auto bg-gray-50 p-4 rounded border border-gray-300 text-left">
+          {messages.map((message, idx) => (
+            <p
+              key={idx}
+              className="text-sm text-gray-900 font-mono leading-relaxed break-words mb-3"
+            >
+              {message}
+            </p>
+          ))}
+        </div>
+      </div>
+    </StyledModal>
+  )
+}
+
+export default DownloadURLModal

admin/inertia/components/StyledModal.tsx

@@ -3,11 +3,13 @@ import StyledButton, { StyledButtonProps } from './StyledButton'
 import React from 'react'
 import classNames from '~/lib/classNames'
 
-interface StyledModalProps {
+export type StyledModalProps = {
   onClose?: () => void
   title: string
   cancelText?: string
+  cancelIcon?: StyledButtonProps['icon']
   confirmText?: string
+  confirmIcon?: StyledButtonProps['icon']
   confirmVariant?: StyledButtonProps['variant']
   open: boolean
   onCancel?: () => void
@@ -23,7 +25,9 @@ const StyledModal: React.FC<StyledModalProps> = ({
   open,
   onClose,
   cancelText = 'Cancel',
+  cancelIcon,
   confirmText = 'Confirm',
+  confirmIcon,
   confirmVariant = 'action',
   onCancel,
   onConfirm,
@@ -68,10 +72,11 @@ const StyledModal: React.FC<StyledModalProps> = ({
             <div className="mt-5 sm:mt-6 sm:grid sm:grid-flow-row-dense sm:grid-cols-2 sm:gap-3">
               {cancelText && onCancel && (
                 <StyledButton
-                  variant="secondary"
+                  variant="outline"
                   onClick={() => {
                     if (onCancel) onCancel()
                   }}
+                  icon={cancelIcon}
                 >
                   {cancelText}
                 </StyledButton>
@@ -82,6 +87,7 @@ const StyledModal: React.FC<StyledModalProps> = ({
                   onClick={() => {
                     if (onConfirm) onConfirm()
                   }}
+                  icon={confirmIcon}
                 >
                   {confirmText}
                 </StyledButton>

admin/inertia/components/StyledTable.tsx

@@ -48,7 +48,7 @@ function StyledTable<T extends { [key: string]: any }>({
   return (
     <div
       className={classNames(
-        'w-full overflow-x-auto bg-white ring-1 ring-gray-300 sm:mx-0 sm:rounded-lg p-3 shadow-md',
+        'w-full overflow-x-auto bg-white ring-1 ring-gray-300 sm:mx-0 sm:rounded-lg p-1 shadow-md',
         className
       )}
       ref={ref}
@@ -107,7 +107,7 @@ function StyledTable<T extends { [key: string]: any }>({
             ))}
           {!loading && data.length === 0 && (
             <tr>
-              <td colSpan={columns.length} className="!text-center ">
+              <td colSpan={columns.length} className="!text-center py-8 text-gray-500">
                 {noDataText}
               </td>
             </tr>

admin/inertia/components/inputs/Input.tsx

@@ -0,0 +1,59 @@
+import classNames from "classnames";
+import { InputHTMLAttributes } from "react";
+
+export interface InputProps extends InputHTMLAttributes<HTMLInputElement> {
+  name: string;
+  label: string;
+  className?: string;
+  labelClassName?: string;
+  inputClassName?: string;
+  leftIcon?: React.ReactNode;
+  error?: boolean;
+  required?: boolean;
+}
+
+const Input: React.FC<InputProps> = ({
+  className,
+  label,
+  name,
+  labelClassName,
+  inputClassName,
+  leftIcon,
+  error,
+  required,
+  ...props
+}) => {
+  return (
+    <div className={classNames(className)}>
+      <label
+        htmlFor={name}
+        className={classNames("block text-base/6 font-medium text-gray-700", labelClassName)}
+      >
+        {label}{required ? "*" : ""}
+      </label>
+      <div className="mt-1.5">
+        <div className="relative">
+          {leftIcon && (
+            <div className="absolute left-3 top-1/2 transform -translate-y-1/2">
+              {leftIcon}
+            </div>
+          )}
+          <input
+            id={name}
+            name={name}
+            placeholder={props.placeholder || label}
+            className={classNames(
+              inputClassName,
+              "block w-full rounded-md bg-white px-3 py-2 text-base text-gray-900 border border-gray-400 placeholder:text-gray-400 focus:outline focus:outline-2 focus:-outline-offset-2 focus:outline-primary sm:text-sm/6",
+              leftIcon ? "pl-10" : "pl-3",
+              error ? "!border-red-500 focus:outline-red-500 !bg-red-100" : ""
+            )}
+            {...props}
+          />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default Input;

admin/inertia/hooks/useDebounce.ts

@@ -0,0 +1,23 @@
+import { useRef, useEffect } from "react";
+
+const useDebounce = () => {
+  const timeout = useRef<number | undefined>(400);
+
+  const debounce =
+    (func: Function, wait: number = 0) =>
+    (...args: any[]) => {
+      clearTimeout(timeout.current);
+      timeout.current = window.setTimeout(() => func(...args), wait);
+    };
+
+  useEffect(() => {
+    return () => {
+      if (!timeout.current) return;
+      clearTimeout(timeout.current);
+    };
+  }, []);
+
+  return { debounce };
+};
+
+export default useDebounce;

admin/inertia/lib/api.ts

@@ -46,6 +46,31 @@ class API {
     }
   }
 
+  async downloadRemoteMapRegion(url: string) {
+    try {
+      const response = await this.client.post<{ message: string; filename: string; url: string }>(
+        '/maps/download-remote',
+        { url }
+      )
+      return response.data
+    } catch (error) {
+      console.error('Error downloading remote map region:', error)
+      throw error
+    }
+  }
+
+  async downloadRemoteMapRegionPreflight(url: string) {
+    try {
+      const response = await this.client.post<
+        { filename: string; size: number } | { message: string }
+      >('/maps/download-remote-preflight', { url })
+      return response.data
+    } catch (error) {
+      console.error('Error preflighting remote map region download:', error)
+      throw error
+    }
+  }
+
   async listServices() {
     try {
       const response = await this.client.get<Array<ServiceSlim>>('/system/services')
@@ -86,15 +111,34 @@ class API {
     return await this.client.get<ListZimFilesResponse>('/zim/list')
   }
 
-  async listRemoteZimFiles({ start = 0, count = 12 }: { start?: number; count?: number }) {
+  async listRemoteZimFiles({
+    start = 0,
+    count = 12,
+    query,
+  }: {
+    start?: number
+    count?: number
+    query?: string
+  }) {
     return await this.client.get<ListRemoteZimFilesResponse>('/zim/list-remote', {
       params: {
         start,
         count,
+        query,
       },
     })
   }
 
+  async listActiveZimDownloads(): Promise<string[]> {
+    try {
+      const response = await this.client.get<string[]>('/zim/active-downloads')
+      return response.data
+    } catch (error) {
+      console.error('Error listing active ZIM downloads:', error)
+      throw error
+    }
+  }
+
   async downloadRemoteZimFile(url: string): Promise<{
     message: string
     filename: string

admin/inertia/pages/settings/maps.tsx

@@ -9,6 +9,7 @@ import MissingBaseAssetsAlert from '~/components/layout/MissingBaseAssetsAlert'
 import { useNotifications } from '~/context/NotificationContext'
 import { useState } from 'react'
 import api from '~/lib/api'
+import DownloadURLModal from '~/components/DownloadURLModal'
 
 export default function MapsManager(props: {
   maps: { baseAssetsExist: boolean; regionFiles: FileEntry[] }
@@ -61,6 +62,17 @@ export default function MapsManager(props: {
     )
   }
 
+  async function openDownloadModal() {
+    openModal(
+      <DownloadURLModal
+        title="Download Map File"
+        suggestedURL="https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/"
+        onCancel={() => closeAllModals()}
+      />,
+      'download-map-file-modal'
+    )
+  }
+
   return (
     <SettingsLayout>
       <Head title="Maps Manager" />
@@ -73,9 +85,9 @@ export default function MapsManager(props: {
             </div>
             <StyledButton
               variant="primary"
-              onClick={downloadBaseAssets}
+              onClick={openDownloadModal}
               loading={downloading}
-              icon='CloudArrowDownIcon'
+              icon="CloudArrowDownIcon"
             >
               Download New Map File
             </StyledButton>

admin/inertia/pages/settings/zim/remote-explorer.tsx

@@ -16,6 +16,9 @@ import { useNotifications } from '~/context/NotificationContext'
 import useInternetStatus from '~/hooks/useInternetStatus'
 import Alert from '~/components/Alert'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
+import Input from '~/components/inputs/Input'
+import { IconSearch } from '@tabler/icons-react'
+import useDebounce from '~/hooks/useDebounce'
 
 export default function ZimRemoteExplorer() {
   const tableParentRef = useRef<HTMLDivElement>(null)
@@ -24,17 +27,26 @@ export default function ZimRemoteExplorer() {
   const { addNotification } = useNotifications()
   const { isOnline } = useInternetStatus()
   const { isInstalled } = useServiceInstalledStatus('nomad_kiwix_serve')
+  const { debounce } = useDebounce()
+
+  const [query, setQuery] = useState('')
+  const [queryUI, setQueryUI] = useState('')
+
   const [activeDownloads, setActiveDownloads] = useState<
     Map<string, { status: string; progress: number; speed: string }>
   >(new Map<string, { status: string; progress: number; speed: string }>())
 
+  const debouncedSetQuery = debounce((val: string) => {
+    setQuery(val)
+  }, 400)
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery<ListRemoteZimFilesResponse>({
-      queryKey: ['remote-zim-files'],
+      queryKey: ['remote-zim-files', query],
       queryFn: async ({ pageParam = 0 }) => {
         const pageParsed = parseInt((pageParam as number).toString(), 10)
         const start = isNaN(pageParsed) ? 0 : pageParsed * 12
-        const res = await api.listRemoteZimFiles({ start, count: 12 })
+        const res = await api.listRemoteZimFiles({ start, count: 12, query: query || undefined })
         return res.data
       },
       initialPageParam: 0,
@@ -166,6 +178,20 @@ export default function ZimRemoteExplorer() {
               className="!mt-6"
             />
           )}
+          <div className="flex justify-start mt-4">
+            <Input
+              name="search"
+              label=""
+              placeholder="Search available ZIM files..."
+              value={queryUI}
+              onChange={(e) => {
+                setQueryUI(e.target.value)
+                debouncedSetQuery(e.target.value)
+              }}
+              className="w-1/3"
+              leftIcon={<IconSearch className="w-5 h-5 text-gray-400" />}
+            />
+          </div>
           <StyledTable<RemoteZimFileEntry & { actions?: any }>
             data={flatData.map((i, idx) => {
               const row = virtualizer.getVirtualItems().find((v) => v.index === idx)

admin/start/routes.ts

@@ -53,6 +53,7 @@ router
     router.get('/preflight', [MapsController, 'checkBaseAssets'])
     router.post('/download-base-assets', [MapsController, 'downloadBaseAssets'])
     router.post('/download-remote', [MapsController, 'downloadRemote'])
+    router.post('/download-remote-preflight', [MapsController, 'downloadRemotePreflight'])
     router.delete('/:filename', [MapsController, 'delete'])
   })
   .prefix('/api/maps')
@@ -76,6 +77,7 @@ router
   .group(() => {
     router.get('/list', [ZimController, 'list'])
     router.get('/list-remote', [ZimController, 'listRemote'])
+    router.get('/active-downloads', [ZimController, 'listActiveDownloads'])
     router.post('/download-remote', [ZimController, 'downloadRemote'])
     router.delete('/:filename', [ZimController, 'delete'])
   })

admin/types/zim.ts

@@ -1,59 +1,59 @@
 import { FileEntry } from './files.js'
 
 export type ListZimFilesResponse = {
-    files: FileEntry[]
+  files: FileEntry[]
-    next?: string
+  next?: string
 }
 
 export type ListRemoteZimFilesResponse = {
-    items: RemoteZimFileEntry[];
+  items: RemoteZimFileEntry[]
-    has_more: boolean;
+  has_more: boolean
-    total_count: number;
+  total_count: number
 }
 
 export type RawRemoteZimFileEntry = {
-    id: string;
+  'id': string
-    title: string;
+  'title': string
-    updated: string;
+  'updated': string
-    summary: string;
+  'summary': string
-    language: string;
+  'language': string
-    name: string;
+  'name': string
-    flavour: string;
+  'flavour': string
-    category: string;
+  'category': string
-    tags: string;
+  'tags': string
-    articleCount: number;
+  'articleCount': number
-    mediaCount: number;
+  'mediaCount': number
-    link: Record<string, string>[];
+  'link': Record<string, string>[]
-    author: {
+  'author': {
-        name: string
+    name: string
-    };
+  }
-    publisher: {
+  'publisher': {
-        name: string;
+    name: string
-    };
+  }
-    'dc:issued': string;
+  'dc:issued': string
 }
 
 export type RawListRemoteZimFilesResponse = {
-    '?xml': string;
+  '?xml': string
-    feed: {
+  'feed': {
-        id: string;
+    id: string
-        link: string[];
+    link: string[]
-        title: string;
+    title: string
-        updated: string;
+    updated: string
-        totalResults: number;
+    totalResults: number
-        startIndex: number;
+    startIndex: number
-        itemsPerPage: number;
+    itemsPerPage: number
-        entry: any[];
+    entry: RawRemoteZimFileEntry | RawRemoteZimFileEntry[]
-    }
+  }
 }
 
 export type RemoteZimFileEntry = {
-    id: string;
+  id: string
-    title: string;
+  title: string
-    updated: string;
+  updated: string
-    summary: string;
+  summary: string
-    size_bytes: number;
+  size_bytes: number
-    download_url: string;
+  download_url: string
-    author: string;
+  author: string
-    file_name: string;
+  file_name: string
 }

admin/util/zim.ts

@@ -1,10 +1,15 @@
-import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from "../types/zim.js";
+import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from '../types/zim.js'
-
 
 export function isRawListRemoteZimFilesResponse(obj: any): obj is RawListRemoteZimFilesResponse {
-    return obj && typeof obj === 'object' && 'feed' in obj && 'entry' in obj.feed && Array.isArray(obj.feed.entry);
+  return (
+    obj &&
+    typeof obj === 'object' &&
+    'feed' in obj &&
+    'entry' in obj.feed &&
+    typeof obj.feed.entry === 'object' // could be array or single object but typeof array is technically 'object'
+  )
 }
 
 export function isRawRemoteZimFileEntry(obj: any): obj is RawRemoteZimFileEntry {
-    return obj && typeof obj === 'object' && 'id' in obj && 'title' in obj && 'summary' in obj;
+  return obj && typeof obj === 'object' && 'id' in obj && 'title' in obj && 'summary' in obj
 }