feat: curated zim collections

admin/app/controllers/maps_controller.ts

@@ -39,7 +39,6 @@ export default class MapsController {
   // For providing a "preflight" check in the UI before actually starting a background download
   async downloadRemotePreflight({ request }: HttpContext) {
     const payload = await request.validateUsing(remoteDownloadValidator)
-    console.log(payload)
     const info = await this.mapService.downloadRemotePreflight(payload.url)
     return info
   }

admin/app/controllers/zim_controller.ts

@@ -1,5 +1,9 @@
 import { ZimService } from '#services/zim_service'
-import { filenameValidator, remoteDownloadValidator } from '#validators/common'
+import {
+  downloadCollectionValidator,
+  filenameValidator,
+  remoteDownloadValidator,
+} from '#validators/common'
 import { listRemoteZimValidator } from '#validators/zim'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
@@ -29,10 +33,30 @@ export default class ZimController {
     }
   }
 
+  async downloadCollection({ request }: HttpContext) {
+    const payload = await request.validateUsing(downloadCollectionValidator)
+    const resource_count = await this.zimService.downloadCollection(payload.slug)
+
+    return {
+      message: 'Download started successfully',
+      slug: payload.slug,
+      resource_count,
+    }
+  }
+
   async listActiveDownloads({}: HttpContext) {
     return this.zimService.listActiveDownloads()
   }
 
+  async listCuratedCollections({}: HttpContext) {
+    return this.zimService.listCuratedCollections()
+  }
+
+  async fetchLatestCollections({}: HttpContext) {
+    const success = await this.zimService.fetchLatestCollections()
+    return { success }
+  }
+
   async delete({ request, response }: HttpContext) {
     const payload = await request.validateUsing(filenameValidator)
 

admin/app/models/curated_collection.ts

@@ -0,0 +1,39 @@
+import { DateTime } from 'luxon'
+import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import CuratedCollectionResource from './curated_collection_resource.js'
+import type { HasMany } from '@adonisjs/lucid/types/relations'
+import type { CuratedCollectionType } from '../../types/curated_collections.js'
+
+export default class CuratedCollection extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare slug: string
+
+  @column()
+  declare type: CuratedCollectionType
+
+  @column()
+  declare name: string
+
+  @column()
+  declare description: string
+
+  @column()
+  declare icon: string
+
+  @column()
+  declare language: string
+
+  @hasMany(() => CuratedCollectionResource, {
+    foreignKey: 'curated_collection_slug',
+    localKey: 'slug',
+  })
+  declare resources: HasMany<typeof CuratedCollectionResource>
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/models/curated_collection_resource.ts

@@ -0,0 +1,41 @@
+import { DateTime } from 'luxon'
+import { BaseModel, belongsTo, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
+import CuratedCollection from './curated_collection.js'
+import type { BelongsTo } from '@adonisjs/lucid/types/relations'
+
+export default class CuratedCollectionResource extends BaseModel {
+  static namingStrategy = new SnakeCaseNamingStrategy()
+
+  @column({ isPrimary: true })
+  declare id: number
+
+  @column()
+  declare curated_collection_slug: string
+
+  @belongsTo(() => CuratedCollection, {
+    foreignKey: 'slug',
+    localKey: 'curated_collection_slug',
+  })
+  declare curated_collection: BelongsTo<typeof CuratedCollection>
+
+  @column()
+  declare title: string
+
+  @column()
+  declare url: string
+
+  @column()
+  declare description: string
+
+  @column()
+  declare size_mb: number
+
+  @column()
+  declare downloaded: boolean
+
+  @column.dateTime({ autoCreate: true })
+  declare created_at: DateTime
+
+  @column.dateTime({ autoCreate: true, autoUpdate: true })
+  declare updated_at: DateTime
+}

admin/app/services/map_service.ts

@@ -9,17 +9,18 @@ import {
   getFileStatsIfExists,
   deleteFileIfExists,
   getFile,
+  ensureDirectoryExists,
 } from '../utils/fs.js'
 import { join } from 'path'
 import urlJoin from 'url-join'
 import axios from 'axios'
+import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
 
 const BASE_ASSETS_MIME_TYPES = [
   'application/gzip',
   'application/x-gzip',
   'application/octet-stream',
 ]
-const BROADCAST_CHANNEL = 'map-downloads'
 
 const PMTILES_ATTRIBUTION =
   '<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
@@ -30,6 +31,7 @@ export class MapService {
   private readonly baseStylesFile = 'nomad-base-styles.json'
   private readonly basemapsAssetsDir = 'basemaps-assets'
   private readonly baseAssetsTarFile = 'base-assets.tar.gz'
+  private readonly baseDirPath = join(process.cwd(), this.mapStoragePath)
   private activeDownloads = new Map<string, AbortController>()
 
   async listRegions() {
@@ -43,7 +45,7 @@ export class MapService {
   }
 
   async downloadBaseAssets(url?: string) {
-    const tempTarPath = join(process.cwd(), this.mapStoragePath, this.baseAssetsTarFile)
+    const tempTarPath = join(this.baseDirPath, this.baseAssetsTarFile)
 
     const defaultTarFileURL = new URL(
       this.baseAssetsTarFile,
@@ -54,15 +56,10 @@ export class MapService {
     const resolvedURL = url ? new URL(url) : defaultTarFileURL
     await doResumableDownloadWithRetry({
       url: resolvedURL.toString(),
-      path: tempTarPath,
+      filepath: tempTarPath,
       timeout: 30000,
       max_retries: 2,
       allowedMimeTypes: BASE_ASSETS_MIME_TYPES,
-      onProgress(progress) {
-        console.log(
-          `Downloading: ${progress.downloadedBytes.toFixed(2)}b / ${progress.totalBytes.toFixed(2)}b`
-        )
-      },
       onAttemptError(error, attempt) {
         console.error(`Attempt ${attempt} to download tar file failed: ${error.message}`)
       },
@@ -99,16 +96,16 @@ export class MapService {
       throw new Error('Could not determine filename from URL')
     }
 
-    const path = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
+    const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
 
     // Don't await the download, run it in the background
     doBackgroundDownload({
       url,
-      path,
+      filepath,
       timeout: 30000,
       allowedMimeTypes: PMTILES_MIME_TYPES,
       forceNew: true,
-      channel: BROADCAST_CHANNEL,
+      channel: BROADCAST_CHANNELS.MAP,
       activeDownloads: this.activeDownloads,
     })
 
@@ -150,7 +147,7 @@ export class MapService {
       throw new Error('Base map assets are missing from storage/maps')
     }
 
-    const baseStylePath = join(process.cwd(), this.mapStoragePath, this.baseStylesFile)
+    const baseStylePath = join(this.baseDirPath, this.baseStylesFile)
     const baseStyle = await getFile(baseStylePath, 'string')
     if (!baseStyle) {
       throw new Error('Base styles file not found in storage/maps')
@@ -189,13 +186,13 @@ export class MapService {
   }
 
   private async listMapStorageItems(): Promise<FileEntry[]> {
-    const dirPath = join(process.cwd(), this.mapStoragePath)
+    await ensureDirectoryExists(this.baseDirPath)
-    return await listDirectoryContents(dirPath)
+    return await listDirectoryContents(this.baseDirPath)
   }
 
   private async listAllMapStorageItems(): Promise<FileEntry[]> {
-    const dirPath = join(process.cwd(), this.mapStoragePath)
+    await ensureDirectoryExists(this.baseDirPath)
-    return await listDirectoryContentsRecursive(dirPath)
+    return await listDirectoryContentsRecursive(this.baseDirPath)
   }
 
   private generateSourcesArray(regions: FileEntry[]): BaseStylesFile['sources'][] {
@@ -261,7 +258,7 @@ export class MapService {
       fileName += '.zim'
     }
 
-    const fullPath = join(process.cwd(), this.mapStoragePath, fileName)
+    const fullPath = join(this.baseDirPath, fileName)
 
     const exists = await getFileStatsIfExists(fullPath)
     if (!exists) {

admin/app/services/zim_service.ts

@@ -18,9 +18,16 @@ import {
   listDirectoryContents,
 } from '../utils/fs.js'
 import { join } from 'path'
+import { CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js'
+import vine from '@vinejs/vine'
+import { curatedCollectionsFileSchema } from '#validators/curated_collections'
+import CuratedCollection from '#models/curated_collection'
+import CuratedCollectionResource from '#models/curated_collection_resource'
+import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
-const BROADCAST_CHANNEL = 'zim-downloads'
+const COLLECTIONS_URL =
+  'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json'
 
 @inject()
 export class ZimService {
@@ -34,7 +41,8 @@ export class ZimService {
 
     await ensureDirectoryExists(dirPath)
 
-    const files = await listDirectoryContents(dirPath)
+    const all = await listDirectoryContents(dirPath)
+    const files = all.filter((item) => item.name.endsWith('.zim'))
 
     return {
       files,
@@ -74,7 +82,12 @@ export class ZimService {
       throw new Error('Invalid response format from remote library')
     }
 
-    const entries = Array.isArray(result.feed.entry) ? result.feed.entry : [result.feed.entry]
+    const entries = result.feed.entry
+      ? Array.isArray(result.feed.entry)
+        ? result.feed.entry
+        : [result.feed.entry]
+      : []
+
     const filtered = entries.filter((entry: any) => {
       return isRawRemoteZimFileEntry(entry)
     })
@@ -138,38 +151,97 @@ export class ZimService {
       throw new Error(`Download already in progress for URL ${url}`)
     }
 
-    await ensureDirectoryExists(join(process.cwd(), this.zimStoragePath))
-
     // Extract the filename from the URL
     const filename = url.split('/').pop()
     if (!filename) {
       throw new Error('Could not determine filename from URL')
     }
 
-    const path = join(process.cwd(), this.zimStoragePath, filename)
+    const filepath = join(process.cwd(), this.zimStoragePath, filename)
 
     // Don't await the download, run it in the background
     doBackgroundDownload({
       url,
-      path,
+      filepath,
-      channel: BROADCAST_CHANNEL,
+      channel: BROADCAST_CHANNELS.ZIM,
       activeDownloads: this.activeDownloads,
       allowedMimeTypes: ZIM_MIME_TYPES,
       timeout: 30000,
       forceNew: true,
-      onComplete: async () => {
+      onComplete: (url, filepath) => this._downloadRemoteSuccessCallback([url], filepath),
-        // Restart KIWIX container to pick up new ZIM file
-        await this.dockerService
-          .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
-          .catch((error) => {
-            logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
-          })
-      },
     })
 
     return filename
   }
 
+  async downloadCollection(slug: string): Promise<string[] | null> {
+    const collection = await CuratedCollection.find(slug)
+    if (!collection) {
+      return null
+    }
+
+    const resources = await collection.related('resources').query().where('downloaded', false)
+    if (resources.length === 0) {
+      return null
+    }
+
+    const downloadUrls = resources.map((res) => res.url)
+    const downloadFilenames: string[] = []
+
+    for (const [idx, url] of downloadUrls.entries()) {
+      const existing = this.activeDownloads.get(url)
+      if (existing) {
+        logger.warn(`Download already in progress for URL ${url}, skipping.`)
+        continue
+      }
+
+      // Extract the filename from the URL
+      const filename = url.split('/').pop()
+      if (!filename) {
+        logger.warn(`Could not determine filename from URL ${url}, skipping.`)
+        continue
+      }
+
+      const filepath = join(process.cwd(), this.zimStoragePath, filename)
+      downloadFilenames.push(filename)
+
+      const isLastDownload = idx === downloadUrls.length - 1
+
+      // Don't await the download, run it in the background
+      doBackgroundDownload({
+        url,
+        filepath,
+        channel: BROADCAST_CHANNELS.ZIM,
+        activeDownloads: this.activeDownloads,
+        allowedMimeTypes: ZIM_MIME_TYPES,
+        timeout: 30000,
+        forceNew: true,
+        onComplete: (url, filepath) =>
+          this._downloadRemoteSuccessCallback([url], filepath, isLastDownload),
+      })
+    }
+
+    return downloadFilenames.length > 0 ? downloadFilenames : null
+  }
+
+  async _downloadRemoteSuccessCallback(urls: string[], filepath: string, restart = true) {
+    // Restart KIWIX container to pick up new ZIM file
+    if (restart) {
+      await this.dockerService
+        .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
+        .catch((error) => {
+          logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error.
+        })
+    }
+
+    // Mark any curated collection resources with this download URL as downloaded
+    const resources = await CuratedCollectionResource.query().whereIn('url', urls)
+    for (const resource of resources) {
+      resource.downloaded = true
+      await resource.save()
+    }
+  }
+
   listActiveDownloads(): string[] {
     return Array.from(this.activeDownloads.keys())
   }
@@ -179,12 +251,52 @@ export class ZimService {
     if (entry) {
       entry.abort()
       this.activeDownloads.delete(url)
-      transmit.broadcast(BROADCAST_CHANNEL, { url, status: 'cancelled' })
+      transmit.broadcast(BROADCAST_CHANNELS.ZIM, { url, status: 'cancelled' })
       return true
     }
     return false
   }
 
+  async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
+    const collections = await CuratedCollection.query().preload('resources')
+    return collections.map((collection) => ({
+      ...(collection.serialize() as CuratedCollection),
+      all_downloaded: collection.resources.every((res) => res.downloaded),
+    }))
+  }
+
+  async fetchLatestCollections(): Promise<boolean> {
+    try {
+      const response = await axios.get<CuratedCollectionsFile>(COLLECTIONS_URL)
+
+      const validated = await vine.validate({
+        schema: curatedCollectionsFileSchema,
+        data: response.data,
+      })
+
+      for (const collection of validated.collections) {
+        const collectionResult = await CuratedCollection.updateOrCreate(
+          { slug: collection.slug },
+          {
+            ...collection,
+            type: 'zim',
+          }
+        )
+        logger.info(`Upserted curated collection: ${collection.slug}`)
+
+        await collectionResult.related('resources').createMany(collection.resources)
+        logger.info(
+          `Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
+        )
+      }
+
+      return true
+    } catch (error) {
+      logger.error('Failed to download latest Kiwix collections:', error)
+      return false
+    }
+  }
+
   async delete(file: string): Promise<void> {
     let fileName = file
     if (!fileName.endsWith('.zim')) {

admin/app/utils/downloads.ts

@@ -3,15 +3,44 @@ import {
   DoResumableDownloadParams,
   DoResumableDownloadProgress,
   DoResumableDownloadWithRetryParams,
+  DoSimpleDownloadParams,
 } from '../../types/downloads.js'
 import axios from 'axios'
 import { Transform } from 'stream'
-import { deleteFileIfExists, getFileStatsIfExists } from './fs.js'
+import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
 import { createWriteStream } from 'fs'
 import { formatSpeed } from './misc.js'
 import { DownloadProgress } from '../../types/files.js'
 import transmit from '@adonisjs/transmit/services/main'
 import logger from '@adonisjs/core/services/logger'
+import path from 'path'
+
+export async function doSimpleDownload({
+  url,
+  filepath,
+  timeout = 30000,
+  signal,
+}: DoSimpleDownloadParams): Promise<string> {
+  const dirname = path.dirname(filepath)
+  await ensureDirectoryExists(dirname)
+
+  const response = await axios.get(url, {
+    responseType: 'stream',
+    signal,
+    timeout,
+  })
+  const writer = createWriteStream(filepath)
+  response.data.pipe(writer)
+
+  return new Promise((resolve, reject) => {
+    writer.on('finish', () => {
+      resolve(filepath)
+    })
+    writer.on('error', (error) => {
+      reject(error)
+    })
+  })
+}
 
 /**
  * Perform a resumable download with progress tracking
@@ -21,20 +50,21 @@ import logger from '@adonisjs/core/services/logger'
  */
 export async function doResumableDownload({
   url,
-  path,
+  filepath,
   timeout = 30000,
   signal,
   onProgress,
   forceNew = false,
   allowedMimeTypes,
 }: DoResumableDownloadParams): Promise<string> {
+  const dirname = path.dirname(filepath)
+  await ensureDirectoryExists(dirname)
+
   // Check if partial file exists for resume
   let startByte = 0
   let appendMode = false
 
-  console.log(`Starting download from ${url} to ${path}`)
+  const existingStats = await getFileStatsIfExists(filepath)
-  console.log('Checking for existing file to resume...')
-  const existingStats = await getFileStatsIfExists(path)
   if (existingStats && !forceNew) {
     startByte = existingStats.size
     appendMode = true
@@ -58,14 +88,14 @@ export async function doResumableDownload({
     }
   }
 
-  // If file is already complete and not forcing overwrite just return path
+  // If file is already complete and not forcing overwrite just return filepath
   if (startByte === totalBytes && totalBytes > 0 && !forceNew) {
-    return path
+    return filepath
   }
 
   // If server doesn't support range requests and we have a partial file, delete it
   if (!supportsRangeRequests && startByte > 0) {
-    await deleteFileIfExists(path)
+    await deleteFileIfExists(filepath)
     startByte = 0
     appendMode = false
   }
@@ -115,7 +145,7 @@ export async function doResumableDownload({
       },
     })
 
-    const writeStream = createWriteStream(path, {
+    const writeStream = createWriteStream(filepath, {
       flags: appendMode ? 'a' : 'w',
     })
 
@@ -148,7 +178,7 @@ export async function doResumableDownload({
           url,
         })
       }
-      resolve(path)
+      resolve(filepath)
     })
 
     // Pipe: response -> progressStream -> writeStream
@@ -158,7 +188,7 @@ export async function doResumableDownload({
 
 export async function doResumableDownloadWithRetry({
   url,
-  path,
+  filepath,
   signal,
   timeout = 30000,
   onProgress,
@@ -167,6 +197,9 @@ export async function doResumableDownloadWithRetry({
   onAttemptError,
   allowedMimeTypes,
 }: DoResumableDownloadWithRetryParams): Promise<string> {
+  const dirname = path.dirname(filepath)
+  await ensureDirectoryExists(dirname)
+
   let attempt = 0
   let lastError: Error | null = null
 
@@ -174,7 +207,7 @@ export async function doResumableDownloadWithRetry({
     try {
       const result = await doResumableDownload({
         url,
-        path,
+        filepath,
         signal,
         timeout,
         allowedMimeTypes,
@@ -212,15 +245,18 @@ export async function doResumableDownloadWithRetry({
 }
 
 export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise<void> {
-  const { url, path, channel, activeDownloads, onComplete, ...restParams } = params
+  const { url, filepath, channel, activeDownloads, onComplete, ...restParams } = params
 
   try {
+    const dirname = path.dirname(filepath)
+    await ensureDirectoryExists(dirname)
+
     const abortController = new AbortController()
     activeDownloads.set(url, abortController)
 
     await doResumableDownloadWithRetry({
       url,
-      path,
+      filepath,
       signal: abortController.signal,
       ...restParams,
       onProgress: (progressData) => {
@@ -228,10 +264,10 @@ export async function doBackgroundDownload(params: DoBackgroundDownloadParams):
       },
     })
 
-    sendCompletedBroadcast(channel, url, path)
+    sendCompletedBroadcast(channel, url, filepath)
 
     if (onComplete) {
-      await onComplete(url, path)
+      await onComplete(url, filepath)
     }
   } catch (error) {
     logger.error(`Background download failed for ${url}: ${error.message}`)

admin/app/validators/common.ts

@@ -2,17 +2,24 @@ import vine from '@vinejs/vine'
 
 export const remoteDownloadValidator = vine.compile(
   vine.object({
-    url: vine.string().url({
+    url: vine
-      require_tld: false, // Allow local URLs
+      .string()
-    }).trim(),
+      .url({
+        require_tld: false, // Allow local URLs
+      })
+      .trim(),
   })
 )
 
 export const remoteDownloadValidatorOptional = vine.compile(
   vine.object({
-    url: vine.string().url({
+    url: vine
-      require_tld: false, // Allow local URLs
+      .string()
-    }).trim().optional(),
+      .url({
+        require_tld: false, // Allow local URLs
+      })
+      .trim()
+      .optional(),
   })
 )
 
@@ -21,3 +28,9 @@ export const filenameValidator = vine.compile(
     filename: vine.string().trim().minLength(1).maxLength(4096),
   })
 )
+
+export const downloadCollectionValidator = vine.compile(
+  vine.object({
+    slug: vine.string(),
+  })
+)

admin/app/validators/curated_collections.ts

@@ -0,0 +1,21 @@
+import vine from '@vinejs/vine'
+
+export const curatedCollectionResourceValidator = vine.object({
+  title: vine.string(),
+  description: vine.string(),
+  url: vine.string().url(),
+  size_mb: vine.number().min(0).optional(),
+})
+
+export const curatedCollectionValidator = vine.object({
+  slug: vine.string(),
+  name: vine.string(),
+  description: vine.string(),
+  icon: vine.string(),
+  language: vine.string().minLength(2).maxLength(5),
+  resources: vine.array(curatedCollectionResourceValidator).minLength(1),
+})
+
+export const curatedCollectionsFileSchema = vine.object({
+  collections: vine.array(curatedCollectionValidator).minLength(1),
+})

admin/database/migrations/1764912210741_create_curated_collections_table.ts

@@ -0,0 +1,22 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'curated_collections'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.string('slug').primary()
+      table.enum('type', ['zim', 'map']).notNullable()
+      table.string('name').notNullable()
+      table.text('description').notNullable()
+      table.string('icon').notNullable()
+      table.string('language').notNullable()
+      table.timestamp('created_at')
+      table.timestamp('updated_at')
+    })
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

admin/database/migrations/1764912270123_create_curated_collection_resources_table.ts

@@ -0,0 +1,23 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'curated_collection_resources'
+
+  async up() {
+    this.schema.createTable(this.tableName, (table) => {
+      table.increments('id')
+      table.string('curated_collection_slug').notNullable().references('slug').inTable('curated_collections').onDelete('CASCADE')
+      table.string('title').notNullable()
+      table.string('url').notNullable()
+      table.text('description').notNullable()
+      table.integer('size_mb').notNullable()
+      table.boolean('downloaded').notNullable().defaultTo(false)
+      table.timestamp('created_at')
+      table.timestamp('updated_at')
+    })
+  }
+
+  async down() {
+    this.schema.dropTable(this.tableName)
+  }
+}

admin/inertia/components/CuratedCollectionCard.tsx

@@ -0,0 +1,56 @@
+import { formatBytes } from '~/lib/util'
+import DynamicIcon, { DynamicIconName } from './DynamicIcon'
+import { CuratedCollectionWithStatus } from '../../types/downloads'
+import classNames from 'classnames'
+import { IconCircleCheck } from '@tabler/icons-react'
+
+export interface CuratedCollectionCardProps {
+  collection: CuratedCollectionWithStatus
+  onClick?: (collection: CuratedCollectionWithStatus) => void
+}
+
+const CuratedCollectionCard: React.FC<CuratedCollectionCardProps> = ({ collection, onClick }) => {
+  const totalSizeBytes = collection.resources?.reduce(
+    (acc, resource) => acc + resource.size_mb * 1024 * 1024,
+    0
+  )
+  return (
+    <div
+      className={classNames(
+        'flex flex-col bg-desert-green rounded-lg p-6 text-white border border-b-desert-green shadow-sm hover:shadow-lg transition-shadow cursor-pointer',
+        { 'opacity-65 cursor-not-allowed !hover:shadow-sm': collection.all_downloaded }
+      )}
+      onClick={() => {
+        if (collection.all_downloaded) {
+          return
+        }
+        if (onClick) {
+          onClick(collection)
+        }
+      }}
+    >
+      <div className="flex items-center mb-4">
+        <div className="flex justify-between w-full items-center">
+          <div className="flex">
+            <DynamicIcon icon={collection.icon as DynamicIconName} className="w-6 h-6 mr-2" />
+            <h3 className="text-lg font-semibold">{collection.name}</h3>
+          </div>
+          {collection.all_downloaded && (
+            <div className="flex items-center">
+              <IconCircleCheck
+                className="w-5 h-5 text-lime-400 ml-2"
+                title="All items downloaded"
+              />
+              <p className="text-lime-400 text-sm ml-1">All items downloaded</p>
+            </div>
+          )}
+        </div>
+      </div>
+      <p className="text-gray-200 grow">{collection.description}</p>
+      <p className="text-gray-200 text-xs mt-2">
+        Items: {collection.resources?.length} | Size: {formatBytes(totalSizeBytes, 0)}
+      </p>
+    </div>
+  )
+}
+export default CuratedCollectionCard

admin/inertia/components/DynamicIcon.tsx

@@ -0,0 +1,36 @@
+import classNames from 'classnames'
+import * as TablerIcons from '@tabler/icons-react'
+
+export type DynamicIconName = keyof typeof TablerIcons
+
+interface DynamicIconProps {
+  icon?: DynamicIconName
+  className?: string
+  stroke?: number
+  onClick?: () => void
+}
+
+/**
+ * Renders a dynamic icon from the TablerIcons library based on the provided icon name.
+ * @param icon - The name of the icon to render.
+ * @param className - Optional additional CSS classes to apply to the icon.
+ * @param stroke - Optional stroke width for the icon.
+ * @returns A React element representing the icon, or null if no matching icon is found.
+ */
+const DynamicIcon: React.FC<DynamicIconProps> = ({ icon, className, stroke, onClick }) => {
+  if (!icon) return null
+
+  const Icon = TablerIcons[icon]
+
+  if (!Icon) {
+    console.warn(`Icon "${icon}" not found in TablerIcons.`)
+    return null
+  }
+
+  return (
+    // @ts-ignore
+    <Icon className={classNames('h-5 w-5', className)} stroke={stroke || 2} onClick={onClick} />
+  )
+}
+
+export default DynamicIcon

admin/inertia/components/StyledSectionHeader.tsx

@@ -0,0 +1,25 @@
+import classNames from 'classnames'
+import { JSX } from 'react'
+
+export interface StyledSectionHeaderProps {
+  title: string
+  level?: 1 | 2 | 3 | 4 | 5 | 6
+  className?: string
+}
+
+const StyledSectionHeader = ({ title, level = 2, className }: StyledSectionHeaderProps) => {
+  const Heading = `h${level}` as keyof JSX.IntrinsicElements
+  return (
+    <Heading
+      className={classNames(
+        'text-2xl font-bold text-desert-green mb-6 flex items-center gap-2',
+        className
+      )}
+    >
+      <div className="w-1 h-6 bg-desert-green" />
+      {title}
+    </Heading>
+  )
+}
+
+export default StyledSectionHeader

admin/inertia/components/layout/MissingBaseAssetsAlert.tsx

@@ -10,6 +10,7 @@ const MissingBaseAssetsAlert = (props: MissingBaseAssetsAlertProps) => {
     <AlertWithButton
       title="The base map assets have not been installed. Please download them first to enable map functionality."
       type="warning"
+      variant="solid"
       className="!mt-6"
       buttonProps={{
         variant: 'secondary',

admin/inertia/lib/api.ts

@@ -3,6 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi
 import { ServiceSlim } from '../../types/services'
 import { FileEntry } from '../../types/files'
 import { SystemInformationResponse } from '../../types/system'
+import { CuratedCollectionWithStatus } from '../../types/downloads'
 
 class API {
   private client: AxiosInstance
@@ -129,6 +130,18 @@ class API {
     })
   }
 
+  async listCuratedZimCollections() {
+    try {
+      const response = await this.client.get<CuratedCollectionWithStatus[]>(
+        '/zim/curated-collections'
+      )
+      return response.data
+    } catch (error) {
+      console.error('Error listing curated ZIM collections:', error)
+      throw error
+    }
+  }
+
   async listActiveZimDownloads(): Promise<string[]> {
     try {
       const response = await this.client.get<string[]>('/zim/active-downloads')
@@ -153,6 +166,30 @@ class API {
     }
   }
 
+  async downloadZimCollection(slug: string): Promise<{
+    message: string
+    slug: string
+    resource_count: number
+  }> {
+    try {
+      const response = await this.client.post('/zim/download-collection', { slug })
+      return response.data
+    } catch (error) {
+      console.error('Error downloading ZIM collection:', error)
+      throw error
+    }
+  }
+
+  async fetchLatestZimCollections(): Promise<{ success: boolean }> {
+    try {
+      const response = await this.client.post<{ success: boolean }>('/zim/fetch-latest-collections')
+      return response.data
+    } catch (error) {
+      console.error('Error fetching latest ZIM collections:', error)
+      throw error
+    }
+  }
+
   async deleteZimFile(key: string) {
     try {
       const response = await this.client.delete(`/zim/${key}`)

admin/inertia/pages/settings/apps.tsx

@@ -98,6 +98,26 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
     }
   }
 
+  async function handleAffectAction(record: ServiceSlim, action: 'start' | 'stop' | 'restart') {
+    try {
+      setLoading(true)
+      const response = await api.affectService(record.service_name, action)
+      if (!response.success) {
+        throw new Error(response.message)
+      }
+
+      closeAllModals()
+
+      setTimeout(() => {
+        setLoading(false)
+        window.location.reload() // Reload the page to reflect changes
+      }, 3000) // Add small delay to allow for the action to complete
+    } catch (error) {
+      console.error(`Error affecting service ${record.service_name}:`, error)
+      showError(`Failed to ${action} service: ${error.message || 'Unknown error'}`)
+    }
+  }
+
   const AppActions = ({ record }: { record: ServiceSlim }) => {
     if (!record) return null
     if (!record.installed) {
@@ -116,26 +136,6 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
       )
     }
 
-    async function handleAffectAction(action: 'start' | 'stop' | 'restart') {
-      try {
-        setLoading(true)
-        const response = await api.affectService(record.service_name, action)
-        if (!response.success) {
-          throw new Error(response.message)
-        }
-
-        closeAllModals()
-
-        setTimeout(() => {
-          setLoading(false)
-          window.location.reload() // Reload the page to reflect changes
-        }, 3000) // Add small delay to allow for the action to complete
-      } catch (error) {
-        console.error(`Error affecting service ${record.service_name}:`, error)
-        showError(`Failed to ${action} service: ${error.message || 'Unknown error'}`)
-      }
-    }
-
     return (
       <div className="flex space-x-2">
         <StyledButton
@@ -156,7 +156,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
                   <StyledModal
                     title={`${record.status === 'running' ? 'Stop' : 'Start'} Service?`}
                     onConfirm={() =>
-                      handleAffectAction(record.status === 'running' ? 'stop' : 'start')
+                      handleAffectAction(record, record.status === 'running' ? 'stop' : 'start')
                     }
                     onCancel={closeAllModals}
                     open={true}
@@ -183,7 +183,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
                   openModal(
                     <StyledModal
                       title={'Restart Service?'}
-                      onConfirm={() => handleAffectAction('restart')}
+                      onConfirm={() => handleAffectAction(record, 'restart')}
                       onCancel={closeAllModals}
                       open={true}
                       confirmText={'Restart'}
@@ -227,7 +227,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
                   title: 'Name',
                   render(record) {
                     return (
-                      <div className='flex flex-col'>
+                      <div className="flex flex-col">
                         <p>{record.friendly_name || record.service_name}</p>
                         <p className="text-sm text-gray-500">{record.description}</p>
                       </div>
@@ -251,7 +251,8 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
                 {
                   accessor: 'installed',
                   title: 'Installed',
-                  render: (record) => (record.installed ? <IconCheck className="h-6 w-6 text-desert-green" /> : ''),
+                  render: (record) =>
+                    record.installed ? <IconCheck className="h-6 w-6 text-desert-green" /> : '',
                 },
                 {
                   accessor: 'actions',

admin/inertia/pages/settings/zim/remote-explorer.tsx

@@ -1,4 +1,10 @@
-import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query'
+import {
+  keepPreviousData,
+  useInfiniteQuery,
+  useMutation,
+  useQuery,
+  useQueryClient,
+} from '@tanstack/react-query'
 import api from '~/lib/api'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useVirtualizer } from '@tanstack/react-virtual'
@@ -19,9 +25,17 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import Input from '~/components/inputs/Input'
 import { IconSearch } from '@tabler/icons-react'
 import useDebounce from '~/hooks/useDebounce'
+import CuratedCollectionCard from '~/components/CuratedCollectionCard'
+import StyledSectionHeader from '~/components/StyledSectionHeader'
+import { CuratedCollectionWithStatus } from '../../../../types/downloads'
+import { BROADCAST_CHANNELS } from '../../../../util/broadcast_channels'
+
+const CURATED_COLLECTIONS_KEY = 'curated-zim-collections'
 
 export default function ZimRemoteExplorer() {
+  const queryClient = useQueryClient()
   const tableParentRef = useRef<HTMLDivElement>(null)
+
   const { subscribe } = useTransmit()
   const { openModal, closeAllModals } = useModals()
   const { addNotification } = useNotifications()
@@ -31,7 +45,6 @@ export default function ZimRemoteExplorer() {
 
   const [query, setQuery] = useState('')
   const [queryUI, setQueryUI] = useState('')
-
   const [activeDownloads, setActiveDownloads] = useState<
     Map<string, { status: string; progress: number; speed: string }>
   >(new Map<string, { status: string; progress: number; speed: string }>())
@@ -40,6 +53,36 @@ export default function ZimRemoteExplorer() {
     setQuery(val)
   }, 400)
 
+  useEffect(() => {
+    const unsubscribe = subscribe(BROADCAST_CHANNELS.ZIM, (data: any) => {
+      if (data.url && data.progress?.percentage) {
+        setActiveDownloads((prev) =>
+          new Map(prev).set(data.url, {
+            status: data.status,
+            progress: data.progress.percentage || 0,
+            speed: data.progress.speed || '0 KB/s',
+          })
+        )
+        if (data.status === 'completed') {
+          addNotification({
+            message: `The download for ${data.url} has completed successfully.`,
+            type: 'success',
+          })
+        }
+      }
+    })
+
+    return () => {
+      unsubscribe()
+    }
+  }, [])
+
+  const { data: curatedCollections } = useQuery({
+    queryKey: [CURATED_COLLECTIONS_KEY],
+    queryFn: () => api.listCuratedZimCollections(),
+    refetchOnWindowFocus: false,
+  })
+
   const { data, fetchNextPage, isFetching, isLoading } =
     useInfiniteQuery<ListRemoteZimFilesResponse>({
       queryKey: ['remote-zim-files', query],
@@ -68,12 +111,17 @@ export default function ZimRemoteExplorer() {
       if (parentRef) {
         const { scrollHeight, scrollTop, clientHeight } = parentRef
         //once the user has scrolled within 200px of the bottom of the table, fetch more data if we can
-        if (scrollHeight - scrollTop - clientHeight < 200 && !isFetching && hasMore) {
+        if (
+          scrollHeight - scrollTop - clientHeight < 200 &&
+          !isFetching &&
+          hasMore &&
+          flatData.length > 0
+        ) {
           fetchNextPage()
         }
       }
     },
-    [fetchNextPage, isFetching, hasMore]
+    [fetchNextPage, isFetching, hasMore, flatData.length]
   )
 
   const virtualizer = useVirtualizer({
@@ -88,12 +136,24 @@ export default function ZimRemoteExplorer() {
     fetchOnBottomReached(tableParentRef.current)
   }, [fetchOnBottomReached])
 
-  async function confirmDownload(record: RemoteZimFileEntry) {
+  async function confirmDownload(record: RemoteZimFileEntry | CuratedCollectionWithStatus) {
+    const isCollection = 'resources' in record
     openModal(
       <StyledModal
         title="Confirm Download?"
         onConfirm={() => {
-          downloadFile(record)
+          if (isCollection) {
+            if (record.all_downloaded) {
+              addNotification({
+                message: `All resources in the collection "${record.name}" have already been downloaded.`,
+                type: 'info',
+              })
+              return
+            }
+            downloadCollection(record)
+          } else {
+            downloadFile(record)
+          }
           closeAllModals()
         }}
         onCancel={closeAllModals}
@@ -103,8 +163,9 @@ export default function ZimRemoteExplorer() {
         confirmVariant="primary"
       >
         <p className="text-gray-700">
-          Are you sure you want to download <strong>{record.title}</strong>? It may take some time
+          Are you sure you want to download{' '}
-          for it to be available depending on the file size and your internet connection. The Kiwix
+          <strong>{isCollection ? record.name : record.title}</strong>? It may take some time for it
+          to be available depending on the file size and your internet connection. The Kiwix
           application will be restarted after the download is complete.
         </p>
       </StyledModal>,
@@ -112,30 +173,6 @@ export default function ZimRemoteExplorer() {
     )
   }
 
-  useEffect(() => {
-    const unsubscribe = subscribe('zim-downloads', (data: any) => {
-      if (data.url && data.progress?.percentage) {
-        setActiveDownloads((prev) =>
-          new Map(prev).set(data.url, {
-            status: data.status,
-            progress: data.progress.percentage || 0,
-            speed: data.progress.speed || '0 KB/s',
-          })
-        )
-        if (data.status === 'completed') {
-          addNotification({
-            message: `The download for ${data.url} has completed successfully.`,
-            type: 'success',
-          })
-        }
-      }
-    })
-
-    return () => {
-      unsubscribe()
-    }
-  }, [])
-
   async function downloadFile(record: RemoteZimFileEntry) {
     try {
       await api.downloadRemoteZimFile(record.download_url)
@@ -144,6 +181,14 @@ export default function ZimRemoteExplorer() {
     }
   }
 
+  async function downloadCollection(record: CuratedCollectionWithStatus) {
+    try {
+      await api.downloadZimCollection(record.slug)
+    } catch (error) {
+      console.error('Error downloading collection:', error)
+    }
+  }
+
   const EntryProgressBar = useCallback(
     ({ url }: { url: string }) => {
       const entry = activeDownloads.get(url)
@@ -152,15 +197,35 @@ export default function ZimRemoteExplorer() {
     [activeDownloads]
   )
 
+  const fetchLatestCollections = useMutation({
+    mutationFn: () => api.fetchLatestZimCollections(),
+    onSuccess: () => {
+      addNotification({
+        message: 'Successfully fetched the latest ZIM collections.',
+        type: 'success',
+      })
+      queryClient.invalidateQueries({ queryKey: [CURATED_COLLECTIONS_KEY] })
+    },
+  })
+
   return (
     <SettingsLayout>
       <Head title="ZIM Remote Explorer | Project N.O.M.A.D." />
       <div className="xl:pl-72 w-full">
         <main className="px-12 py-6">
-          <h1 className="text-4xl font-semibold mb-2">ZIM Remote Explorer</h1>
+          <div className="flex justify-between items-center">
-          <p className="text-gray-500">
+            <div className="flex flex-col">
-            Browse and download remote ZIM files from the Kiwix repository!
+              <h1 className="text-4xl font-semibold mb-2">ZIM Remote Explorer</h1>
-          </p>
+              <p className="text-gray-500">Browse and download ZIM files for offline reading!</p>
+            </div>
+            <StyledButton
+              onClick={() => fetchLatestCollections.mutate()}
+              disabled={fetchLatestCollections.isPending}
+              icon="CloudArrowDownIcon"
+            >
+              Fetch Latest Collections
+            </StyledButton>
+          </div>
           {!isOnline && (
             <Alert
               title="No internet connection. You may not be able to download files."
@@ -178,6 +243,17 @@ export default function ZimRemoteExplorer() {
               className="!mt-6"
             />
           )}
+          <StyledSectionHeader title="Curated ZIM Collections" className="mt-8 mb-4" />
+          <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-8">
+            {curatedCollections?.map((collection) => (
+              <CuratedCollectionCard
+                key={collection.slug}
+                collection={collection}
+                onClick={(collection) => confirmDownload(collection)}
+              />
+            ))}
+          </div>
+          <StyledSectionHeader title="Browse the Kiwix Library" className="mt-12 mb-4" />
           <div className="flex justify-start mt-4">
             <Input
               name="search"

admin/start/routes.ts

@@ -78,7 +78,10 @@ router
     router.get('/list', [ZimController, 'list'])
     router.get('/list-remote', [ZimController, 'listRemote'])
     router.get('/active-downloads', [ZimController, 'listActiveDownloads'])
+    router.get('/curated-collections', [ZimController, 'listCuratedCollections'])
+    router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections'])
     router.post('/download-remote', [ZimController, 'downloadRemote'])
+    router.post('/download-collection', [ZimController, 'downloadCollection'])
     router.delete('/:filename', [ZimController, 'delete'])
   })
   .prefix('/api/zim')

admin/types/curated_collections.ts

@@ -0,0 +1,2 @@
+
+export type CuratedCollectionType = 'zim' | 'map'

admin/types/downloads.ts

@@ -1,6 +1,13 @@
+export type DoSimpleDownloadParams = {
+  url: string
+  filepath: string
+  timeout: number
+  signal?: AbortSignal
+}
+
 export type DoResumableDownloadParams = {
   url: string
-  path: string
+  filepath: string
   timeout: number
   allowedMimeTypes: string[]
   signal?: AbortSignal
@@ -30,3 +37,25 @@ export type DoBackgroundDownloadParams = Omit<
   activeDownloads: Map<string, AbortController>
   onComplete?: (url: string, path: string) => void | Promise<void>
 }
+
+export type CuratedCollection = {
+  name: string
+  slug: string
+  description: string
+  icon: string
+  language: string
+  resources: {
+    title: string
+    description: string
+    size_mb: number
+    url: string
+  }[]
+}
+
+export type CuratedCollectionWithStatus = CuratedCollection & {
+  all_downloaded: boolean
+}
+
+export type CuratedCollectionsFile = {
+  collections: CuratedCollection[]
+}

admin/types/zim.ts

@@ -43,7 +43,7 @@ export type RawListRemoteZimFilesResponse = {
     totalResults: number
     startIndex: number
     itemsPerPage: number
-    entry: RawRemoteZimFileEntry | RawRemoteZimFileEntry[]
+    entry?: RawRemoteZimFileEntry | RawRemoteZimFileEntry[]
   }
 }
 

admin/util/broadcast_channels.ts

@@ -0,0 +1,5 @@
+
+export const BROADCAST_CHANNELS = {
+    ZIM: 'zim-downloads',
+    MAP: 'map-downloads',
+}

admin/util/zim.ts

@@ -1,13 +1,21 @@
 import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from '../types/zim.js'
 
 export function isRawListRemoteZimFilesResponse(obj: any): obj is RawListRemoteZimFilesResponse {
-  return (
+  if (!(obj && typeof obj === 'object' && 'feed' in obj)) {
-    obj &&
+    return false
-    typeof obj === 'object' &&
+  }
-    'feed' in obj &&
+  if (!obj.feed || typeof obj.feed !== 'object') {
-    'entry' in obj.feed &&
+    return false
-    typeof obj.feed.entry === 'object' // could be array or single object but typeof array is technically 'object'
+  }
-  )
+  if (!('entry' in obj.feed)) {
+    return true // entry is optional and may be missing if there are no results
+  }
+
+  if ('entry' in obj.feed && typeof obj.feed.entry !== 'object') {
+    return false // If entry exists, it must be an object or array
+  }
+
+  return true
 }
 
 export function isRawRemoteZimFileEntry(obj: any): obj is RawRemoteZimFileEntry {