From dd4e7c2c4f3a7063e40b4fab4da73be343fa6551 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Fri, 5 Dec 2025 15:38:04 -0800 Subject: [PATCH] feat: curated zim collections --- admin/app/controllers/maps_controller.ts | 1 - admin/app/controllers/zim_controller.ts | 26 ++- admin/app/models/curated_collection.ts | 39 +++++ .../app/models/curated_collection_resource.ts | 41 +++++ admin/app/services/map_service.ts | 31 ++-- admin/app/services/zim_service.ts | 146 +++++++++++++++-- admin/app/utils/downloads.ts | 68 ++++++-- admin/app/validators/common.ts | 25 ++- admin/app/validators/curated_collections.ts | 21 +++ ...210741_create_curated_collections_table.ts | 22 +++ ...eate_curated_collection_resources_table.ts | 23 +++ .../components/CuratedCollectionCard.tsx | 56 +++++++ admin/inertia/components/DynamicIcon.tsx | 36 +++++ .../components/StyledSectionHeader.tsx | 25 +++ .../layout/MissingBaseAssetsAlert.tsx | 1 + admin/inertia/lib/api.ts | 37 +++++ admin/inertia/pages/settings/apps.tsx | 49 +++--- .../pages/settings/zim/remote-explorer.tsx | 148 +++++++++++++----- admin/start/routes.ts | 3 + admin/types/curated_collections.ts | 2 + admin/types/downloads.ts | 31 +++- admin/types/zim.ts | 2 +- admin/util/broadcast_channels.ts | 5 + admin/util/zim.ts | 22 ++- 24 files changed, 733 insertions(+), 127 deletions(-) create mode 100644 admin/app/models/curated_collection.ts create mode 100644 admin/app/models/curated_collection_resource.ts create mode 100644 admin/app/validators/curated_collections.ts create mode 100644 admin/database/migrations/1764912210741_create_curated_collections_table.ts create mode 100644 admin/database/migrations/1764912270123_create_curated_collection_resources_table.ts create mode 100644 admin/inertia/components/CuratedCollectionCard.tsx create mode 100644 admin/inertia/components/DynamicIcon.tsx create mode 100644 admin/inertia/components/StyledSectionHeader.tsx create mode 100644 admin/types/curated_collections.ts create mode 100644 admin/util/broadcast_channels.ts diff --git a/admin/app/controllers/maps_controller.ts b/admin/app/controllers/maps_controller.ts index 72f65b1..23bf39f 100644 --- a/admin/app/controllers/maps_controller.ts +++ b/admin/app/controllers/maps_controller.ts @@ -39,7 +39,6 @@ export default class MapsController { // For providing a "preflight" check in the UI before actually starting a background download async downloadRemotePreflight({ request }: HttpContext) { const payload = await request.validateUsing(remoteDownloadValidator) - console.log(payload) const info = await this.mapService.downloadRemotePreflight(payload.url) return info } diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index e52c676..5e4a029 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -1,5 +1,9 @@ import { ZimService } from '#services/zim_service' -import { filenameValidator, remoteDownloadValidator } from '#validators/common' +import { + downloadCollectionValidator, + filenameValidator, + remoteDownloadValidator, +} from '#validators/common' import { listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -29,10 +33,30 @@ export default class ZimController { } } + async downloadCollection({ request }: HttpContext) { + const payload = await request.validateUsing(downloadCollectionValidator) + const resource_count = await this.zimService.downloadCollection(payload.slug) + + return { + message: 'Download started successfully', + slug: payload.slug, + resource_count, + } + } + async listActiveDownloads({}: HttpContext) { return this.zimService.listActiveDownloads() } + async listCuratedCollections({}: HttpContext) { + return this.zimService.listCuratedCollections() + } + + async fetchLatestCollections({}: HttpContext) { + const success = await this.zimService.fetchLatestCollections() + return { success } + } + async delete({ request, response }: HttpContext) { const payload = await request.validateUsing(filenameValidator) diff --git a/admin/app/models/curated_collection.ts b/admin/app/models/curated_collection.ts new file mode 100644 index 0000000..c760512 --- /dev/null +++ b/admin/app/models/curated_collection.ts @@ -0,0 +1,39 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import CuratedCollectionResource from './curated_collection_resource.js' +import type { HasMany } from '@adonisjs/lucid/types/relations' +import type { CuratedCollectionType } from '../../types/curated_collections.js' + +export default class CuratedCollection extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare slug: string + + @column() + declare type: CuratedCollectionType + + @column() + declare name: string + + @column() + declare description: string + + @column() + declare icon: string + + @column() + declare language: string + + @hasMany(() => CuratedCollectionResource, { + foreignKey: 'curated_collection_slug', + localKey: 'slug', + }) + declare resources: HasMany + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/models/curated_collection_resource.ts b/admin/app/models/curated_collection_resource.ts new file mode 100644 index 0000000..03e44a1 --- /dev/null +++ b/admin/app/models/curated_collection_resource.ts @@ -0,0 +1,41 @@ +import { DateTime } from 'luxon' +import { BaseModel, belongsTo, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import CuratedCollection from './curated_collection.js' +import type { BelongsTo } from '@adonisjs/lucid/types/relations' + +export default class CuratedCollectionResource extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare curated_collection_slug: string + + @belongsTo(() => CuratedCollection, { + foreignKey: 'slug', + localKey: 'curated_collection_slug', + }) + declare curated_collection: BelongsTo + + @column() + declare title: string + + @column() + declare url: string + + @column() + declare description: string + + @column() + declare size_mb: number + + @column() + declare downloaded: boolean + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index 6fd614c..330d09e 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -9,17 +9,18 @@ import { getFileStatsIfExists, deleteFileIfExists, getFile, + ensureDirectoryExists, } from '../utils/fs.js' import { join } from 'path' import urlJoin from 'url-join' import axios from 'axios' +import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js' const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', 'application/x-gzip', 'application/octet-stream', ] -const BROADCAST_CHANNEL = 'map-downloads' const PMTILES_ATTRIBUTION = 'Protomaps © OpenStreetMap' @@ -30,6 +31,7 @@ export class MapService { private readonly baseStylesFile = 'nomad-base-styles.json' private readonly basemapsAssetsDir = 'basemaps-assets' private readonly baseAssetsTarFile = 'base-assets.tar.gz' + private readonly baseDirPath = join(process.cwd(), this.mapStoragePath) private activeDownloads = new Map() async listRegions() { @@ -43,7 +45,7 @@ export class MapService { } async downloadBaseAssets(url?: string) { - const tempTarPath = join(process.cwd(), this.mapStoragePath, this.baseAssetsTarFile) + const tempTarPath = join(this.baseDirPath, this.baseAssetsTarFile) const defaultTarFileURL = new URL( this.baseAssetsTarFile, @@ -54,15 +56,10 @@ export class MapService { const resolvedURL = url ? new URL(url) : defaultTarFileURL await doResumableDownloadWithRetry({ url: resolvedURL.toString(), - path: tempTarPath, + filepath: tempTarPath, timeout: 30000, max_retries: 2, allowedMimeTypes: BASE_ASSETS_MIME_TYPES, - onProgress(progress) { - console.log( - `Downloading: ${progress.downloadedBytes.toFixed(2)}b / ${progress.totalBytes.toFixed(2)}b` - ) - }, onAttemptError(error, attempt) { console.error(`Attempt ${attempt} to download tar file failed: ${error.message}`) }, @@ -99,16 +96,16 @@ export class MapService { throw new Error('Could not determine filename from URL') } - const path = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) + const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) // Don't await the download, run it in the background doBackgroundDownload({ url, - path, + filepath, timeout: 30000, allowedMimeTypes: PMTILES_MIME_TYPES, forceNew: true, - channel: BROADCAST_CHANNEL, + channel: BROADCAST_CHANNELS.MAP, activeDownloads: this.activeDownloads, }) @@ -150,7 +147,7 @@ export class MapService { throw new Error('Base map assets are missing from storage/maps') } - const baseStylePath = join(process.cwd(), this.mapStoragePath, this.baseStylesFile) + const baseStylePath = join(this.baseDirPath, this.baseStylesFile) const baseStyle = await getFile(baseStylePath, 'string') if (!baseStyle) { throw new Error('Base styles file not found in storage/maps') @@ -189,13 +186,13 @@ export class MapService { } private async listMapStorageItems(): Promise { - const dirPath = join(process.cwd(), this.mapStoragePath) - return await listDirectoryContents(dirPath) + await ensureDirectoryExists(this.baseDirPath) + return await listDirectoryContents(this.baseDirPath) } private async listAllMapStorageItems(): Promise { - const dirPath = join(process.cwd(), this.mapStoragePath) - return await listDirectoryContentsRecursive(dirPath) + await ensureDirectoryExists(this.baseDirPath) + return await listDirectoryContentsRecursive(this.baseDirPath) } private generateSourcesArray(regions: FileEntry[]): BaseStylesFile['sources'][] { @@ -261,7 +258,7 @@ export class MapService { fileName += '.zim' } - const fullPath = join(process.cwd(), this.mapStoragePath, fileName) + const fullPath = join(this.baseDirPath, fileName) const exists = await getFileStatsIfExists(fullPath) if (!exists) { diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 656fb0b..439baa4 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -18,9 +18,16 @@ import { listDirectoryContents, } from '../utils/fs.js' import { join } from 'path' +import { CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js' +import vine from '@vinejs/vine' +import { curatedCollectionsFileSchema } from '#validators/curated_collections' +import CuratedCollection from '#models/curated_collection' +import CuratedCollectionResource from '#models/curated_collection_resource' +import { BROADCAST_CHANNELS } from '../../util/broadcast_channels.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] -const BROADCAST_CHANNEL = 'zim-downloads' +const COLLECTIONS_URL = + 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json' @inject() export class ZimService { @@ -34,7 +41,8 @@ export class ZimService { await ensureDirectoryExists(dirPath) - const files = await listDirectoryContents(dirPath) + const all = await listDirectoryContents(dirPath) + const files = all.filter((item) => item.name.endsWith('.zim')) return { files, @@ -74,7 +82,12 @@ export class ZimService { throw new Error('Invalid response format from remote library') } - const entries = Array.isArray(result.feed.entry) ? result.feed.entry : [result.feed.entry] + const entries = result.feed.entry + ? Array.isArray(result.feed.entry) + ? result.feed.entry + : [result.feed.entry] + : [] + const filtered = entries.filter((entry: any) => { return isRawRemoteZimFileEntry(entry) }) @@ -138,38 +151,97 @@ export class ZimService { throw new Error(`Download already in progress for URL ${url}`) } - await ensureDirectoryExists(join(process.cwd(), this.zimStoragePath)) - // Extract the filename from the URL const filename = url.split('/').pop() if (!filename) { throw new Error('Could not determine filename from URL') } - const path = join(process.cwd(), this.zimStoragePath, filename) + const filepath = join(process.cwd(), this.zimStoragePath, filename) // Don't await the download, run it in the background doBackgroundDownload({ url, - path, - channel: BROADCAST_CHANNEL, + filepath, + channel: BROADCAST_CHANNELS.ZIM, activeDownloads: this.activeDownloads, allowedMimeTypes: ZIM_MIME_TYPES, timeout: 30000, forceNew: true, - onComplete: async () => { - // Restart KIWIX container to pick up new ZIM file - await this.dockerService - .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart') - .catch((error) => { - logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. - }) - }, + onComplete: (url, filepath) => this._downloadRemoteSuccessCallback([url], filepath), }) return filename } + async downloadCollection(slug: string): Promise { + const collection = await CuratedCollection.find(slug) + if (!collection) { + return null + } + + const resources = await collection.related('resources').query().where('downloaded', false) + if (resources.length === 0) { + return null + } + + const downloadUrls = resources.map((res) => res.url) + const downloadFilenames: string[] = [] + + for (const [idx, url] of downloadUrls.entries()) { + const existing = this.activeDownloads.get(url) + if (existing) { + logger.warn(`Download already in progress for URL ${url}, skipping.`) + continue + } + + // Extract the filename from the URL + const filename = url.split('/').pop() + if (!filename) { + logger.warn(`Could not determine filename from URL ${url}, skipping.`) + continue + } + + const filepath = join(process.cwd(), this.zimStoragePath, filename) + downloadFilenames.push(filename) + + const isLastDownload = idx === downloadUrls.length - 1 + + // Don't await the download, run it in the background + doBackgroundDownload({ + url, + filepath, + channel: BROADCAST_CHANNELS.ZIM, + activeDownloads: this.activeDownloads, + allowedMimeTypes: ZIM_MIME_TYPES, + timeout: 30000, + forceNew: true, + onComplete: (url, filepath) => + this._downloadRemoteSuccessCallback([url], filepath, isLastDownload), + }) + } + + return downloadFilenames.length > 0 ? downloadFilenames : null + } + + async _downloadRemoteSuccessCallback(urls: string[], filepath: string, restart = true) { + // Restart KIWIX container to pick up new ZIM file + if (restart) { + await this.dockerService + .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart') + .catch((error) => { + logger.error(`Failed to restart KIWIX container:`, error) // Don't stop the download completion, just log the error. + }) + } + + // Mark any curated collection resources with this download URL as downloaded + const resources = await CuratedCollectionResource.query().whereIn('url', urls) + for (const resource of resources) { + resource.downloaded = true + await resource.save() + } + } + listActiveDownloads(): string[] { return Array.from(this.activeDownloads.keys()) } @@ -179,12 +251,52 @@ export class ZimService { if (entry) { entry.abort() this.activeDownloads.delete(url) - transmit.broadcast(BROADCAST_CHANNEL, { url, status: 'cancelled' }) + transmit.broadcast(BROADCAST_CHANNELS.ZIM, { url, status: 'cancelled' }) return true } return false } + async listCuratedCollections(): Promise { + const collections = await CuratedCollection.query().preload('resources') + return collections.map((collection) => ({ + ...(collection.serialize() as CuratedCollection), + all_downloaded: collection.resources.every((res) => res.downloaded), + })) + } + + async fetchLatestCollections(): Promise { + try { + const response = await axios.get(COLLECTIONS_URL) + + const validated = await vine.validate({ + schema: curatedCollectionsFileSchema, + data: response.data, + }) + + for (const collection of validated.collections) { + const collectionResult = await CuratedCollection.updateOrCreate( + { slug: collection.slug }, + { + ...collection, + type: 'zim', + } + ) + logger.info(`Upserted curated collection: ${collection.slug}`) + + await collectionResult.related('resources').createMany(collection.resources) + logger.info( + `Upserted ${collection.resources.length} resources for collection: ${collection.slug}` + ) + } + + return true + } catch (error) { + logger.error('Failed to download latest Kiwix collections:', error) + return false + } + } + async delete(file: string): Promise { let fileName = file if (!fileName.endsWith('.zim')) { diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index eb99133..81c6222 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -3,15 +3,44 @@ import { DoResumableDownloadParams, DoResumableDownloadProgress, DoResumableDownloadWithRetryParams, + DoSimpleDownloadParams, } from '../../types/downloads.js' import axios from 'axios' import { Transform } from 'stream' -import { deleteFileIfExists, getFileStatsIfExists } from './fs.js' +import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js' import { createWriteStream } from 'fs' import { formatSpeed } from './misc.js' import { DownloadProgress } from '../../types/files.js' import transmit from '@adonisjs/transmit/services/main' import logger from '@adonisjs/core/services/logger' +import path from 'path' + +export async function doSimpleDownload({ + url, + filepath, + timeout = 30000, + signal, +}: DoSimpleDownloadParams): Promise { + const dirname = path.dirname(filepath) + await ensureDirectoryExists(dirname) + + const response = await axios.get(url, { + responseType: 'stream', + signal, + timeout, + }) + const writer = createWriteStream(filepath) + response.data.pipe(writer) + + return new Promise((resolve, reject) => { + writer.on('finish', () => { + resolve(filepath) + }) + writer.on('error', (error) => { + reject(error) + }) + }) +} /** * Perform a resumable download with progress tracking @@ -21,20 +50,21 @@ import logger from '@adonisjs/core/services/logger' */ export async function doResumableDownload({ url, - path, + filepath, timeout = 30000, signal, onProgress, forceNew = false, allowedMimeTypes, }: DoResumableDownloadParams): Promise { + const dirname = path.dirname(filepath) + await ensureDirectoryExists(dirname) + // Check if partial file exists for resume let startByte = 0 let appendMode = false - console.log(`Starting download from ${url} to ${path}`) - console.log('Checking for existing file to resume...') - const existingStats = await getFileStatsIfExists(path) + const existingStats = await getFileStatsIfExists(filepath) if (existingStats && !forceNew) { startByte = existingStats.size appendMode = true @@ -58,14 +88,14 @@ export async function doResumableDownload({ } } - // If file is already complete and not forcing overwrite just return path + // If file is already complete and not forcing overwrite just return filepath if (startByte === totalBytes && totalBytes > 0 && !forceNew) { - return path + return filepath } // If server doesn't support range requests and we have a partial file, delete it if (!supportsRangeRequests && startByte > 0) { - await deleteFileIfExists(path) + await deleteFileIfExists(filepath) startByte = 0 appendMode = false } @@ -115,7 +145,7 @@ export async function doResumableDownload({ }, }) - const writeStream = createWriteStream(path, { + const writeStream = createWriteStream(filepath, { flags: appendMode ? 'a' : 'w', }) @@ -148,7 +178,7 @@ export async function doResumableDownload({ url, }) } - resolve(path) + resolve(filepath) }) // Pipe: response -> progressStream -> writeStream @@ -158,7 +188,7 @@ export async function doResumableDownload({ export async function doResumableDownloadWithRetry({ url, - path, + filepath, signal, timeout = 30000, onProgress, @@ -167,6 +197,9 @@ export async function doResumableDownloadWithRetry({ onAttemptError, allowedMimeTypes, }: DoResumableDownloadWithRetryParams): Promise { + const dirname = path.dirname(filepath) + await ensureDirectoryExists(dirname) + let attempt = 0 let lastError: Error | null = null @@ -174,7 +207,7 @@ export async function doResumableDownloadWithRetry({ try { const result = await doResumableDownload({ url, - path, + filepath, signal, timeout, allowedMimeTypes, @@ -212,15 +245,18 @@ export async function doResumableDownloadWithRetry({ } export async function doBackgroundDownload(params: DoBackgroundDownloadParams): Promise { - const { url, path, channel, activeDownloads, onComplete, ...restParams } = params + const { url, filepath, channel, activeDownloads, onComplete, ...restParams } = params try { + const dirname = path.dirname(filepath) + await ensureDirectoryExists(dirname) + const abortController = new AbortController() activeDownloads.set(url, abortController) await doResumableDownloadWithRetry({ url, - path, + filepath, signal: abortController.signal, ...restParams, onProgress: (progressData) => { @@ -228,10 +264,10 @@ export async function doBackgroundDownload(params: DoBackgroundDownloadParams): }, }) - sendCompletedBroadcast(channel, url, path) + sendCompletedBroadcast(channel, url, filepath) if (onComplete) { - await onComplete(url, path) + await onComplete(url, filepath) } } catch (error) { logger.error(`Background download failed for ${url}: ${error.message}`) diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 56f431e..285d4b9 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -2,17 +2,24 @@ import vine from '@vinejs/vine' export const remoteDownloadValidator = vine.compile( vine.object({ - url: vine.string().url({ - require_tld: false, // Allow local URLs - }).trim(), + url: vine + .string() + .url({ + require_tld: false, // Allow local URLs + }) + .trim(), }) ) export const remoteDownloadValidatorOptional = vine.compile( vine.object({ - url: vine.string().url({ - require_tld: false, // Allow local URLs - }).trim().optional(), + url: vine + .string() + .url({ + require_tld: false, // Allow local URLs + }) + .trim() + .optional(), }) ) @@ -21,3 +28,9 @@ export const filenameValidator = vine.compile( filename: vine.string().trim().minLength(1).maxLength(4096), }) ) + +export const downloadCollectionValidator = vine.compile( + vine.object({ + slug: vine.string(), + }) +) diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts new file mode 100644 index 0000000..f6bfe33 --- /dev/null +++ b/admin/app/validators/curated_collections.ts @@ -0,0 +1,21 @@ +import vine from '@vinejs/vine' + +export const curatedCollectionResourceValidator = vine.object({ + title: vine.string(), + description: vine.string(), + url: vine.string().url(), + size_mb: vine.number().min(0).optional(), +}) + +export const curatedCollectionValidator = vine.object({ + slug: vine.string(), + name: vine.string(), + description: vine.string(), + icon: vine.string(), + language: vine.string().minLength(2).maxLength(5), + resources: vine.array(curatedCollectionResourceValidator).minLength(1), +}) + +export const curatedCollectionsFileSchema = vine.object({ + collections: vine.array(curatedCollectionValidator).minLength(1), +}) diff --git a/admin/database/migrations/1764912210741_create_curated_collections_table.ts b/admin/database/migrations/1764912210741_create_curated_collections_table.ts new file mode 100644 index 0000000..a8cf303 --- /dev/null +++ b/admin/database/migrations/1764912210741_create_curated_collections_table.ts @@ -0,0 +1,22 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'curated_collections' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.string('slug').primary() + table.enum('type', ['zim', 'map']).notNullable() + table.string('name').notNullable() + table.text('description').notNullable() + table.string('icon').notNullable() + table.string('language').notNullable() + table.timestamp('created_at') + table.timestamp('updated_at') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/migrations/1764912270123_create_curated_collection_resources_table.ts b/admin/database/migrations/1764912270123_create_curated_collection_resources_table.ts new file mode 100644 index 0000000..c032489 --- /dev/null +++ b/admin/database/migrations/1764912270123_create_curated_collection_resources_table.ts @@ -0,0 +1,23 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'curated_collection_resources' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id') + table.string('curated_collection_slug').notNullable().references('slug').inTable('curated_collections').onDelete('CASCADE') + table.string('title').notNullable() + table.string('url').notNullable() + table.text('description').notNullable() + table.integer('size_mb').notNullable() + table.boolean('downloaded').notNullable().defaultTo(false) + table.timestamp('created_at') + table.timestamp('updated_at') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} \ No newline at end of file diff --git a/admin/inertia/components/CuratedCollectionCard.tsx b/admin/inertia/components/CuratedCollectionCard.tsx new file mode 100644 index 0000000..85e5722 --- /dev/null +++ b/admin/inertia/components/CuratedCollectionCard.tsx @@ -0,0 +1,56 @@ +import { formatBytes } from '~/lib/util' +import DynamicIcon, { DynamicIconName } from './DynamicIcon' +import { CuratedCollectionWithStatus } from '../../types/downloads' +import classNames from 'classnames' +import { IconCircleCheck } from '@tabler/icons-react' + +export interface CuratedCollectionCardProps { + collection: CuratedCollectionWithStatus + onClick?: (collection: CuratedCollectionWithStatus) => void +} + +const CuratedCollectionCard: React.FC = ({ collection, onClick }) => { + const totalSizeBytes = collection.resources?.reduce( + (acc, resource) => acc + resource.size_mb * 1024 * 1024, + 0 + ) + return ( +
{ + if (collection.all_downloaded) { + return + } + if (onClick) { + onClick(collection) + } + }} + > +
+
+
+ +

{collection.name}

+
+ {collection.all_downloaded && ( +
+ +

All items downloaded

+
+ )} +
+
+

{collection.description}

+

+ Items: {collection.resources?.length} | Size: {formatBytes(totalSizeBytes, 0)} +

+
+ ) +} +export default CuratedCollectionCard diff --git a/admin/inertia/components/DynamicIcon.tsx b/admin/inertia/components/DynamicIcon.tsx new file mode 100644 index 0000000..e1de9c7 --- /dev/null +++ b/admin/inertia/components/DynamicIcon.tsx @@ -0,0 +1,36 @@ +import classNames from 'classnames' +import * as TablerIcons from '@tabler/icons-react' + +export type DynamicIconName = keyof typeof TablerIcons + +interface DynamicIconProps { + icon?: DynamicIconName + className?: string + stroke?: number + onClick?: () => void +} + +/** + * Renders a dynamic icon from the TablerIcons library based on the provided icon name. + * @param icon - The name of the icon to render. + * @param className - Optional additional CSS classes to apply to the icon. + * @param stroke - Optional stroke width for the icon. + * @returns A React element representing the icon, or null if no matching icon is found. + */ +const DynamicIcon: React.FC = ({ icon, className, stroke, onClick }) => { + if (!icon) return null + + const Icon = TablerIcons[icon] + + if (!Icon) { + console.warn(`Icon "${icon}" not found in TablerIcons.`) + return null + } + + return ( + // @ts-ignore + + ) +} + +export default DynamicIcon diff --git a/admin/inertia/components/StyledSectionHeader.tsx b/admin/inertia/components/StyledSectionHeader.tsx new file mode 100644 index 0000000..13e8124 --- /dev/null +++ b/admin/inertia/components/StyledSectionHeader.tsx @@ -0,0 +1,25 @@ +import classNames from 'classnames' +import { JSX } from 'react' + +export interface StyledSectionHeaderProps { + title: string + level?: 1 | 2 | 3 | 4 | 5 | 6 + className?: string +} + +const StyledSectionHeader = ({ title, level = 2, className }: StyledSectionHeaderProps) => { + const Heading = `h${level}` as keyof JSX.IntrinsicElements + return ( + +
+ {title} + + ) +} + +export default StyledSectionHeader diff --git a/admin/inertia/components/layout/MissingBaseAssetsAlert.tsx b/admin/inertia/components/layout/MissingBaseAssetsAlert.tsx index e4a2fc6..b06ef53 100644 --- a/admin/inertia/components/layout/MissingBaseAssetsAlert.tsx +++ b/admin/inertia/components/layout/MissingBaseAssetsAlert.tsx @@ -10,6 +10,7 @@ const MissingBaseAssetsAlert = (props: MissingBaseAssetsAlertProps) => { ( + '/zim/curated-collections' + ) + return response.data + } catch (error) { + console.error('Error listing curated ZIM collections:', error) + throw error + } + } + async listActiveZimDownloads(): Promise { try { const response = await this.client.get('/zim/active-downloads') @@ -153,6 +166,30 @@ class API { } } + async downloadZimCollection(slug: string): Promise<{ + message: string + slug: string + resource_count: number + }> { + try { + const response = await this.client.post('/zim/download-collection', { slug }) + return response.data + } catch (error) { + console.error('Error downloading ZIM collection:', error) + throw error + } + } + + async fetchLatestZimCollections(): Promise<{ success: boolean }> { + try { + const response = await this.client.post<{ success: boolean }>('/zim/fetch-latest-collections') + return response.data + } catch (error) { + console.error('Error fetching latest ZIM collections:', error) + throw error + } + } + async deleteZimFile(key: string) { try { const response = await this.client.delete(`/zim/${key}`) diff --git a/admin/inertia/pages/settings/apps.tsx b/admin/inertia/pages/settings/apps.tsx index 5eb5231..f1388f0 100644 --- a/admin/inertia/pages/settings/apps.tsx +++ b/admin/inertia/pages/settings/apps.tsx @@ -98,6 +98,26 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[] } } + async function handleAffectAction(record: ServiceSlim, action: 'start' | 'stop' | 'restart') { + try { + setLoading(true) + const response = await api.affectService(record.service_name, action) + if (!response.success) { + throw new Error(response.message) + } + + closeAllModals() + + setTimeout(() => { + setLoading(false) + window.location.reload() // Reload the page to reflect changes + }, 3000) // Add small delay to allow for the action to complete + } catch (error) { + console.error(`Error affecting service ${record.service_name}:`, error) + showError(`Failed to ${action} service: ${error.message || 'Unknown error'}`) + } + } + const AppActions = ({ record }: { record: ServiceSlim }) => { if (!record) return null if (!record.installed) { @@ -116,26 +136,6 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[] ) } - async function handleAffectAction(action: 'start' | 'stop' | 'restart') { - try { - setLoading(true) - const response = await api.affectService(record.service_name, action) - if (!response.success) { - throw new Error(response.message) - } - - closeAllModals() - - setTimeout(() => { - setLoading(false) - window.location.reload() // Reload the page to reflect changes - }, 3000) // Add small delay to allow for the action to complete - } catch (error) { - console.error(`Error affecting service ${record.service_name}:`, error) - showError(`Failed to ${action} service: ${error.message || 'Unknown error'}`) - } - } - return (
- handleAffectAction(record.status === 'running' ? 'stop' : 'start') + handleAffectAction(record, record.status === 'running' ? 'stop' : 'start') } onCancel={closeAllModals} open={true} @@ -183,7 +183,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[] openModal( handleAffectAction('restart')} + onConfirm={() => handleAffectAction(record, 'restart')} onCancel={closeAllModals} open={true} confirmText={'Restart'} @@ -227,7 +227,7 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[] title: 'Name', render(record) { return ( -
+

{record.friendly_name || record.service_name}

{record.description}

@@ -251,7 +251,8 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[] { accessor: 'installed', title: 'Installed', - render: (record) => (record.installed ? : ''), + render: (record) => + record.installed ? : '', }, { accessor: 'actions', diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index f48f273..0b86dc8 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -1,4 +1,10 @@ -import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query' +import { + keepPreviousData, + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from '@tanstack/react-query' import api from '~/lib/api' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useVirtualizer } from '@tanstack/react-virtual' @@ -19,9 +25,17 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Input from '~/components/inputs/Input' import { IconSearch } from '@tabler/icons-react' import useDebounce from '~/hooks/useDebounce' +import CuratedCollectionCard from '~/components/CuratedCollectionCard' +import StyledSectionHeader from '~/components/StyledSectionHeader' +import { CuratedCollectionWithStatus } from '../../../../types/downloads' +import { BROADCAST_CHANNELS } from '../../../../util/broadcast_channels' + +const CURATED_COLLECTIONS_KEY = 'curated-zim-collections' export default function ZimRemoteExplorer() { + const queryClient = useQueryClient() const tableParentRef = useRef(null) + const { subscribe } = useTransmit() const { openModal, closeAllModals } = useModals() const { addNotification } = useNotifications() @@ -31,7 +45,6 @@ export default function ZimRemoteExplorer() { const [query, setQuery] = useState('') const [queryUI, setQueryUI] = useState('') - const [activeDownloads, setActiveDownloads] = useState< Map >(new Map()) @@ -40,6 +53,36 @@ export default function ZimRemoteExplorer() { setQuery(val) }, 400) + useEffect(() => { + const unsubscribe = subscribe(BROADCAST_CHANNELS.ZIM, (data: any) => { + if (data.url && data.progress?.percentage) { + setActiveDownloads((prev) => + new Map(prev).set(data.url, { + status: data.status, + progress: data.progress.percentage || 0, + speed: data.progress.speed || '0 KB/s', + }) + ) + if (data.status === 'completed') { + addNotification({ + message: `The download for ${data.url} has completed successfully.`, + type: 'success', + }) + } + } + }) + + return () => { + unsubscribe() + } + }, []) + + const { data: curatedCollections } = useQuery({ + queryKey: [CURATED_COLLECTIONS_KEY], + queryFn: () => api.listCuratedZimCollections(), + refetchOnWindowFocus: false, + }) + const { data, fetchNextPage, isFetching, isLoading } = useInfiniteQuery({ queryKey: ['remote-zim-files', query], @@ -68,12 +111,17 @@ export default function ZimRemoteExplorer() { if (parentRef) { const { scrollHeight, scrollTop, clientHeight } = parentRef //once the user has scrolled within 200px of the bottom of the table, fetch more data if we can - if (scrollHeight - scrollTop - clientHeight < 200 && !isFetching && hasMore) { + if ( + scrollHeight - scrollTop - clientHeight < 200 && + !isFetching && + hasMore && + flatData.length > 0 + ) { fetchNextPage() } } }, - [fetchNextPage, isFetching, hasMore] + [fetchNextPage, isFetching, hasMore, flatData.length] ) const virtualizer = useVirtualizer({ @@ -88,12 +136,24 @@ export default function ZimRemoteExplorer() { fetchOnBottomReached(tableParentRef.current) }, [fetchOnBottomReached]) - async function confirmDownload(record: RemoteZimFileEntry) { + async function confirmDownload(record: RemoteZimFileEntry | CuratedCollectionWithStatus) { + const isCollection = 'resources' in record openModal( { - downloadFile(record) + if (isCollection) { + if (record.all_downloaded) { + addNotification({ + message: `All resources in the collection "${record.name}" have already been downloaded.`, + type: 'info', + }) + return + } + downloadCollection(record) + } else { + downloadFile(record) + } closeAllModals() }} onCancel={closeAllModals} @@ -103,8 +163,9 @@ export default function ZimRemoteExplorer() { confirmVariant="primary" >

- Are you sure you want to download {record.title}? It may take some time - for it to be available depending on the file size and your internet connection. The Kiwix + Are you sure you want to download{' '} + {isCollection ? record.name : record.title}? It may take some time for it + to be available depending on the file size and your internet connection. The Kiwix application will be restarted after the download is complete.

, @@ -112,30 +173,6 @@ export default function ZimRemoteExplorer() { ) } - useEffect(() => { - const unsubscribe = subscribe('zim-downloads', (data: any) => { - if (data.url && data.progress?.percentage) { - setActiveDownloads((prev) => - new Map(prev).set(data.url, { - status: data.status, - progress: data.progress.percentage || 0, - speed: data.progress.speed || '0 KB/s', - }) - ) - if (data.status === 'completed') { - addNotification({ - message: `The download for ${data.url} has completed successfully.`, - type: 'success', - }) - } - } - }) - - return () => { - unsubscribe() - } - }, []) - async function downloadFile(record: RemoteZimFileEntry) { try { await api.downloadRemoteZimFile(record.download_url) @@ -144,6 +181,14 @@ export default function ZimRemoteExplorer() { } } + async function downloadCollection(record: CuratedCollectionWithStatus) { + try { + await api.downloadZimCollection(record.slug) + } catch (error) { + console.error('Error downloading collection:', error) + } + } + const EntryProgressBar = useCallback( ({ url }: { url: string }) => { const entry = activeDownloads.get(url) @@ -152,15 +197,35 @@ export default function ZimRemoteExplorer() { [activeDownloads] ) + const fetchLatestCollections = useMutation({ + mutationFn: () => api.fetchLatestZimCollections(), + onSuccess: () => { + addNotification({ + message: 'Successfully fetched the latest ZIM collections.', + type: 'success', + }) + queryClient.invalidateQueries({ queryKey: [CURATED_COLLECTIONS_KEY] }) + }, + }) + return (
-

ZIM Remote Explorer

-

- Browse and download remote ZIM files from the Kiwix repository! -

+
+
+

ZIM Remote Explorer

+

Browse and download ZIM files for offline reading!

+
+ fetchLatestCollections.mutate()} + disabled={fetchLatestCollections.isPending} + icon="CloudArrowDownIcon" + > + Fetch Latest Collections + +
{!isOnline && ( )} + +
+ {curatedCollections?.map((collection) => ( + confirmDownload(collection)} + /> + ))} +
+
onComplete?: (url: string, path: string) => void | Promise } + +export type CuratedCollection = { + name: string + slug: string + description: string + icon: string + language: string + resources: { + title: string + description: string + size_mb: number + url: string + }[] +} + +export type CuratedCollectionWithStatus = CuratedCollection & { + all_downloaded: boolean +} + +export type CuratedCollectionsFile = { + collections: CuratedCollection[] +} diff --git a/admin/types/zim.ts b/admin/types/zim.ts index 600a46f..305d52d 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -43,7 +43,7 @@ export type RawListRemoteZimFilesResponse = { totalResults: number startIndex: number itemsPerPage: number - entry: RawRemoteZimFileEntry | RawRemoteZimFileEntry[] + entry?: RawRemoteZimFileEntry | RawRemoteZimFileEntry[] } } diff --git a/admin/util/broadcast_channels.ts b/admin/util/broadcast_channels.ts new file mode 100644 index 0000000..18c0114 --- /dev/null +++ b/admin/util/broadcast_channels.ts @@ -0,0 +1,5 @@ + +export const BROADCAST_CHANNELS = { + ZIM: 'zim-downloads', + MAP: 'map-downloads', +} \ No newline at end of file diff --git a/admin/util/zim.ts b/admin/util/zim.ts index ce9221e..cedebdd 100644 --- a/admin/util/zim.ts +++ b/admin/util/zim.ts @@ -1,13 +1,21 @@ import { RawListRemoteZimFilesResponse, RawRemoteZimFileEntry } from '../types/zim.js' export function isRawListRemoteZimFilesResponse(obj: any): obj is RawListRemoteZimFilesResponse { - return ( - obj && - typeof obj === 'object' && - 'feed' in obj && - 'entry' in obj.feed && - typeof obj.feed.entry === 'object' // could be array or single object but typeof array is technically 'object' - ) + if (!(obj && typeof obj === 'object' && 'feed' in obj)) { + return false + } + if (!obj.feed || typeof obj.feed !== 'object') { + return false + } + if (!('entry' in obj.feed)) { + return true // entry is optional and may be missing if there are no results + } + + if ('entry' in obj.feed && typeof obj.feed.entry !== 'object') { + return false // If entry exists, it must be an object or array + } + + return true } export function isRawRemoteZimFileEntry(obj: any): obj is RawRemoteZimFileEntry {