From 32d206cfd735f1ad921733752d2ec981955d029a Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Wed, 11 Feb 2026 15:44:46 -0800 Subject: [PATCH] feat: curated content system overhaul --- .../collection_updates_controller.ts | 9 + .../app/controllers/easy_setup_controller.ts | 19 ++ admin/app/controllers/zim_controller.ts | 29 +- admin/app/jobs/run_download_job.ts | 24 +- admin/app/models/collection_manifest.ts | 22 ++ admin/app/models/curated_collection.ts | 39 --- .../app/models/curated_collection_resource.ts | 49 --- admin/app/models/installed_resource.ts | 33 ++ admin/app/models/zim_file_metadata.ts | 30 -- .../services/collection_manifest_service.ts | 311 ++++++++++++++++++ .../app/services/collection_update_service.ts | 130 ++++++++ admin/app/services/map_service.ts | 171 +++++----- admin/app/services/zim_service.ts | 273 +++++---------- admin/app/validators/common.ts | 7 + admin/app/validators/curated_collections.ts | 65 ++-- admin/bin/server.ts | 9 + ...reate_create_collection_manifests_table.ts | 18 + ...create_create_installed_resources_table.ts | 25 ++ ...create_drop_legacy_curated_tables_table.ts | 13 + admin/inertia/components/CategoryCard.tsx | 10 +- .../components/CuratedCollectionCard.tsx | 12 +- .../inertia/components/TierSelectionModal.tsx | 24 +- admin/inertia/lib/api.ts | 45 ++- admin/inertia/lib/collections.ts | 31 ++ admin/inertia/pages/easy-setup/index.tsx | 179 ++-------- admin/inertia/pages/settings/maps.tsx | 8 +- .../pages/settings/zim/remote-explorer.tsx | 108 +----- admin/start/routes.ts | 8 +- admin/types/collections.ts | 86 +++++ admin/types/curated_collections.ts | 2 - admin/types/downloads.ts | 58 +--- admin/types/files.ts | 1 - collections/kiwix-categories.json | 103 ++++++ collections/kiwix.json | 70 ---- collections/maps.json | 201 ++++++++--- collections/wikipedia.json | 19 +- 36 files changed, 1329 insertions(+), 912 deletions(-) create mode 100644 admin/app/controllers/collection_updates_controller.ts create mode 100644 admin/app/models/collection_manifest.ts delete mode 100644 admin/app/models/curated_collection.ts delete mode 100644 admin/app/models/curated_collection_resource.ts create mode 100644 admin/app/models/installed_resource.ts delete mode 100644 admin/app/models/zim_file_metadata.ts create mode 100644 admin/app/services/collection_manifest_service.ts create mode 100644 admin/app/services/collection_update_service.ts create mode 100644 admin/database/migrations/1770849108030_create_create_collection_manifests_table.ts create mode 100644 admin/database/migrations/1770849119787_create_create_installed_resources_table.ts create mode 100644 admin/database/migrations/1770850092871_create_drop_legacy_curated_tables_table.ts create mode 100644 admin/inertia/lib/collections.ts create mode 100644 admin/types/collections.ts delete mode 100644 admin/types/curated_collections.ts delete mode 100644 collections/kiwix.json diff --git a/admin/app/controllers/collection_updates_controller.ts b/admin/app/controllers/collection_updates_controller.ts new file mode 100644 index 0000000..182d1e2 --- /dev/null +++ b/admin/app/controllers/collection_updates_controller.ts @@ -0,0 +1,9 @@ +import { CollectionUpdateService } from '#services/collection_update_service' +import type { HttpContext } from '@adonisjs/core/http' + +export default class CollectionUpdatesController { + async checkForUpdates({}: HttpContext) { + const collectionUpdateService = new CollectionUpdateService() + return await collectionUpdateService.checkForUpdates() + } +} diff --git a/admin/app/controllers/easy_setup_controller.ts b/admin/app/controllers/easy_setup_controller.ts index b5a61b3..29fafdf 100644 --- a/admin/app/controllers/easy_setup_controller.ts +++ b/admin/app/controllers/easy_setup_controller.ts @@ -1,5 +1,6 @@ import { SystemService } from '#services/system_service' import { ZimService } from '#services/zim_service' +import { CollectionManifestService } from '#services/collection_manifest_service' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -26,4 +27,22 @@ export default class EasySetupController { async listCuratedCategories({}: HttpContext) { return await this.zimService.listCuratedCategories() } + + async refreshManifests({}: HttpContext) { + const manifestService = new CollectionManifestService() + const [zimChanged, mapsChanged, wikiChanged] = await Promise.all([ + manifestService.fetchAndCacheSpec('zim_categories'), + manifestService.fetchAndCacheSpec('maps'), + manifestService.fetchAndCacheSpec('wikipedia'), + ]) + + return { + success: true, + changed: { + zim_categories: zimChanged, + maps: mapsChanged, + wikipedia: wikiChanged, + }, + } + } } diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index a095592..17c692e 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -1,6 +1,6 @@ import { ZimService } from '#services/zim_service' import { - downloadCollectionValidator, + downloadCategoryTierValidator, filenameParamValidator, remoteDownloadWithMetadataValidator, selectWikipediaValidator, @@ -25,7 +25,7 @@ export default class ZimController { async downloadRemote({ request }: HttpContext) { const payload = await request.validateUsing(remoteDownloadWithMetadataValidator) - const { filename, jobId } = await this.zimService.downloadRemote(payload.url, payload.metadata) + const { filename, jobId } = await this.zimService.downloadRemote(payload.url) return { message: 'Download started successfully', @@ -35,26 +35,25 @@ export default class ZimController { } } - async downloadCollection({ request }: HttpContext) { - const payload = await request.validateUsing(downloadCollectionValidator) - const resources = await this.zimService.downloadCollection(payload.slug) + async listCuratedCategories({}: HttpContext) { + return await this.zimService.listCuratedCategories() + } + + async downloadCategoryTier({ request }: HttpContext) { + const payload = await request.validateUsing(downloadCategoryTierValidator) + const resources = await this.zimService.downloadCategoryTier( + payload.categorySlug, + payload.tierSlug + ) return { message: 'Download started successfully', - slug: payload.slug, + categorySlug: payload.categorySlug, + tierSlug: payload.tierSlug, resources, } } - async listCuratedCollections({}: HttpContext) { - return this.zimService.listCuratedCollections() - } - - async fetchLatestCollections({}: HttpContext) { - const success = await this.zimService.fetchLatestCollections() - return { success } - } - async delete({ request, response }: HttpContext) { const payload = await request.validateUsing(filenameParamValidator) diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index e3df7cb..24e4031 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -22,7 +22,7 @@ export class RunDownloadJob { } async handle(job: Job) { - const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } = + const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype, resourceMetadata } = job.data as RunDownloadJobParams await doResumableDownload({ @@ -37,6 +37,26 @@ export class RunDownloadJob { }, async onComplete(url) { try { + // Create InstalledResource entry if metadata was provided + if (resourceMetadata) { + const { default: InstalledResource } = await import('#models/installed_resource') + const { DateTime } = await import('luxon') + const { getFileStatsIfExists } = await import('../utils/fs.js') + const stats = await getFileStatsIfExists(filepath) + + await InstalledResource.updateOrCreate( + { resource_id: resourceMetadata.resource_id, resource_type: filetype as 'zim' | 'map' }, + { + version: resourceMetadata.version, + collection_ref: resourceMetadata.collection_ref, + url: url, + file_path: filepath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), + } + ) + } + if (filetype === 'zim') { const dockerService = new DockerService() const zimService = new ZimService(dockerService) @@ -57,7 +77,7 @@ export class RunDownloadJob { } } catch (error) { console.error( - `[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`, + `[RunDownloadJob] Error in download success callback for URL ${url}:`, error ) } diff --git a/admin/app/models/collection_manifest.ts b/admin/app/models/collection_manifest.ts new file mode 100644 index 0000000..4074a44 --- /dev/null +++ b/admin/app/models/collection_manifest.ts @@ -0,0 +1,22 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { ManifestType } from '../../types/collections.js' + +export default class CollectionManifest extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare type: ManifestType + + @column() + declare spec_version: string + + @column({ + consume: (value: string) => (typeof value === 'string' ? JSON.parse(value) : value), + prepare: (value: any) => JSON.stringify(value), + }) + declare spec_data: any + + @column.dateTime() + declare fetched_at: DateTime +} diff --git a/admin/app/models/curated_collection.ts b/admin/app/models/curated_collection.ts deleted file mode 100644 index c760512..0000000 --- a/admin/app/models/curated_collection.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { DateTime } from 'luxon' -import { BaseModel, column, hasMany, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' -import CuratedCollectionResource from './curated_collection_resource.js' -import type { HasMany } from '@adonisjs/lucid/types/relations' -import type { CuratedCollectionType } from '../../types/curated_collections.js' - -export default class CuratedCollection extends BaseModel { - static namingStrategy = new SnakeCaseNamingStrategy() - - @column({ isPrimary: true }) - declare slug: string - - @column() - declare type: CuratedCollectionType - - @column() - declare name: string - - @column() - declare description: string - - @column() - declare icon: string - - @column() - declare language: string - - @hasMany(() => CuratedCollectionResource, { - foreignKey: 'curated_collection_slug', - localKey: 'slug', - }) - declare resources: HasMany - - @column.dateTime({ autoCreate: true }) - declare created_at: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updated_at: DateTime -} diff --git a/admin/app/models/curated_collection_resource.ts b/admin/app/models/curated_collection_resource.ts deleted file mode 100644 index 0b43dd0..0000000 --- a/admin/app/models/curated_collection_resource.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { DateTime } from 'luxon' -import { BaseModel, belongsTo, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' -import CuratedCollection from './curated_collection.js' -import type { BelongsTo } from '@adonisjs/lucid/types/relations' - -export default class CuratedCollectionResource extends BaseModel { - static namingStrategy = new SnakeCaseNamingStrategy() - - static indexes = [ - { - name: 'curated_collection_resources_unique', - columns: ['curated_collection_slug', 'url'], - unique: true, - }, - ] - - @column({ isPrimary: true }) - declare id: number - - @column() - declare curated_collection_slug: string - - @belongsTo(() => CuratedCollection, { - foreignKey: 'slug', - localKey: 'curated_collection_slug', - }) - declare curated_collection: BelongsTo - - @column() - declare title: string - - @column() - declare url: string - - @column() - declare description: string - - @column() - declare size_mb: number - - @column() - declare downloaded: boolean - - @column.dateTime({ autoCreate: true }) - declare created_at: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updated_at: DateTime -} diff --git a/admin/app/models/installed_resource.ts b/admin/app/models/installed_resource.ts new file mode 100644 index 0000000..cee2866 --- /dev/null +++ b/admin/app/models/installed_resource.ts @@ -0,0 +1,33 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' + +export default class InstalledResource extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare resource_id: string + + @column() + declare resource_type: 'zim' | 'map' + + @column() + declare collection_ref: string | null + + @column() + declare version: string + + @column() + declare url: string + + @column() + declare file_path: string + + @column() + declare file_size_bytes: number | null + + @column.dateTime() + declare installed_at: DateTime +} diff --git a/admin/app/models/zim_file_metadata.ts b/admin/app/models/zim_file_metadata.ts deleted file mode 100644 index bc7107d..0000000 --- a/admin/app/models/zim_file_metadata.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { DateTime } from 'luxon' -import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' - -export default class ZimFileMetadata extends BaseModel { - static namingStrategy = new SnakeCaseNamingStrategy() - - @column({ isPrimary: true }) - declare id: number - - @column() - declare filename: string - - @column() - declare title: string - - @column() - declare summary: string | null - - @column() - declare author: string | null - - @column() - declare size_bytes: number | null - - @column.dateTime({ autoCreate: true }) - declare created_at: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updated_at: DateTime -} diff --git a/admin/app/services/collection_manifest_service.ts b/admin/app/services/collection_manifest_service.ts new file mode 100644 index 0000000..9f43464 --- /dev/null +++ b/admin/app/services/collection_manifest_service.ts @@ -0,0 +1,311 @@ +import axios from 'axios' +import vine from '@vinejs/vine' +import logger from '@adonisjs/core/services/logger' +import { DateTime } from 'luxon' +import { join } from 'path' +import CollectionManifest from '#models/collection_manifest' +import InstalledResource from '#models/installed_resource' +import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections' +import { + ensureDirectoryExists, + listDirectoryContents, + getFileStatsIfExists, + ZIM_STORAGE_PATH, +} from '../utils/fs.js' +import type { + ManifestType, + ZimCategoriesSpec, + MapsSpec, + CategoryWithStatus, + CollectionWithStatus, + SpecResource, + SpecTier, +} from '../../types/collections.js' + +const SPEC_URLS: Record = { + zim_categories: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json', + maps: 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json', + wikipedia: 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/wikipedia.json', +} + +const VALIDATORS: Record = { + zim_categories: zimCategoriesSpecSchema, + maps: mapsSpecSchema, + wikipedia: wikipediaSpecSchema, +} + +export class CollectionManifestService { + private readonly mapStoragePath = '/storage/maps' + + // ---- Spec management ---- + + async fetchAndCacheSpec(type: ManifestType): Promise { + try { + const response = await axios.get(SPEC_URLS[type], { timeout: 15000 }) + + const validated = await vine.validate({ + schema: VALIDATORS[type], + data: response.data, + }) + + const existing = await CollectionManifest.find(type) + const specVersion = validated.spec_version + + if (existing) { + const changed = existing.spec_version !== specVersion + existing.spec_version = specVersion + existing.spec_data = validated + existing.fetched_at = DateTime.now() + await existing.save() + return changed + } + + await CollectionManifest.create({ + type, + spec_version: specVersion, + spec_data: validated, + fetched_at: DateTime.now(), + }) + + return true + } catch (error) { + logger.error(`[CollectionManifestService] Failed to fetch spec for ${type}:`, error?.message || error) + return false + } + } + + async getCachedSpec(type: ManifestType): Promise { + const manifest = await CollectionManifest.find(type) + if (!manifest) return null + return manifest.spec_data as T + } + + async getSpecWithFallback(type: ManifestType): Promise { + try { + await this.fetchAndCacheSpec(type) + } catch { + // Fetch failed, will fall back to cache + } + return this.getCachedSpec(type) + } + + // ---- Status computation ---- + + async getCategoriesWithStatus(): Promise { + const spec = await this.getSpecWithFallback('zim_categories') + if (!spec) return [] + + const installedResources = await InstalledResource.query().where('resource_type', 'zim') + const installedMap = new Map(installedResources.map((r) => [r.resource_id, r])) + + return spec.categories.map((category) => ({ + ...category, + installedTierSlug: this.getInstalledTierForCategory(category.tiers, installedMap), + })) + } + + async getMapCollectionsWithStatus(): Promise { + const spec = await this.getSpecWithFallback('maps') + if (!spec) return [] + + const installedResources = await InstalledResource.query().where('resource_type', 'map') + const installedIds = new Set(installedResources.map((r) => r.resource_id)) + + return spec.collections.map((collection) => { + const installedCount = collection.resources.filter((r) => installedIds.has(r.id)).length + return { + ...collection, + all_installed: installedCount === collection.resources.length, + installed_count: installedCount, + total_count: collection.resources.length, + } + }) + } + + // ---- Tier resolution ---- + + static resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] { + const visited = new Set() + return CollectionManifestService._resolveTierResourcesInner(tier, allTiers, visited) + } + + private static _resolveTierResourcesInner( + tier: SpecTier, + allTiers: SpecTier[], + visited: Set + ): SpecResource[] { + if (visited.has(tier.slug)) return [] // cycle detection + visited.add(tier.slug) + + const resources: SpecResource[] = [] + + if (tier.includesTier) { + const included = allTiers.find((t) => t.slug === tier.includesTier) + if (included) { + resources.push(...CollectionManifestService._resolveTierResourcesInner(included, allTiers, visited)) + } + } + + resources.push(...tier.resources) + return resources + } + + getInstalledTierForCategory( + tiers: SpecTier[], + installedMap: Map + ): string | undefined { + // Check from highest tier to lowest (tiers are ordered low to high in spec) + const reversedTiers = [...tiers].reverse() + + for (const tier of reversedTiers) { + const resolved = CollectionManifestService.resolveTierResources(tier, tiers) + if (resolved.length === 0) continue + + const allInstalled = resolved.every((r) => installedMap.has(r.id)) + if (allInstalled) { + return tier.slug + } + } + + return undefined + } + + // ---- Filename parsing ---- + + static parseZimFilename(filename: string): { resource_id: string; version: string } | null { + const name = filename.replace(/\.zim$/, '') + const match = name.match(/^(.+)_(\d{4}-\d{2})$/) + if (!match) return null + return { resource_id: match[1], version: match[2] } + } + + static parseMapFilename(filename: string): { resource_id: string; version: string } | null { + const name = filename.replace(/\.pmtiles$/, '') + const match = name.match(/^(.+)_(\d{4}-\d{2})$/) + if (!match) return null + return { resource_id: match[1], version: match[2] } + } + + // ---- Filesystem reconciliation ---- + + async reconcileFromFilesystem(): Promise<{ zim: number; map: number }> { + let zimCount = 0 + let mapCount = 0 + + // Reconcile ZIM files + try { + const zimDir = join(process.cwd(), ZIM_STORAGE_PATH) + await ensureDirectoryExists(zimDir) + const zimItems = await listDirectoryContents(zimDir) + const zimFiles = zimItems.filter((f) => f.name.endsWith('.zim')) + + // Get spec for URL lookup + const zimSpec = await this.getCachedSpec('zim_categories') + const specResourceMap = new Map() + if (zimSpec) { + for (const cat of zimSpec.categories) { + for (const tier of cat.tiers) { + for (const res of tier.resources) { + specResourceMap.set(res.id, res) + } + } + } + } + + const seenZimIds = new Set() + + for (const file of zimFiles) { + // Skip Wikipedia files (managed by WikipediaSelection model) + if (file.name.startsWith('wikipedia_en_')) continue + + const parsed = CollectionManifestService.parseZimFilename(file.name) + if (!parsed) continue + + seenZimIds.add(parsed.resource_id) + + const specRes = specResourceMap.get(parsed.resource_id) + const filePath = join(zimDir, file.name) + const stats = await getFileStatsIfExists(filePath) + + await InstalledResource.updateOrCreate( + { resource_id: parsed.resource_id, resource_type: 'zim' }, + { + version: parsed.version, + url: specRes?.url || '', + file_path: filePath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), + } + ) + zimCount++ + } + + // Remove entries for ZIM files no longer on disk + const existingZim = await InstalledResource.query().where('resource_type', 'zim') + for (const entry of existingZim) { + if (!seenZimIds.has(entry.resource_id)) { + await entry.delete() + } + } + } catch (error) { + logger.error('[CollectionManifestService] Error reconciling ZIM files:', error) + } + + // Reconcile map files + try { + const mapDir = join(process.cwd(), this.mapStoragePath, 'pmtiles') + await ensureDirectoryExists(mapDir) + const mapItems = await listDirectoryContents(mapDir) + const mapFiles = mapItems.filter((f) => f.name.endsWith('.pmtiles')) + + // Get spec for URL/version lookup + const mapSpec = await this.getCachedSpec('maps') + const mapResourceMap = new Map() + if (mapSpec) { + for (const col of mapSpec.collections) { + for (const res of col.resources) { + mapResourceMap.set(res.id, res) + } + } + } + + const seenMapIds = new Set() + + for (const file of mapFiles) { + const parsed = CollectionManifestService.parseMapFilename(file.name) + if (!parsed) continue + + seenMapIds.add(parsed.resource_id) + + const specRes = mapResourceMap.get(parsed.resource_id) + const filePath = join(mapDir, file.name) + const stats = await getFileStatsIfExists(filePath) + + await InstalledResource.updateOrCreate( + { resource_id: parsed.resource_id, resource_type: 'map' }, + { + version: parsed.version, + url: specRes?.url || '', + file_path: filePath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), + } + ) + mapCount++ + } + + // Remove entries for map files no longer on disk + const existingMaps = await InstalledResource.query().where('resource_type', 'map') + for (const entry of existingMaps) { + if (!seenMapIds.has(entry.resource_id)) { + await entry.delete() + } + } + } catch (error) { + logger.error('[CollectionManifestService] Error reconciling map files:', error) + } + + logger.info(`[CollectionManifestService] Reconciled ${zimCount} ZIM files, ${mapCount} map files`) + return { zim: zimCount, map: mapCount } + } +} diff --git a/admin/app/services/collection_update_service.ts b/admin/app/services/collection_update_service.ts new file mode 100644 index 0000000..8971e2e --- /dev/null +++ b/admin/app/services/collection_update_service.ts @@ -0,0 +1,130 @@ +import logger from '@adonisjs/core/services/logger' +import InstalledResource from '#models/installed_resource' +import { CollectionManifestService } from './collection_manifest_service.js' +import type { + ZimCategoriesSpec, + MapsSpec, + CollectionResourceUpdateInfo, + CollectionUpdateCheckResult, + SpecResource, +} from '../../types/collections.js' + +export class CollectionUpdateService { + private manifestService = new CollectionManifestService() + + async checkForUpdates(): Promise { + const resourceUpdates: CollectionResourceUpdateInfo[] = [] + let specChanged = false + + // Check if specs have changed + try { + const [zimChanged, mapsChanged] = await Promise.all([ + this.manifestService.fetchAndCacheSpec('zim_categories'), + this.manifestService.fetchAndCacheSpec('maps'), + ]) + specChanged = zimChanged || mapsChanged + } catch (error) { + logger.error('[CollectionUpdateService] Failed to fetch latest specs:', error) + } + + // Check for ZIM resource version updates + const zimUpdates = await this.checkZimUpdates() + resourceUpdates.push(...zimUpdates) + + // Check for map resource version updates + const mapUpdates = await this.checkMapUpdates() + resourceUpdates.push(...mapUpdates) + + logger.info( + `[CollectionUpdateService] Update check complete: spec_changed=${specChanged}, resource_updates=${resourceUpdates.length}` + ) + + return { spec_changed: specChanged, resource_updates: resourceUpdates } + } + + private async checkZimUpdates(): Promise { + const updates: CollectionResourceUpdateInfo[] = [] + + try { + const spec = await this.manifestService.getCachedSpec('zim_categories') + if (!spec) return updates + + const installed = await InstalledResource.query().where('resource_type', 'zim') + if (installed.length === 0) return updates + + // Build a map of spec resources by ID for quick lookup + const specResourceMap = new Map() + for (const category of spec.categories) { + for (const tier of category.tiers) { + for (const resource of tier.resources) { + // Only keep the latest version if there are duplicates + const existing = specResourceMap.get(resource.id) + if (!existing || resource.version > existing.version) { + specResourceMap.set(resource.id, resource) + } + } + } + } + + // Compare installed versions against spec versions + for (const entry of installed) { + const specResource = specResourceMap.get(entry.resource_id) + if (!specResource) continue + + if (specResource.version > entry.version) { + updates.push({ + resource_id: entry.resource_id, + installed_version: entry.version, + latest_version: specResource.version, + latest_url: specResource.url, + latest_size_mb: specResource.size_mb, + }) + } + } + } catch (error) { + logger.error('[CollectionUpdateService] Error checking ZIM updates:', error) + } + + return updates + } + + private async checkMapUpdates(): Promise { + const updates: CollectionResourceUpdateInfo[] = [] + + try { + const spec = await this.manifestService.getCachedSpec('maps') + if (!spec) return updates + + const installed = await InstalledResource.query().where('resource_type', 'map') + if (installed.length === 0) return updates + + // Build a map of spec resources by ID + const specResourceMap = new Map() + for (const collection of spec.collections) { + for (const resource of collection.resources) { + specResourceMap.set(resource.id, resource) + } + } + + // Compare installed versions against spec versions + for (const entry of installed) { + const specResource = specResourceMap.get(entry.resource_id) + if (!specResource) continue + + if (specResource.version > entry.version) { + updates.push({ + resource_id: entry.resource_id, + installed_version: entry.version, + latest_version: specResource.version, + latest_url: specResource.url, + latest_size_mb: specResource.size_mb, + }) + } + } + } catch (error) { + logger.error('[CollectionUpdateService] Error checking map updates:', error) + } + + return updates + } +} diff --git a/admin/app/services/map_service.ts b/admin/app/services/map_service.ts index b745510..987c255 100644 --- a/admin/app/services/map_service.ts +++ b/admin/app/services/map_service.ts @@ -1,6 +1,5 @@ import { BaseStylesFile, MapLayer } from '../../types/maps.js' import { - DownloadCollectionOperation, DownloadRemoteSuccessCallback, FileEntry, } from '../../types/files.js' @@ -16,14 +15,11 @@ import { } from '../utils/fs.js' import { join } from 'path' import urlJoin from 'url-join' -import axios from 'axios' import { RunDownloadJob } from '#jobs/run_download_job' import logger from '@adonisjs/core/services/logger' -import { CuratedCollectionsFile, CuratedCollectionWithStatus } from '../../types/downloads.js' -import CuratedCollection from '#models/curated_collection' -import vine from '@vinejs/vine' -import { curatedCollectionsFileSchema } from '#validators/curated_collections' -import CuratedCollectionResource from '#models/curated_collection_resource' +import InstalledResource from '#models/installed_resource' +import { CollectionManifestService } from './collection_manifest_service.js' +import type { CollectionWithStatus, MapsSpec } from '../../types/collections.js' const BASE_ASSETS_MIME_TYPES = [ 'application/gzip', @@ -31,15 +27,11 @@ const BASE_ASSETS_MIME_TYPES = [ 'application/octet-stream', ] -const COLLECTIONS_URL = - 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json' - const PMTILES_ATTRIBUTION = 'Protomaps © OpenStreetMap' const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream'] interface IMapService { - downloadCollection: DownloadCollectionOperation downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback } @@ -99,34 +91,33 @@ export class MapService implements IMapService { return true } - async downloadCollection(slug: string) { - const collection = await CuratedCollection.query() - .where('slug', slug) - .andWhere('type', 'map') - .first() - if (!collection) { - return null - } + async downloadCollection(slug: string): Promise { + const manifestService = new CollectionManifestService() + const spec = await manifestService.getSpecWithFallback('maps') + if (!spec) return null - const resources = await collection.related('resources').query().where('downloaded', false) - if (resources.length === 0) { - return null - } + const collection = spec.collections.find((c) => c.slug === slug) + if (!collection) return null + + // Filter out already installed + const installed = await InstalledResource.query().where('resource_type', 'map') + const installedIds = new Set(installed.map((r) => r.resource_id)) + const toDownload = collection.resources.filter((r) => !installedIds.has(r.id)) + + if (toDownload.length === 0) return null - const downloadUrls = resources.map((res) => res.url) const downloadFilenames: string[] = [] - for (const url of downloadUrls) { - const existing = await RunDownloadJob.getByUrl(url) + for (const resource of toDownload) { + const existing = await RunDownloadJob.getByUrl(resource.url) if (existing) { - logger.warn(`[MapService] Download already in progress for URL ${url}, skipping.`) + logger.warn(`[MapService] Download already in progress for URL ${resource.url}, skipping.`) continue } - // Extract the filename from the URL - const filename = url.split('/').pop() + const filename = resource.url.split('/').pop() if (!filename) { - logger.warn(`[MapService] Could not determine filename from URL ${url}, skipping.`) + logger.warn(`[MapService] Could not determine filename from URL ${resource.url}, skipping.`) continue } @@ -134,12 +125,17 @@ export class MapService implements IMapService { const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) await RunDownloadJob.dispatch({ - url, + url: resource.url, filepath, timeout: 30000, allowedMimeTypes: PMTILES_MIME_TYPES, forceNew: true, filetype: 'map', + resourceMetadata: { + resource_id: resource.id, + version: resource.version, + collection_ref: slug, + }, }) } @@ -147,11 +143,33 @@ export class MapService implements IMapService { } async downloadRemoteSuccessCallback(urls: string[], _: boolean) { - const resources = await CuratedCollectionResource.query().whereIn('url', urls) - for (const resource of resources) { - resource.downloaded = true - await resource.save() - logger.info(`[MapService] Marked resource as downloaded: ${resource.url}`) + // Create InstalledResource entries for downloaded map files + for (const url of urls) { + const filename = url.split('/').pop() + if (!filename) continue + + const parsed = CollectionManifestService.parseMapFilename(filename) + if (!parsed) continue + + const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename) + const stats = await getFileStatsIfExists(filepath) + + try { + const { DateTime } = await import('luxon') + await InstalledResource.updateOrCreate( + { resource_id: parsed.resource_id, resource_type: 'map' }, + { + version: parsed.version, + url: url, + file_path: filepath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), + } + ) + logger.info(`[MapService] Created InstalledResource entry for: ${parsed.resource_id}`) + } catch (error) { + logger.error(`[MapService] Failed to create InstalledResource for ${filename}:`, error) + } } } @@ -182,6 +200,12 @@ export class MapService implements IMapService { ) } + // Parse resource metadata + const parsedFilename = CollectionManifestService.parseMapFilename(filename) + const resourceMetadata = parsedFilename + ? { resource_id: parsedFilename.resource_id, version: parsedFilename.version, collection_ref: null } + : undefined + // Dispatch background job const result = await RunDownloadJob.dispatch({ url, @@ -190,6 +214,7 @@ export class MapService implements IMapService { allowedMimeTypes: PMTILES_MIME_TYPES, forceNew: true, filetype: 'map', + resourceMetadata, }) if (!result.job) { @@ -219,6 +244,7 @@ export class MapService implements IMapService { } // Perform a HEAD request to get the content length + const { default: axios } = await import('axios') const response = await axios.head(url) if (response.status !== 200) { @@ -229,7 +255,7 @@ export class MapService implements IMapService { const size = contentLength ? parseInt(contentLength, 10) : 0 return { filename, size } - } catch (error) { + } catch (error: any) { return { message: `Preflight check failed: ${error.message}` } } } @@ -253,7 +279,7 @@ export class MapService implements IMapService { * This is mainly useful because we need to know what host the user is accessing from in order to * properly generate URLs in the styles file * e.g. user is accessing from "example.com", but we would by default generate "localhost:8080/..." so maps would - * fail to load. + * fail to load. */ const sources = this.generateSourcesArray(host, regions) const baseUrl = this.getPublicFileBaseUrl(host, this.basemapsAssetsDir) @@ -268,53 +294,14 @@ export class MapService implements IMapService { return styles } - async listCuratedCollections(): Promise { - const collections = await CuratedCollection.query().where('type', 'map').preload('resources') - return collections.map((collection) => ({ - ...(collection.serialize() as CuratedCollection), - all_downloaded: collection.resources.every((res) => res.downloaded), - })) + async listCuratedCollections(): Promise { + const manifestService = new CollectionManifestService() + return manifestService.getMapCollectionsWithStatus() } async fetchLatestCollections(): Promise { - try { - const response = await axios.get(COLLECTIONS_URL) - - const validated = await vine.validate({ - schema: curatedCollectionsFileSchema, - data: response.data, - }) - - for (const collection of validated.collections) { - const { resources, ...restCollection } = collection; // we'll handle resources separately - - // Upsert the collection itself - await CuratedCollection.updateOrCreate( - { slug: restCollection.slug }, - { - ...restCollection, - type: 'map', - } - ) - logger.info(`[MapService] Upserted curated collection: ${restCollection.slug}`) - - // Upsert collection's resources - const resourcesResult = await CuratedCollectionResource.updateOrCreateMany('url', resources.map((res) => ({ - ...res, - curated_collection_slug: restCollection.slug, // add the foreign key - }))) - - logger.info( - `[MapService] Upserted ${resourcesResult.length} resources for collection: ${restCollection.slug}` - ) - } - - return true - } catch (error) { - console.error(error) - logger.error(`[MapService] Failed to download latest Kiwix collections:`, error) - return false - } + const manifestService = new CollectionManifestService() + return manifestService.fetchAndCacheSpec('maps') } async ensureBaseAssets(): Promise { @@ -361,7 +348,9 @@ export class MapService implements IMapService { for (const region of regions) { if (region.type === 'file' && region.name.endsWith('.pmtiles')) { - const regionName = region.name.replace('.pmtiles', '') + // Strip .pmtiles and date suffix (e.g. "alaska_2025-12" -> "alaska") for stable source names + const parsed = CollectionManifestService.parseMapFilename(region.name) + const regionName = parsed ? parsed.resource_id : region.name.replace('.pmtiles', '') const source: BaseStylesFile['sources'] = {} const sourceUrl = urlJoin(baseUrl, region.name) @@ -411,11 +400,11 @@ export class MapService implements IMapService { async delete(file: string): Promise { let fileName = file - if (!fileName.endsWith('.zim')) { - fileName += '.zim' + if (!fileName.endsWith('.pmtiles')) { + fileName += '.pmtiles' } - const fullPath = join(this.baseDirPath, fileName) + const fullPath = join(this.baseDirPath, 'pmtiles', fileName) const exists = await getFileStatsIfExists(fullPath) if (!exists) { @@ -423,6 +412,16 @@ export class MapService implements IMapService { } await deleteFileIfExists(fullPath) + + // Clean up InstalledResource entry + const parsed = CollectionManifestService.parseMapFilename(fileName) + if (parsed) { + await InstalledResource.query() + .where('resource_id', parsed.resource_id) + .where('resource_type', 'map') + .delete() + logger.info(`[MapService] Deleted InstalledResource entry for: ${parsed.resource_id}`) + } } /* diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 3f67b50..3818529 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -17,32 +17,21 @@ import { ZIM_STORAGE_PATH, } from '../utils/fs.js' import { join } from 'path' -import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile, WikipediaOption, WikipediaState } from '../../types/downloads.js' +import { WikipediaOption, WikipediaState } from '../../types/downloads.js' import vine from '@vinejs/vine' -import { curatedCategoriesFileSchema, curatedCollectionsFileSchema, wikipediaOptionsFileSchema } from '#validators/curated_collections' -import CuratedCollection from '#models/curated_collection' -import CuratedCollectionResource from '#models/curated_collection_resource' +import { wikipediaOptionsFileSchema } from '#validators/curated_collections' import WikipediaSelection from '#models/wikipedia_selection' -import ZimFileMetadata from '#models/zim_file_metadata' +import InstalledResource from '#models/installed_resource' import { RunDownloadJob } from '#jobs/run_download_job' -import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js' import { SERVICE_NAMES } from '../../constants/service_names.js' +import { CollectionManifestService } from './collection_manifest_service.js' +import type { CategoryWithStatus } from '../../types/collections.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] -const CATEGORIES_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json' -const COLLECTIONS_URL = - 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json' const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/wikipedia.json' - - -interface IZimService { - downloadCollection: DownloadCollectionOperation - downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback -} - @inject() -export class ZimService implements IZimService { +export class ZimService { constructor(private dockerService: DockerService) { } async list() { @@ -52,24 +41,8 @@ export class ZimService implements IZimService { const all = await listDirectoryContents(dirPath) const files = all.filter((item) => item.name.endsWith('.zim')) - // Fetch metadata for all files - const metadataRecords = await ZimFileMetadata.all() - const metadataMap = new Map(metadataRecords.map((m) => [m.filename, m])) - - // Enrich files with metadata - const enrichedFiles = files.map((file) => { - const metadata = metadataMap.get(file.name) - return { - ...file, - title: metadata?.title || null, - summary: metadata?.summary || null, - author: metadata?.author || null, - size_bytes: metadata?.size_bytes || null, - } - }) - return { - files: enrichedFiles, + files, } } @@ -164,10 +137,7 @@ export class ZimService implements IZimService { } } - async downloadRemote( - url: string, - metadata?: { title: string; summary?: string; author?: string; size_bytes?: number } - ): Promise<{ filename: string; jobId?: string }> { + async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> { const parsed = new URL(url) if (!parsed.pathname.endsWith('.zim')) { throw new Error(`Invalid ZIM file URL: ${url}. URL must end with .zim`) @@ -186,19 +156,11 @@ export class ZimService implements IZimService { const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) - // Store metadata if provided - if (metadata) { - await ZimFileMetadata.updateOrCreate( - { filename }, - { - title: metadata.title, - summary: metadata.summary || null, - author: metadata.author || null, - size_bytes: metadata.size_bytes || null, - } - ) - logger.info(`[ZimService] Stored metadata for ZIM file: ${filename}`) - } + // Parse resource metadata for the download job + const parsedFilename = CollectionManifestService.parseZimFilename(filename) + const resourceMetadata = parsedFilename + ? { resource_id: parsedFilename.resource_id, version: parsedFilename.version, collection_ref: null } + : undefined // Dispatch a background download job const result = await RunDownloadJob.dispatch({ @@ -208,6 +170,7 @@ export class ZimService implements IZimService { allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, filetype: 'zim', + resourceMetadata, }) if (!result || !result.job) { @@ -222,44 +185,64 @@ export class ZimService implements IZimService { } } - async downloadCollection(slug: string) { - const collection = await CuratedCollection.query().where('slug', slug).andWhere('type', 'zim').first() - if (!collection) { - return null + async listCuratedCategories(): Promise { + const manifestService = new CollectionManifestService() + return manifestService.getCategoriesWithStatus() + } + + async downloadCategoryTier(categorySlug: string, tierSlug: string): Promise { + const manifestService = new CollectionManifestService() + const spec = await manifestService.getSpecWithFallback('zim_categories') + if (!spec) { + throw new Error('Could not load ZIM categories spec') } - const resources = await collection.related('resources').query().where('downloaded', false) - if (resources.length === 0) { - return null + const category = spec.categories.find((c) => c.slug === categorySlug) + if (!category) { + throw new Error(`Category not found: ${categorySlug}`) } - const downloadUrls = resources.map((res) => res.url) + const tier = category.tiers.find((t) => t.slug === tierSlug) + if (!tier) { + throw new Error(`Tier not found: ${tierSlug}`) + } + + const allResources = CollectionManifestService.resolveTierResources(tier, category.tiers) + + // Filter out already installed + const installed = await InstalledResource.query().where('resource_type', 'zim') + const installedIds = new Set(installed.map((r) => r.resource_id)) + const toDownload = allResources.filter((r) => !installedIds.has(r.id)) + + if (toDownload.length === 0) return null + const downloadFilenames: string[] = [] - for (const url of downloadUrls) { - const existing = await RunDownloadJob.getByUrl(url) - if (existing) { - logger.warn(`[ZimService] Download already in progress for URL ${url}, skipping.`) + for (const resource of toDownload) { + const existingJob = await RunDownloadJob.getByUrl(resource.url) + if (existingJob) { + logger.warn(`[ZimService] Download already in progress for ${resource.url}, skipping.`) continue } - // Extract the filename from the URL - const filename = url.split('/').pop() - if (!filename) { - logger.warn(`[ZimService] Could not determine filename from URL ${url}, skipping.`) - continue - } + const filename = resource.url.split('/').pop() + if (!filename) continue downloadFilenames.push(filename) const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) await RunDownloadJob.dispatch({ - url, + url: resource.url, filepath, timeout: 30000, allowedMimeTypes: ZIM_MIME_TYPES, forceNew: true, filetype: 'zim', + resourceMetadata: { + resource_id: resource.id, + version: resource.version, + collection_ref: categorySlug, + }, }) } @@ -310,130 +293,36 @@ export class ZimService implements IZimService { } } - // Mark any curated collection resources with this download URL as downloaded - const resources = await CuratedCollectionResource.query().whereIn('url', urls) - for (const resource of resources) { - resource.downloaded = true - await resource.save() - } - } + // Create InstalledResource entries for downloaded files + for (const url of urls) { + // Skip Wikipedia files (managed separately) + if (url.includes('wikipedia_en_')) continue - async listCuratedCategories(): Promise { - try { - const response = await axios.get(CATEGORIES_URL) - const data = response.data + const filename = url.split('/').pop() + if (!filename) continue - const validated = await vine.validate({ - schema: curatedCategoriesFileSchema, - data, - }); + const parsed = CollectionManifestService.parseZimFilename(filename) + if (!parsed) continue - // Dynamically determine installed tier for each category - const categoriesWithStatus = await Promise.all( - validated.categories.map(async (category) => { - const installedTierSlug = await this.getInstalledTierForCategory(category) - return { - ...category, - installedTierSlug, - } - }) - ) + const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) + const stats = await getFileStatsIfExists(filepath) - return categoriesWithStatus - } catch (error) { - logger.error(`[ZimService] Failed to fetch curated categories:`, error) - throw new Error('Failed to fetch curated categories or invalid format was received') - } - } - - /** - * Dynamically determines which tier is installed for a category by checking - * which tier's resources are all downloaded. Returns the highest tier that - * is fully installed (considering that higher tiers include lower tier resources) - */ - private async getInstalledTierForCategory(category: CuratedCategory): Promise { - const { files: diskFiles } = await this.list() - const diskFilenames = new Set(diskFiles.map((f) => f.name)) - - // Get all CuratedCollectionResources marked as downloaded - const downloadedResources = await CuratedCollectionResource.query() - .where('downloaded', true) - .select('url') - const downloadedUrls = new Set(downloadedResources.map((r) => r.url)) - - // Check each tier from highest to lowest (assuming tiers are ordered from low to high) - // We check in reverse to find the highest fully-installed tier - const reversedTiers = [...category.tiers].reverse() - - for (const tier of reversedTiers) { - const allResourcesInstalled = tier.resources.every((resource) => { - // Check if resource is marked as downloaded in database - if (downloadedUrls.has(resource.url)) { - return true - } - - // Fallback: check if file exists on disk (for resources not tracked in CuratedCollectionResource) - const filename = resource.url.split('/').pop() - if (filename && diskFilenames.has(filename)) { - return true - } - - return false - }) - - if (allResourcesInstalled && tier.resources.length > 0) { - return tier.slug - } - } - - return undefined - } - - async listCuratedCollections(): Promise { - const collections = await CuratedCollection.query().where('type', 'zim').preload('resources') - return collections.map((collection) => ({ - ...(collection.serialize() as CuratedCollection), - all_downloaded: collection.resources.every((res) => res.downloaded), - })) - } - - async fetchLatestCollections(): Promise { - try { - const response = await axios.get(COLLECTIONS_URL) - - const validated = await vine.validate({ - schema: curatedCollectionsFileSchema, - data: response.data, - }) - - for (const collection of validated.collections) { - const { resources, ...restCollection } = collection; // we'll handle resources separately - - // Upsert the collection itself - await CuratedCollection.updateOrCreate( - { slug: restCollection.slug }, + try { + const { DateTime } = await import('luxon') + await InstalledResource.updateOrCreate( + { resource_id: parsed.resource_id, resource_type: 'zim' }, { - ...restCollection, - type: 'zim', + version: parsed.version, + url: url, + file_path: filepath, + file_size_bytes: stats ? Number(stats.size) : null, + installed_at: DateTime.now(), } ) - logger.info(`[ZimService] Upserted curated collection: ${restCollection.slug}`) - - // Upsert collection's resources - const resourcesResult = await CuratedCollectionResource.updateOrCreateMany('url', resources.map((res) => ({ - ...res, - curated_collection_slug: restCollection.slug, // add the foreign key - }))) - - logger.info( - `[ZimService] Upserted ${resourcesResult.length} resources for collection: ${restCollection.slug}` - ) + logger.info(`[ZimService] Created InstalledResource entry for: ${parsed.resource_id}`) + } catch (error) { + logger.error(`[ZimService] Failed to create InstalledResource for ${filename}:`, error) } - - return true - } catch (error) { - logger.error(`[ZimService] Failed to download latest Kiwix collections:`, error) - return false } } @@ -452,9 +341,15 @@ export class ZimService implements IZimService { await deleteFileIfExists(fullPath) - // Clean up metadata - await ZimFileMetadata.query().where('filename', fileName).delete() - logger.info(`[ZimService] Deleted metadata for ZIM file: ${fileName}`) + // Clean up InstalledResource entry + const parsed = CollectionManifestService.parseZimFilename(fileName) + if (parsed) { + await InstalledResource.query() + .where('resource_id', parsed.resource_id) + .where('resource_type', 'zim') + .delete() + logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`) + } } // Wikipedia selector methods diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index 564a090..183e40c 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -56,6 +56,13 @@ export const downloadCollectionValidator = vine.compile( }) ) +export const downloadCategoryTierValidator = vine.compile( + vine.object({ + categorySlug: vine.string().trim().minLength(1), + tierSlug: vine.string().trim().minLength(1), + }) +) + export const selectWikipediaValidator = vine.compile( vine.object({ optionId: vine.string().trim().minLength(1), diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index fad987c..af0d2b9 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -1,30 +1,20 @@ import vine from '@vinejs/vine' -export const curatedCollectionResourceValidator = vine.object({ +// ---- Versioned resource validators (with id + version) ---- + +export const specResourceValidator = vine.object({ + id: vine.string(), + version: vine.string(), title: vine.string(), description: vine.string(), url: vine.string().url(), size_mb: vine.number().min(0).optional(), }) -export const curatedCollectionValidator = vine.object({ - slug: vine.string(), - name: vine.string(), - description: vine.string(), - icon: vine.string(), - language: vine.string().minLength(2).maxLength(5), - resources: vine.array(curatedCollectionResourceValidator).minLength(1), -}) +// ---- ZIM Categories spec (versioned) ---- -export const curatedCollectionsFileSchema = vine.object({ - collections: vine.array(curatedCollectionValidator).minLength(1), -}) - -/** - * For validating the categories file, which has a different structure than the collections file - * since it includes tiers within each category. - */ -export const curatedCategoriesFileSchema = vine.object({ +export const zimCategoriesSpecSchema = vine.object({ + spec_version: vine.string(), categories: vine.array( vine.object({ name: vine.string(), @@ -39,16 +29,47 @@ export const curatedCategoriesFileSchema = vine.object({ description: vine.string(), recommended: vine.boolean().optional(), includesTier: vine.string().optional(), - resources: vine.array(curatedCollectionResourceValidator), + resources: vine.array(specResourceValidator), }) ), }) ), }) -/** - * For validating the Wikipedia options file - */ +// ---- Maps spec (versioned) ---- + +export const mapsSpecSchema = vine.object({ + spec_version: vine.string(), + collections: vine.array( + vine.object({ + slug: vine.string(), + name: vine.string(), + description: vine.string(), + icon: vine.string(), + language: vine.string().minLength(2).maxLength(5), + resources: vine.array(specResourceValidator).minLength(1), + }) + ).minLength(1), +}) + +// ---- Wikipedia spec (versioned) ---- + +export const wikipediaSpecSchema = vine.object({ + spec_version: vine.string(), + options: vine.array( + vine.object({ + id: vine.string(), + name: vine.string(), + description: vine.string(), + size_mb: vine.number().min(0), + url: vine.string().url().nullable(), + version: vine.string().nullable(), + }) + ).minLength(1), +}) + +// ---- Wikipedia validators (used by ZimService) ---- + export const wikipediaOptionSchema = vine.object({ id: vine.string(), name: vine.string(), diff --git a/admin/bin/server.ts b/admin/bin/server.ts index fe0fefb..79fc533 100644 --- a/admin/bin/server.ts +++ b/admin/bin/server.ts @@ -36,6 +36,15 @@ new Ignitor(APP_ROOT, { importer: IMPORTER }) }) app.listen('SIGTERM', () => app.terminate()) app.listenIf(app.managedByPm2, 'SIGINT', () => app.terminate()) + app.ready(async () => { + try { + const collectionManifestService = new (await import('#services/collection_manifest_service')).CollectionManifestService() + await collectionManifestService.reconcileFromFilesystem() + } catch (error) { + // Catch and log any errors during reconciliation to prevent the server from crashing + console.error('Error during collection manifest reconciliation:', error) + } + }) }) .httpServer() .start() diff --git a/admin/database/migrations/1770849108030_create_create_collection_manifests_table.ts b/admin/database/migrations/1770849108030_create_create_collection_manifests_table.ts new file mode 100644 index 0000000..b8f20ba --- /dev/null +++ b/admin/database/migrations/1770849108030_create_create_collection_manifests_table.ts @@ -0,0 +1,18 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'collection_manifests' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.string('type').primary() // 'zim_categories' | 'maps' | 'wikipedia' + table.string('spec_version').notNullable() + table.json('spec_data').notNullable() + table.timestamp('fetched_at').notNullable() + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/migrations/1770849119787_create_create_installed_resources_table.ts b/admin/database/migrations/1770849119787_create_create_installed_resources_table.ts new file mode 100644 index 0000000..60784eb --- /dev/null +++ b/admin/database/migrations/1770849119787_create_create_installed_resources_table.ts @@ -0,0 +1,25 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'installed_resources' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + table.string('resource_id').notNullable() + table.enum('resource_type', ['zim', 'map']).notNullable() + table.string('collection_ref').nullable() + table.string('version').notNullable() + table.string('url').notNullable() + table.string('file_path').notNullable() + table.bigInteger('file_size_bytes').nullable() + table.timestamp('installed_at').notNullable() + + table.unique(['resource_id', 'resource_type']) + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/migrations/1770850092871_create_drop_legacy_curated_tables_table.ts b/admin/database/migrations/1770850092871_create_drop_legacy_curated_tables_table.ts new file mode 100644 index 0000000..f9781a5 --- /dev/null +++ b/admin/database/migrations/1770850092871_create_drop_legacy_curated_tables_table.ts @@ -0,0 +1,13 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + async up() { + this.schema.dropTableIfExists('curated_collection_resources') + this.schema.dropTableIfExists('curated_collections') + this.schema.dropTableIfExists('zim_file_metadata') + } + + async down() { + // These tables are legacy and intentionally not recreated + } +} diff --git a/admin/inertia/components/CategoryCard.tsx b/admin/inertia/components/CategoryCard.tsx index ebfb0fe..ae60b4c 100644 --- a/admin/inertia/components/CategoryCard.tsx +++ b/admin/inertia/components/CategoryCard.tsx @@ -1,18 +1,18 @@ import { formatBytes } from '~/lib/util' import DynamicIcon, { DynamicIconName } from './DynamicIcon' -import { CuratedCategory, CategoryTier } from '../../types/downloads' +import type { CategoryWithStatus, SpecTier } from '../../types/collections' import classNames from 'classnames' import { IconChevronRight, IconCircleCheck } from '@tabler/icons-react' export interface CategoryCardProps { - category: CuratedCategory - selectedTier?: CategoryTier | null - onClick?: (category: CuratedCategory) => void + category: CategoryWithStatus + selectedTier?: SpecTier | null + onClick?: (category: CategoryWithStatus) => void } const CategoryCard: React.FC = ({ category, selectedTier, onClick }) => { // Calculate total size range across all tiers - const getTierTotalSize = (tier: CategoryTier, allTiers: CategoryTier[]): number => { + const getTierTotalSize = (tier: SpecTier, allTiers: SpecTier[]): number => { let total = tier.resources.reduce((acc, r) => acc + r.size_mb * 1024 * 1024, 0) // Add included tier sizes recursively diff --git a/admin/inertia/components/CuratedCollectionCard.tsx b/admin/inertia/components/CuratedCollectionCard.tsx index 22703f6..dc1c587 100644 --- a/admin/inertia/components/CuratedCollectionCard.tsx +++ b/admin/inertia/components/CuratedCollectionCard.tsx @@ -1,12 +1,12 @@ import { formatBytes } from '~/lib/util' import DynamicIcon, { DynamicIconName } from './DynamicIcon' -import { CuratedCollectionWithStatus } from '../../types/downloads' +import type { CollectionWithStatus } from '../../types/collections' import classNames from 'classnames' import { IconCircleCheck } from '@tabler/icons-react' export interface CuratedCollectionCardProps { - collection: CuratedCollectionWithStatus - onClick?: (collection: CuratedCollectionWithStatus) => void; + collection: CollectionWithStatus + onClick?: (collection: CollectionWithStatus) => void; size?: 'small' | 'large' } @@ -19,11 +19,11 @@ const CuratedCollectionCard: React.FC = ({ collectio
{ - if (collection.all_downloaded) { + if (collection.all_installed) { return } if (onClick) { @@ -37,7 +37,7 @@ const CuratedCollectionCard: React.FC = ({ collectio

{collection.name}

- {collection.all_downloaded && ( + {collection.all_installed && (
void - category: CuratedCategory | null + category: CategoryWithStatus | null selectedTierSlug?: string | null - onSelectTier: (category: CuratedCategory, tier: CategoryTier) => void + onSelectTier: (category: CategoryWithStatus, tier: SpecTier) => void } const TierSelectionModal: React.FC = ({ @@ -34,24 +35,15 @@ const TierSelectionModal: React.FC = ({ if (!category) return null // Get all resources for a tier (including inherited resources) - const getAllResourcesForTier = (tier: CategoryTier): CategoryResource[] => { - const resources = [...tier.resources] - - if (tier.includesTier) { - const includedTier = category.tiers.find(t => t.slug === tier.includesTier) - if (includedTier) { - resources.unshift(...getAllResourcesForTier(includedTier)) - } - } - - return resources + const getAllResourcesForTier = (tier: SpecTier): SpecResource[] => { + return resolveTierResources(tier, category.tiers) } - const getTierTotalSize = (tier: CategoryTier): number => { + const getTierTotalSize = (tier: SpecTier): number => { return getAllResourcesForTier(tier).reduce((acc, r) => acc + r.size_mb * 1024 * 1024, 0) } - const handleTierClick = (tier: CategoryTier) => { + const handleTierClick = (tier: SpecTier) => { // Toggle selection: if clicking the same tier, deselect it if (localSelectedSlug === tier.slug) { setLocalSelectedSlug(null) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index fd67b57..e33b1bf 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -3,12 +3,8 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' -import { - CuratedCategory, - CuratedCollectionWithStatus, - DownloadJobWithProgress, - WikipediaState, -} from '../../types/downloads' +import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' +import type { CategoryWithStatus, CollectionWithStatus, CollectionUpdateCheckResult } from '../../types/collections' import { catchInternal } from './util' import { NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' import { ChatResponse, ModelResponse } from 'ollama' @@ -78,13 +74,14 @@ class API { })() } - async downloadZimCollection(slug: string): Promise<{ + async downloadCategoryTier(categorySlug: string, tierSlug: string): Promise<{ message: string - slug: string + categorySlug: string + tierSlug: string resources: string[] | null }> { return catchInternal(async () => { - const response = await this.client.post('/zim/download-collection', { slug }) + const response = await this.client.post('/zim/download-category-tier', { categorySlug, tierSlug }) return response.data })() } @@ -130,9 +127,18 @@ class API { })() } - async fetchLatestZimCollections(): Promise<{ success: boolean } | undefined> { + async checkForCollectionUpdates() { return catchInternal(async () => { - const response = await this.client.post<{ success: boolean }>('/zim/fetch-latest-collections') + const response = await this.client.post('/collection-updates/check') + return response.data + })() + } + + async refreshManifests(): Promise<{ success: boolean; changed: Record } | undefined> { + return catchInternal(async () => { + const response = await this.client.post<{ success: boolean; changed: Record }>( + '/manifests/refresh' + ) return response.data })() } @@ -189,14 +195,14 @@ class API { async getBenchmarkResults() { return catchInternal(async () => { - const response = await this.client.get<{ results: BenchmarkResult[], total: number}>('/benchmark/results') + const response = await this.client.get<{ results: BenchmarkResult[], total: number }>('/benchmark/results') return response.data })() } async getLatestBenchmarkResult() { return catchInternal(async () => { - const response = await this.client.get<{ result: BenchmarkResult | null}>('/benchmark/results/latest') + const response = await this.client.get<{ result: BenchmarkResult | null }>('/benchmark/results/latest') return response.data })() } @@ -341,25 +347,16 @@ class API { async listCuratedMapCollections() { return catchInternal(async () => { - const response = await this.client.get( + const response = await this.client.get( '/maps/curated-collections' ) return response.data })() } - async listCuratedZimCollections() { - return catchInternal(async () => { - const response = await this.client.get( - '/zim/curated-collections' - ) - return response.data - })() - } - async listCuratedCategories() { return catchInternal(async () => { - const response = await this.client.get('/easy-setup/curated-categories') + const response = await this.client.get('/easy-setup/curated-categories') return response.data })() } diff --git a/admin/inertia/lib/collections.ts b/admin/inertia/lib/collections.ts new file mode 100644 index 0000000..6529346 --- /dev/null +++ b/admin/inertia/lib/collections.ts @@ -0,0 +1,31 @@ +import type { SpecResource, SpecTier } from '../../types/collections' + +/** + * Resolve all resources for a tier, including inherited resources from includesTier chain. + * Shared between frontend components (TierSelectionModal, CategoryCard, EasySetup). + */ +export function resolveTierResources(tier: SpecTier, allTiers: SpecTier[]): SpecResource[] { + const visited = new Set() + return resolveTierResourcesInner(tier, allTiers, visited) +} + +function resolveTierResourcesInner( + tier: SpecTier, + allTiers: SpecTier[], + visited: Set +): SpecResource[] { + if (visited.has(tier.slug)) return [] // cycle detection + visited.add(tier.slug) + + const resources: SpecResource[] = [] + + if (tier.includesTier) { + const included = allTiers.find((t) => t.slug === tier.includesTier) + if (included) { + resources.push(...resolveTierResourcesInner(included, allTiers, visited)) + } + } + + resources.push(...tier.resources) + return resources +} diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index 589c089..c929ffe 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -17,7 +17,8 @@ import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' import { useSystemInfo } from '~/hooks/useSystemInfo' import classNames from 'classnames' -import { CuratedCategory, CategoryTier, CategoryResource } from '../../../types/downloads' +import type { CategoryWithStatus, SpecTier, SpecResource } from '../../../types/collections' +import { resolveTierResources } from '~/lib/collections' import { SERVICE_NAMES } from '../../../constants/service_names' // Capability definitions - maps user-friendly categories to services @@ -105,38 +106,21 @@ const ADDITIONAL_TOOLS: Capability[] = [ type WizardStep = 1 | 2 | 3 | 4 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections' -const CURATED_ZIM_COLLECTIONS_KEY = 'curated-zim-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' const WIKIPEDIA_STATE_KEY = 'wikipedia-state' -// Helper to get all resources for a tier (including inherited resources) -const getAllResourcesForTier = ( - tier: CategoryTier, - allTiers: CategoryTier[] -): CategoryResource[] => { - const resources = [...tier.resources] - if (tier.includesTier) { - const includedTier = allTiers.find((t) => t.slug === tier.includesTier) - if (includedTier) { - resources.unshift(...getAllResourcesForTier(includedTier, allTiers)) - } - } - return resources -} - export default function EasySetupWizard(props: { system: { services: ServiceSlim[] } }) { const [currentStep, setCurrentStep] = useState(1) const [selectedServices, setSelectedServices] = useState([]) const [selectedMapCollections, setSelectedMapCollections] = useState([]) - const [selectedZimCollections, setSelectedZimCollections] = useState([]) const [selectedAiModels, setSelectedAiModels] = useState([]) const [isProcessing, setIsProcessing] = useState(false) const [showAdditionalTools, setShowAdditionalTools] = useState(false) // Category/tier selection state - const [selectedTiers, setSelectedTiers] = useState>(new Map()) + const [selectedTiers, setSelectedTiers] = useState>(new Map()) const [tierModalOpen, setTierModalOpen] = useState(false) - const [activeCategory, setActiveCategory] = useState(null) + const [activeCategory, setActiveCategory] = useState(null) // Wikipedia selection state const [selectedWikipedia, setSelectedWikipedia] = useState(null) @@ -149,7 +133,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim const anySelectionMade = selectedServices.length > 0 || selectedMapCollections.length > 0 || - selectedZimCollections.length > 0 || selectedTiers.size > 0 || selectedAiModels.length > 0 || (selectedWikipedia !== null && selectedWikipedia !== 'none') @@ -160,12 +143,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim refetchOnWindowFocus: false, }) - const { data: zimCollections, isLoading: isLoadingZims } = useQuery({ - queryKey: [CURATED_ZIM_COLLECTIONS_KEY], - queryFn: () => api.listCuratedZimCollections(), - refetchOnWindowFocus: false, - }) - // Fetch curated categories with tiers const { data: categories, isLoading: isLoadingCategories } = useQuery({ queryKey: [CURATED_CATEGORIES_KEY], @@ -202,12 +179,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim ) } - const toggleZimCollection = (slug: string) => { - setSelectedZimCollections((prev) => - prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug] - ) - } - const toggleAiModel = (modelName: string) => { setSelectedAiModels((prev) => prev.includes(modelName) ? prev.filter((m) => m !== modelName) : [...prev, modelName] @@ -215,13 +186,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim } // Category/tier handlers - const handleCategoryClick = (category: CuratedCategory) => { + const handleCategoryClick = (category: CategoryWithStatus) => { if (!isOnline) return setActiveCategory(category) setTierModalOpen(true) } - const handleTierSelect = (category: CuratedCategory, tier: CategoryTier) => { + const handleTierSelect = (category: CategoryWithStatus, tier: SpecTier) => { setSelectedTiers((prev) => { const newMap = new Map(prev) // If same tier is selected, deselect it @@ -239,14 +210,14 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim setActiveCategory(null) } - // Get all resources from selected tiers for downloading - const getSelectedTierResources = (): CategoryResource[] => { + // Get all resources from selected tiers for storage projection + const getSelectedTierResources = (): SpecResource[] => { if (!categories) return [] - const resources: CategoryResource[] = [] + const resources: SpecResource[] = [] selectedTiers.forEach((tier, categorySlug) => { const category = categories.find((c) => c.slug === categorySlug) if (category) { - resources.push(...getAllResourcesForTier(tier, category.tiers)) + resources.push(...resolveTierResources(tier, category.tiers)) } }) return resources @@ -270,16 +241,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim }) } - // Add ZIM collections - if (zimCollections) { - selectedZimCollections.forEach((slug) => { - const collection = zimCollections.find((c) => c.slug === slug) - if (collection) { - totalBytes += collection.resources.reduce((sum, r) => sum + r.size_mb * 1024 * 1024, 0) - } - }) - } - // Add AI models if (recommendedModels) { selectedAiModels.forEach((modelName) => { @@ -315,12 +276,10 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim }, [ selectedTiers, selectedMapCollections, - selectedZimCollections, selectedAiModels, selectedWikipedia, categories, mapCollections, - zimCollections, recommendedModels, wikipediaState, ]) @@ -392,12 +351,15 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim await Promise.all(installPromises) - // Download collections, individual tier resources, and AI models - const tierResources = getSelectedTierResources() + // Download collections, category tiers, and AI models + const categoryTierPromises: Promise[] = [] + selectedTiers.forEach((tier, categorySlug) => { + categoryTierPromises.push(api.downloadCategoryTier(categorySlug, tier.slug)) + }) + const downloadPromises = [ ...selectedMapCollections.map((slug) => api.downloadMapCollection(slug)), - ...selectedZimCollections.map((slug) => api.downloadZimCollection(slug)), - ...tierResources.map((resource) => api.downloadRemoteZimFile(resource.url)), + ...categoryTierPromises, ...selectedAiModels.map((modelName) => api.downloadModel(modelName)), ] @@ -425,25 +387,11 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim } } - const fetchLatestMapCollections = useMutation({ - mutationFn: () => api.fetchLatestMapCollections(), + const refreshManifests = useMutation({ + mutationFn: () => api.refreshManifests(), onSuccess: () => { - addNotification({ - message: 'Successfully fetched the latest map collections.', - type: 'success', - }) queryClient.invalidateQueries({ queryKey: [CURATED_MAP_COLLECTIONS_KEY] }) - }, - }) - - const fetchLatestZIMCollections = useMutation({ - mutationFn: () => api.fetchLatestZimCollections(), - onSuccess: () => { - addNotification({ - message: 'Successfully fetched the latest ZIM collections.', - type: 'success', - }) - queryClient.invalidateQueries({ queryKey: [CURATED_ZIM_COLLECTIONS_KEY] }) + queryClient.invalidateQueries({ queryKey: [CURATED_CATEGORIES_KEY] }) }, }) @@ -452,18 +400,12 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim window.scrollTo({ top: 0, behavior: 'smooth' }) }, [currentStep]) - // Auto-fetch latest collections if the list is empty + // Refresh manifests on mount to ensure we have latest data useEffect(() => { - if (mapCollections && mapCollections.length === 0 && !fetchLatestMapCollections.isPending) { - fetchLatestMapCollections.mutate() + if (!refreshManifests.isPending) { + refreshManifests.mutate() } - }, [mapCollections, fetchLatestMapCollections]) - - useEffect(() => { - if (zimCollections && zimCollections.length === 0 && !fetchLatestZIMCollections.isPending) { - fetchLatestZIMCollections.mutate() - } - }, [zimCollections, fetchLatestZIMCollections]) + }, []) // eslint-disable-line react-hooks/exhaustive-deps // Set Easy Setup as visited when user lands on this page useEffect(() => { @@ -789,13 +731,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
- isOnline && !collection.all_downloaded && toggleMapCollection(collection.slug) + isOnline && !collection.all_installed && toggleMapCollection(collection.slug) } className={classNames( 'relative', selectedMapCollections.includes(collection.slug) && 'ring-4 ring-desert-green rounded-lg', - collection.all_downloaded && 'opacity-75', + collection.all_installed && 'opacity-75', !isOnline && 'opacity-50 cursor-not-allowed' )} > @@ -996,49 +938,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim ) : null} - {/* Legacy flat collections - show if available and no categories */} - {(!categories || categories.length === 0) && ( - <> - {isLoadingZims ? ( -
- -
- ) : zimCollections && zimCollections.length > 0 ? ( -
- {zimCollections.map((collection) => ( -
- isOnline && - !collection.all_downloaded && - toggleZimCollection(collection.slug) - } - className={classNames( - 'relative', - selectedZimCollections.includes(collection.slug) && - 'ring-4 ring-desert-green rounded-lg', - collection.all_downloaded && 'opacity-75', - !isOnline && 'opacity-50 cursor-not-allowed' - )} - > - - {selectedZimCollections.includes(collection.slug) && ( -
- -
- )} -
- ))} -
- ) : ( -
-

- No content collections available at this time. -

-
- )} - - )} )} @@ -1059,7 +958,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim const hasSelections = selectedServices.length > 0 || selectedMapCollections.length > 0 || - selectedZimCollections.length > 0 || selectedTiers.size > 0 || selectedAiModels.length > 0 || (selectedWikipedia !== null && selectedWikipedia !== 'none') @@ -1122,25 +1020,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
)} - {selectedZimCollections.length > 0 && ( -
-

- ZIM Collections to Download ({selectedZimCollections.length}) -

-
    - {selectedZimCollections.map((slug) => { - const collection = zimCollections?.find((c) => c.slug === slug) - return ( -
  • - - {collection?.name || slug} -
    • - ) - })} -
-
- )} - {selectedTiers.size > 0 && (

@@ -1149,7 +1028,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim {Array.from(selectedTiers.entries()).map(([categorySlug, tier]) => { const category = categories?.find((c) => c.slug === categorySlug) if (!category) return null - const resources = getAllResourcesForTier(tier, category.tiers) + const resources = resolveTierResources(tier, category.tiers) return (
@@ -1283,8 +1162,8 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim return `${count} ${count === 1 ? 'capability' : 'capabilities'}` })()} , {selectedMapCollections.length} map region - {selectedMapCollections.length !== 1 && 's'}, {selectedZimCollections.length}{' '} - content pack{selectedZimCollections.length !== 1 && 's'},{' '} + {selectedMapCollections.length !== 1 && 's'}, {selectedTiers.size}{' '} + content categor{selectedTiers.size !== 1 ? 'ies' : 'y'},{' '} {selectedAiModels.length} AI model{selectedAiModels.length !== 1 && 's'} selected

diff --git a/admin/inertia/pages/settings/maps.tsx b/admin/inertia/pages/settings/maps.tsx index 4fe1695..a0c1a95 100644 --- a/admin/inertia/pages/settings/maps.tsx +++ b/admin/inertia/pages/settings/maps.tsx @@ -13,7 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import useDownloads from '~/hooks/useDownloads' import StyledSectionHeader from '~/components/StyledSectionHeader' import CuratedCollectionCard from '~/components/CuratedCollectionCard' -import { CuratedCollectionWithStatus } from '../../../types/downloads' +import type { CollectionWithStatus } from '../../../types/collections' import ActiveDownloads from '~/components/ActiveDownloads' import Alert from '~/components/Alert' @@ -65,7 +65,7 @@ export default function MapsManager(props: { } } - async function downloadCollection(record: CuratedCollectionWithStatus) { + async function downloadCollection(record: CollectionWithStatus) { try { await api.downloadMapCollection(record.slug) invalidateDownloads() @@ -112,14 +112,14 @@ export default function MapsManager(props: { ) } - async function confirmDownload(record: CuratedCollectionWithStatus) { + async function confirmDownload(record: CollectionWithStatus) { const isCollection = 'resources' in record openModal( { if (isCollection) { - if (record.all_downloaded) { + if (record.all_installed) { addNotification({ message: `All resources in the collection "${record.name}" have already been downloaded.`, type: 'info', diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index 18cb38d..d918cf7 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -23,37 +23,18 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Input from '~/components/inputs/Input' import { IconSearch, IconBooks } from '@tabler/icons-react' import useDebounce from '~/hooks/useDebounce' -import CuratedCollectionCard from '~/components/CuratedCollectionCard' import CategoryCard from '~/components/CategoryCard' import TierSelectionModal from '~/components/TierSelectionModal' import WikipediaSelector from '~/components/WikipediaSelector' import StyledSectionHeader from '~/components/StyledSectionHeader' -import { - CuratedCollectionWithStatus, - CuratedCategory, - CategoryTier, - CategoryResource, -} from '../../../../types/downloads' +import type { CategoryWithStatus, SpecTier } from '../../../../types/collections' import useDownloads from '~/hooks/useDownloads' import ActiveDownloads from '~/components/ActiveDownloads' import { SERVICE_NAMES } from '../../../../constants/service_names' -const CURATED_COLLECTIONS_KEY = 'curated-zim-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' const WIKIPEDIA_STATE_KEY = 'wikipedia-state' -// Helper to get all resources for a tier (including inherited resources) -const getAllResourcesForTier = (tier: CategoryTier, allTiers: CategoryTier[]): CategoryResource[] => { - const resources = [...tier.resources] - if (tier.includesTier) { - const includedTier = allTiers.find((t) => t.slug === tier.includesTier) - if (includedTier) { - resources.unshift(...getAllResourcesForTier(includedTier, allTiers)) - } - } - return resources -} - export default function ZimRemoteExplorer() { const queryClient = useQueryClient() const tableParentRef = useRef(null) @@ -69,7 +50,7 @@ export default function ZimRemoteExplorer() { // Category/tier selection state const [tierModalOpen, setTierModalOpen] = useState(false) - const [activeCategory, setActiveCategory] = useState(null) + const [activeCategory, setActiveCategory] = useState(null) // Wikipedia selection state const [selectedWikipedia, setSelectedWikipedia] = useState(null) @@ -79,12 +60,6 @@ export default function ZimRemoteExplorer() { setQuery(val) }, 400) - const { data: curatedCollections } = useQuery({ - queryKey: [CURATED_COLLECTIONS_KEY], - queryFn: () => api.listCuratedZimCollections(), - refetchOnWindowFocus: false, - }) - // Fetch curated categories with tiers const { data: categories } = useQuery({ queryKey: [CURATED_CATEGORIES_KEY], @@ -170,24 +145,12 @@ export default function ZimRemoteExplorer() { fetchOnBottomReached(tableParentRef.current) }, [fetchOnBottomReached]) - async function confirmDownload(record: RemoteZimFileEntry | CuratedCollectionWithStatus) { - const isCollection = 'resources' in record + async function confirmDownload(record: RemoteZimFileEntry) { openModal( { - if (isCollection) { - if (record.all_downloaded) { - addNotification({ - message: `All resources in the collection "${record.name}" have already been downloaded.`, - type: 'info', - }) - return - } - downloadCollection(record) - } else { - downloadFile(record) - } + downloadFile(record) closeAllModals() }} onCancel={closeAllModals} @@ -198,7 +161,7 @@ export default function ZimRemoteExplorer() { >

Are you sure you want to download{' '} - {isCollection ? record.name : record.title}? It may take some time for it + {record.title}? It may take some time for it to be available depending on the file size and your internet connection. The Kiwix application will be restarted after the download is complete.

@@ -221,34 +184,19 @@ export default function ZimRemoteExplorer() { } } - async function downloadCollection(record: CuratedCollectionWithStatus) { - try { - await api.downloadZimCollection(record.slug) - invalidateDownloads() - } catch (error) { - console.error('Error downloading collection:', error) - } - } - // Category/tier handlers - const handleCategoryClick = (category: CuratedCategory) => { + const handleCategoryClick = (category: CategoryWithStatus) => { if (!isOnline) return setActiveCategory(category) setTierModalOpen(true) } - const handleTierSelect = async (category: CuratedCategory, tier: CategoryTier) => { - // Get all resources for this tier (including inherited ones) - const resources = getAllResourcesForTier(tier, category.tiers) - - // Download each resource + const handleTierSelect = async (category: CategoryWithStatus, tier: SpecTier) => { try { - for (const resource of resources) { - await api.downloadRemoteZimFile(resource.url) - } + await api.downloadCategoryTier(category.slug, tier.slug) addNotification({ - message: `Started downloading ${resources.length} files from "${category.name} - ${tier.name}"`, + message: `Started downloading "${category.name} - ${tier.name}"`, type: 'success', }) invalidateDownloads() @@ -309,24 +257,17 @@ export default function ZimRemoteExplorer() { } } - const fetchLatestCollections = useMutation({ - mutationFn: () => api.fetchLatestZimCollections(), + const refreshManifests = useMutation({ + mutationFn: () => api.refreshManifests(), onSuccess: () => { addNotification({ - message: 'Successfully fetched the latest ZIM collections.', + message: 'Successfully refreshed content collections.', type: 'success', }) - queryClient.invalidateQueries({ queryKey: [CURATED_COLLECTIONS_KEY] }) + queryClient.invalidateQueries({ queryKey: [CURATED_CATEGORIES_KEY] }) }, }) - // Auto-fetch latest collections if the list is empty - useEffect(() => { - if (curatedCollections && curatedCollections.length === 0 && !fetchLatestCollections.isPending) { - fetchLatestCollections.mutate() - } - }, [curatedCollections, fetchLatestCollections]) - return ( @@ -357,11 +298,11 @@ export default function ZimRemoteExplorer() { )} fetchLatestCollections.mutate()} - disabled={fetchLatestCollections.isPending} + onClick={() => refreshManifests.mutate()} + disabled={refreshManifests.isPending} icon="IconCloudDownload" > - Fetch Latest Collections + Refresh Collections {/* Wikipedia Selector */} @@ -386,7 +327,7 @@ export default function ZimRemoteExplorer() {
) : null} - {/* Tiered Category Collections - matches Easy Setup Wizard */} + {/* Tiered Category Collections */}
@@ -419,20 +360,7 @@ export default function ZimRemoteExplorer() { /> ) : ( - /* Legacy flat collections - fallback if no categories available */ -
- {curatedCollections?.map((collection) => ( - confirmDownload(collection)} - size="large" - /> - ))} - {curatedCollections && curatedCollections.length === 0 && ( -

No curated collections available.

- )} -
+

No curated content categories available.

)}
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 2482f89..cff7d20 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -17,6 +17,7 @@ import OllamaController from '#controllers/ollama_controller' import RagController from '#controllers/rag_controller' import SettingsController from '#controllers/settings_controller' import SystemController from '#controllers/system_controller' +import CollectionUpdatesController from '#controllers/collection_updates_controller' import ZimController from '#controllers/zim_controller' import router from '@adonisjs/core/services/router' import transmit from '@adonisjs/transmit/services/main' @@ -32,6 +33,8 @@ router.get('/maps', [MapsController, 'index']) router.get('/easy-setup', [EasySetupController, 'index']) router.get('/easy-setup/complete', [EasySetupController, 'complete']) router.get('/api/easy-setup/curated-categories', [EasySetupController, 'listCuratedCategories']) +router.post('/api/manifests/refresh', [EasySetupController, 'refreshManifests']) +router.post('/api/collection-updates/check', [CollectionUpdatesController, 'checkForUpdates']) router .group(() => { @@ -145,10 +148,9 @@ router .group(() => { router.get('/list', [ZimController, 'list']) router.get('/list-remote', [ZimController, 'listRemote']) - router.get('/curated-collections', [ZimController, 'listCuratedCollections']) - router.post('/fetch-latest-collections', [ZimController, 'fetchLatestCollections']) + router.get('/curated-categories', [ZimController, 'listCuratedCategories']) router.post('/download-remote', [ZimController, 'downloadRemote']) - router.post('/download-collection', [ZimController, 'downloadCollection']) + router.post('/download-category-tier', [ZimController, 'downloadCategoryTier']) router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) diff --git a/admin/types/collections.ts b/admin/types/collections.ts new file mode 100644 index 0000000..da906f5 --- /dev/null +++ b/admin/types/collections.ts @@ -0,0 +1,86 @@ +export type SpecResource = { + id: string + version: string + title: string + description: string + url: string + size_mb: number +} + +export type SpecTier = { + name: string + slug: string + description: string + recommended?: boolean + includesTier?: string + resources: SpecResource[] +} + +export type SpecCategory = { + name: string + slug: string + icon: string + description: string + language: string + tiers: SpecTier[] +} + +export type SpecCollection = { + name: string + slug: string + description: string + icon: string + language: string + resources: SpecResource[] +} + +export type ZimCategoriesSpec = { + spec_version: string + categories: SpecCategory[] +} + +export type MapsSpec = { + spec_version: string + collections: SpecCollection[] +} + +export type WikipediaOption = { + id: string + name: string + description: string + size_mb: number + url: string | null + version: string | null +} + +export type WikipediaSpec = { + spec_version: string + options: WikipediaOption[] +} + +export type ManifestType = 'zim_categories' | 'maps' | 'wikipedia' + +export type ResourceStatus = 'installed' | 'not_installed' | 'update_available' + +export type CategoryWithStatus = SpecCategory & { + installedTierSlug?: string +} + +export type CollectionWithStatus = SpecCollection & { + all_installed: boolean + installed_count: number + total_count: number +} + +export type CollectionResourceUpdateInfo = { + resource_id: string + installed_version: string + latest_version: string + latest_url: string + latest_size_mb?: number +} + +export type CollectionUpdateCheckResult = { + spec_changed: boolean + resource_updates: CollectionResourceUpdateInfo[] +} diff --git a/admin/types/curated_collections.ts b/admin/types/curated_collections.ts deleted file mode 100644 index 1c782b0..0000000 --- a/admin/types/curated_collections.ts +++ /dev/null @@ -1,2 +0,0 @@ - -export type CuratedCollectionType = 'zim' | 'map' \ No newline at end of file diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index b7b3ab3..11734f6 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -23,33 +23,16 @@ export type DoResumableDownloadProgress = { url: string } -export type CuratedCollection = { - name: string - slug: string - description: string - icon: string - language: string - resources: { - title: string - description: string - size_mb: number - url: string - }[] -} - -export type CuratedCollectionWithStatus = CuratedCollection & { - all_downloaded: boolean -} - -export type CuratedCollectionsFile = { - collections: CuratedCollection[] -} - export type RunDownloadJobParams = Omit< DoResumableDownloadParams, 'onProgress' | 'onComplete' | 'signal' > & { filetype: string + resourceMetadata?: { + resource_id: string + version: string + collection_ref: string | null + } } export type DownloadJobWithProgress = { @@ -60,37 +43,6 @@ export type DownloadJobWithProgress = { filetype: string } -// Tiered category types for curated collections UI -export type CategoryResource = { - title: string - description: string - size_mb?: number - url: string -} - -export type CategoryTier = { - name: string - slug: string - description: string - recommended?: boolean - includesTier?: string - resources: CategoryResource[] -} - -export type CuratedCategory = { - name: string - slug: string - icon: string - description: string - language: string - tiers: CategoryTier[] - installedTierSlug?: string -} - -export type CuratedCategoriesFile = { - categories: CuratedCategory[] -} - // Wikipedia selector types export type WikipediaOption = { id: string diff --git a/admin/types/files.ts b/admin/types/files.ts index 82d9a87..0d61d5c 100644 --- a/admin/types/files.ts +++ b/admin/types/files.ts @@ -29,5 +29,4 @@ export type DownloadOptions = { onComplete?: (filepath: string) => void } -export type DownloadCollectionOperation = (slug: string) => Promise export type DownloadRemoteSuccessCallback = (urls: string[], restart: boolean) => Promise \ No newline at end of file diff --git a/collections/kiwix-categories.json b/collections/kiwix-categories.json index 596acfd..4ae55c7 100644 --- a/collections/kiwix-categories.json +++ b/collections/kiwix-categories.json @@ -1,4 +1,5 @@ { + "spec_version": "2026-02-11", "categories": [ { "name": "Medicine", @@ -14,24 +15,32 @@ "recommended": true, "resources": [ { + "id": "zimgit-medicine_en", + "version": "2024-08", "title": "Medical Library", "description": "Field and emergency medicine books and guides", "url": "https://download.kiwix.org/zim/other/zimgit-medicine_en_2024-08.zim", "size_mb": 67 }, { + "id": "nhs.uk_en_medicines", + "version": "2025-12", "title": "NHS Medicines A to Z", "description": "How medicines work, dosages, side effects, and interactions", "url": "https://download.kiwix.org/zim/zimit/nhs.uk_en_medicines_2025-12.zim", "size_mb": 16 }, { + "id": "fas-military-medicine_en", + "version": "2025-06", "title": "Military Medicine", "description": "Tactical and field medicine manuals", "url": "https://download.kiwix.org/zim/zimit/fas-military-medicine_en_2025-06.zim", "size_mb": 78 }, { + "id": "wwwnc.cdc.gov_en_all", + "version": "2024-11", "title": "CDC Health Information", "description": "Disease prevention, travel health, and outbreak information", "url": "https://download.kiwix.org/zim/zimit/wwwnc.cdc.gov_en_all_2024-11.zim", @@ -46,6 +55,8 @@ "includesTier": "medicine-essential", "resources": [ { + "id": "medlineplus.gov_en_all", + "version": "2025-01", "title": "MedlinePlus", "description": "NIH's consumer health encyclopedia - diseases, conditions, drugs, supplements", "url": "https://download.kiwix.org/zim/zimit/medlineplus.gov_en_all_2025-01.zim", @@ -60,18 +71,24 @@ "includesTier": "medicine-standard", "resources": [ { + "id": "wikipedia_en_medicine_maxi", + "version": "2026-01", "title": "Wikipedia Medicine", "description": "Curated medical articles from Wikipedia with images", "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_medicine_maxi_2026-01.zim", "size_mb": 2000 }, { + "id": "libretexts.org_en_med", + "version": "2025-01", "title": "LibreTexts Medicine", "description": "Open-source medical textbooks and educational content", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_med_2025-01.zim", "size_mb": 1100 }, { + "id": "librepathology_en_all_maxi", + "version": "2025-09", "title": "LibrePathology", "description": "Pathology reference for disease identification", "url": "https://download.kiwix.org/zim/other/librepathology_en_all_maxi_2025-09.zim", @@ -95,12 +112,16 @@ "recommended": true, "resources": [ { + "id": "canadian_prepper_winterprepping_en", + "version": "2025-11", "title": "Canadian Prepper: Winter Prepping", "description": "Video guides for winter survival and cold weather emergencies", "url": "https://download.kiwix.org/zim/videos/canadian_prepper_winterprepping_en_2025-11.zim", "size_mb": 1340 }, { + "id": "canadian_prepper_bugoutroll_en", + "version": "2025-08", "title": "Canadian Prepper: Bug Out Roll", "description": "Essential gear selection for your bug-out bag", "url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutroll_en_2025-08.zim", @@ -115,12 +136,16 @@ "includesTier": "survival-essential", "resources": [ { + "id": "canadian_prepper_bugoutconcepts_en", + "version": "2025-11", "title": "Canadian Prepper: Bug Out Concepts", "description": "Strategies and planning for emergency evacuation", "url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutconcepts_en_2025-11.zim", "size_mb": 2890 }, { + "id": "urban-prepper_en_all", + "version": "2025-11", "title": "Urban Prepper", "description": "Comprehensive urban emergency preparedness video series", "url": "https://download.kiwix.org/zim/videos/urban-prepper_en_all_2025-11.zim", @@ -135,6 +160,8 @@ "includesTier": "survival-standard", "resources": [ { + "id": "canadian_prepper_preppingfood_en", + "version": "2025-09", "title": "Canadian Prepper: Prepping Food", "description": "Long-term food storage and survival meal preparation", "url": "https://download.kiwix.org/zim/videos/canadian_prepper_preppingfood_en_2025-09.zim", @@ -158,6 +185,8 @@ "recommended": true, "resources": [ { + "id": "wikibooks_en_all_nopic", + "version": "2025-10", "title": "Wikibooks", "description": "Open-content textbooks covering math, science, computing, and more", "url": "https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_nopic_2025-10.zim", @@ -172,42 +201,56 @@ "includesTier": "education-essential", "resources": [ { + "id": "ted_mul_ted-ed", + "version": "2025-07", "title": "TED-Ed", "description": "Educational video lessons on science, history, literature, and more", "url": "https://download.kiwix.org/zim/ted/ted_mul_ted-ed_2025-07.zim", "size_mb": 5610 }, { + "id": "wikiversity_en_all_maxi", + "version": "2025-11", "title": "Wikiversity", "description": "Tutorials, courses, and learning materials for all levels", "url": "https://download.kiwix.org/zim/wikiversity/wikiversity_en_all_maxi_2025-11.zim", "size_mb": 2370 }, { + "id": "libretexts.org_en_math", + "version": "2025-01", "title": "LibreTexts Mathematics", "description": "Open-source math textbooks from algebra to calculus", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_math_2025-01.zim", "size_mb": 831 }, { + "id": "libretexts.org_en_phys", + "version": "2025-01", "title": "LibreTexts Physics", "description": "Physics courses and textbooks", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_phys_2025-01.zim", "size_mb": 560 }, { + "id": "libretexts.org_en_chem", + "version": "2025-01", "title": "LibreTexts Chemistry", "description": "Chemistry courses and textbooks", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_chem_2025-01.zim", "size_mb": 2180 }, { + "id": "libretexts.org_en_bio", + "version": "2025-01", "title": "LibreTexts Biology", "description": "Biology courses and textbooks", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_bio_2025-01.zim", "size_mb": 2240 }, { + "id": "gutenberg_en_education", + "version": "2025-12", "title": "Project Gutenberg: Education", "description": "Classic educational texts and resources", "url": "https://download.kiwix.org/zim/gutenberg/gutenberg_en_education_2025-12.zim", @@ -222,36 +265,48 @@ "includesTier": "education-standard", "resources": [ { + "id": "wikibooks_en_all_maxi", + "version": "2025-10", "title": "Wikibooks (With Images)", "description": "Open textbooks with full illustrations and diagrams", "url": "https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2025-10.zim", "size_mb": 5400 }, { + "id": "ted_mul_ted-conference", + "version": "2025-08", "title": "TED Conference", "description": "Main TED conference talks on ideas worth spreading", "url": "https://download.kiwix.org/zim/ted/ted_mul_ted-conference_2025-08.zim", "size_mb": 16500 }, { + "id": "libretexts.org_en_human", + "version": "2025-01", "title": "LibreTexts Humanities", "description": "Literature, philosophy, history, and social sciences", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_human_2025-01.zim", "size_mb": 3730 }, { + "id": "libretexts.org_en_geo", + "version": "2025-01", "title": "LibreTexts Geosciences", "description": "Earth science, geology, and environmental studies", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_geo_2025-01.zim", "size_mb": 1190 }, { + "id": "libretexts.org_en_eng", + "version": "2025-01", "title": "LibreTexts Engineering", "description": "Engineering courses and technical references", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_eng_2025-01.zim", "size_mb": 678 }, { + "id": "libretexts.org_en_biz", + "version": "2025-01", "title": "LibreTexts Business", "description": "Business, economics, and management textbooks", "url": "https://download.kiwix.org/zim/libretexts/libretexts.org_en_biz_2025-01.zim", @@ -275,12 +330,16 @@ "recommended": true, "resources": [ { + "id": "woodworking.stackexchange.com_en_all", + "version": "2025-12", "title": "Woodworking Q&A", "description": "Stack Exchange Q&A for carpentry, joinery, and woodcraft", "url": "https://download.kiwix.org/zim/stack_exchange/woodworking.stackexchange.com_en_all_2025-12.zim", "size_mb": 99 }, { + "id": "mechanics.stackexchange.com_en_all", + "version": "2025-12", "title": "Motor Vehicle Maintenance Q&A", "description": "Stack Exchange Q&A for car and motorcycle repair", "url": "https://download.kiwix.org/zim/stack_exchange/mechanics.stackexchange.com_en_all_2025-12.zim", @@ -295,6 +354,8 @@ "includesTier": "diy-essential", "resources": [ { + "id": "diy.stackexchange.com_en_all", + "version": "2025-12", "title": "DIY & Home Improvement Q&A", "description": "Stack Exchange Q&A for home repairs, electrical, plumbing, and construction", "url": "https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_en_all_2025-12.zim", @@ -309,6 +370,8 @@ "includesTier": "diy-standard", "resources": [ { + "id": "ifixit_en_all", + "version": "2025-12", "title": "iFixit Repair Guides", "description": "Step-by-step repair guides for electronics, appliances, and vehicles", "url": "https://download.kiwix.org/zim/ifixit/ifixit_en_all_2025-12.zim", @@ -332,12 +395,16 @@ "recommended": true, "resources": [ { + "id": "foss.cooking_en_all", + "version": "2025-11", "title": "FOSS Cooking", "description": "Quick and easy cooking guides and recipes", "url": "https://download.kiwix.org/zim/zimit/foss.cooking_en_all_2025-11.zim", "size_mb": 24 }, { + "id": "based.cooking_en_all", + "version": "2025-11", "title": "Based.Cooking", "description": "Simple, practical recipes from the community", "url": "https://download.kiwix.org/zim/zimit/based.cooking_en_all_2025-11.zim", @@ -352,18 +419,24 @@ "includesTier": "agriculture-essential", "resources": [ { + "id": "gardening.stackexchange.com_en_all", + "version": "2025-12", "title": "Gardening Q&A", "description": "Stack Exchange Q&A for growing your own food, plant care, and landscaping", "url": "https://download.kiwix.org/zim/stack_exchange/gardening.stackexchange.com_en_all_2025-12.zim", "size_mb": 923 }, { + "id": "cooking.stackexchange.com_en_all", + "version": "2025-12", "title": "Cooking Q&A", "description": "Stack Exchange Q&A for cooking techniques, food safety, and recipes", "url": "https://download.kiwix.org/zim/stack_exchange/cooking.stackexchange.com_en_all_2025-12.zim", "size_mb": 236 }, { + "id": "zimgit-food-preparation_en", + "version": "2025-04", "title": "Food for Preppers", "description": "Recipes and techniques for food preservation and long-term storage", "url": "https://download.kiwix.org/zim/other/zimgit-food-preparation_en_2025-04.zim", @@ -378,6 +451,8 @@ "includesTier": "agriculture-standard", "resources": [ { + "id": "lrnselfreliance_en_all", + "version": "2025-12", "title": "Learning Self-Reliance: Homesteading", "description": "Beekeeping, animal husbandry, and sustainable living practices", "url": "https://download.kiwix.org/zim/videos/lrnselfreliance_en_all_2025-12.zim", @@ -401,30 +476,40 @@ "recommended": true, "resources": [ { + "id": "freecodecamp_en_all", + "version": "2025-11", "title": "freeCodeCamp", "description": "Interactive programming tutorials - JavaScript, algorithms, and data structures", "url": "https://download.kiwix.org/zim/freecodecamp/freecodecamp_en_all_2025-11.zim", "size_mb": 8 }, { + "id": "devdocs_en_python", + "version": "2026-01", "title": "Python Documentation", "description": "Complete Python language reference and tutorials", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_python_2026-01.zim", "size_mb": 4 }, { + "id": "devdocs_en_javascript", + "version": "2026-01", "title": "JavaScript Documentation", "description": "MDN JavaScript reference and guides", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_javascript_2026-01.zim", "size_mb": 3 }, { + "id": "devdocs_en_html", + "version": "2026-01", "title": "HTML Documentation", "description": "MDN HTML elements and attributes reference", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_html_2026-01.zim", "size_mb": 2 }, { + "id": "devdocs_en_css", + "version": "2026-01", "title": "CSS Documentation", "description": "MDN CSS properties and selectors reference", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_css_2026-01.zim", @@ -439,30 +524,40 @@ "includesTier": "computing-essential", "resources": [ { + "id": "arduino.stackexchange.com_en_all", + "version": "2025-12", "title": "Arduino Q&A", "description": "Stack Exchange Q&A for Arduino microcontroller projects", "url": "https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_en_all_2025-12.zim", "size_mb": 247 }, { + "id": "raspberrypi.stackexchange.com_en_all", + "version": "2025-12", "title": "Raspberry Pi Q&A", "description": "Stack Exchange Q&A for Raspberry Pi projects and troubleshooting", "url": "https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_2025-12.zim", "size_mb": 285 }, { + "id": "devdocs_en_node", + "version": "2026-01", "title": "Node.js Documentation", "description": "Node.js API reference and guides", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_node_2026-01.zim", "size_mb": 1 }, { + "id": "devdocs_en_react", + "version": "2026-01", "title": "React Documentation", "description": "React library reference and tutorials", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_react_2026-01.zim", "size_mb": 3 }, { + "id": "devdocs_en_git", + "version": "2026-01", "title": "Git Documentation", "description": "Git version control reference", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_git_2026-01.zim", @@ -477,24 +572,32 @@ "includesTier": "computing-standard", "resources": [ { + "id": "electronics.stackexchange.com_en_all", + "version": "2025-12", "title": "Electronics Q&A", "description": "Stack Exchange Q&A for circuit design, components, and electrical engineering", "url": "https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_en_all_2025-12.zim", "size_mb": 3800 }, { + "id": "robotics.stackexchange.com_en_all", + "version": "2025-12", "title": "Robotics Q&A", "description": "Stack Exchange Q&A for robotics projects and automation", "url": "https://download.kiwix.org/zim/stack_exchange/robotics.stackexchange.com_en_all_2025-12.zim", "size_mb": 233 }, { + "id": "devdocs_en_docker", + "version": "2026-01", "title": "Docker Documentation", "description": "Docker container reference and guides", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_docker_2026-01.zim", "size_mb": 2 }, { + "id": "devdocs_en_bash", + "version": "2026-01", "title": "Linux Documentation", "description": "Linux command reference and system administration", "url": "https://download.kiwix.org/zim/devdocs/devdocs_en_bash_2026-01.zim", diff --git a/collections/kiwix.json b/collections/kiwix.json deleted file mode 100644 index a899032..0000000 --- a/collections/kiwix.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "collections": [ - { - "name": "Medicine Essentials", - "slug": "medicine-essentials", - "description": "A comprehensive collection of medical textbooks and resources covering various fields of medicine, including anatomy, physiology, pharmacology, pathology, and clinical practice. Ideal for medical students, healthcare professionals, and anyone interested in medical knowledge.", - "icon": "IconStethoscope", - "language": "en", - "resources": [ - { - "title": "Medical Library", - "description": "A collection of books pertaining to field and emergency medicine", - "url": "https://download.kiwix.org/zim/other/zimgit-medicine_en_2024-08.zim", - "size_mb": 67 - }, - { - "title": "Military Medicine", - "description": "Doctrinal and Instructional manuals", - "url": "https://download.kiwix.org/zim/zimit/fas-military-medicine_en_2025-06.zim", - "size_mb": 78 - }, - { - "title": "NHS' Medicines A to Z", - "description": "How medicines work, how and when to take them", - "url": "https://download.kiwix.org/zim/zimit/nhs.uk_en_medicines_2025-09.zim", - "size_mb": 17 - } - ] - }, - { - "name": "Prepper's Library", - "slug": "preppers-library", - "description": "A curated collection of resources for preppers and survivalists, including guides on emergency preparedness, self-sufficiency, first aid, food storage, and survival skills. Perfect for individuals looking to enhance their readiness for various scenarios.", - "icon": "IconShieldCheck", - "language": "en", - "resources": [ - { - "title": "Canadian Prepper: Winter Prepping", - "description": "Winter Prepping: Prepare for winter survival and emergencies", - "size_mb": 1250, - "url": "https://download.kiwix.org/zim/videos/canadian_prepper_winterprepping_en_2025-11.zim" - }, - { - "title": "Canadian Prepper: Prepping Food", - "description": "Prepping Food: Long-term food storage and survival meals", - "size_mb": 2000, - "url": "https://download.kiwix.org/zim/videos/canadian_prepper_preppingfood_en_2025-09.zim" - }, - { - "title": "Urban Prepper", - "description": "A collection of recipes and techniques for preparing food", - "size_mb": 2000, - "url": "https://download.kiwix.org/zim/videos/urban-prepper_en_all_2025-11.zim" - }, - { - "title": "Canadian Prepper: Bug Out Concepts", - "description": "Bug Out Concepts: Strategies and tips for effective bug-out plans", - "size_mb": 2690, - "url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutconcepts_en_2025-11.zim" - }, - { - "title": "Canadian Prepper: Bug Out Roll", - "description": "Bug Out Roll: Essential gear for your ultimate bug-out bag", - "size_mb": 929, - "url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutroll_en_2025-08.zim" - } - ] - } - ] -} diff --git a/collections/maps.json b/collections/maps.json index f6f9e0f..a4279d2 100644 --- a/collections/maps.json +++ b/collections/maps.json @@ -1,4 +1,5 @@ { + "spec_version": "2026-02-11", "collections": [ { "name": "Pacific Region", @@ -8,33 +9,43 @@ "language": "en", "resources": [ { + "id": "alaska", + "version": "2025-12", "title": "Alaska", "description": "Topographic maps for the state of Alaska.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/alaska.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/alaska_2025-12.pmtiles", "size_mb": 684 }, { + "id": "california", + "version": "2025-12", "title": "California", "description": "Topographic maps for the state of California.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/california.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/california_2025-12.pmtiles", "size_mb": 1100 }, { + "id": "hawaii", + "version": "2025-12", "title": "Hawaii", "description": "Topographic maps for the state of Hawaii.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/hawaii.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/hawaii_2025-12.pmtiles", "size_mb": 28 }, { + "id": "oregon", + "version": "2025-12", "title": "Oregon", "description": "Topographic maps for the state of Oregon.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/oregon.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/oregon_2025-12.pmtiles", "size_mb": 379 }, { + "id": "washington", + "version": "2025-12", "title": "Washington", "description": "Topographic maps for the state of Washington.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/washington.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/washington_2025-12.pmtiles", "size_mb": 466 } ] @@ -47,51 +58,67 @@ "language": "en", "resources": [ { + "id": "arizona", + "version": "2025-12", "title": "Arizona", "description": "Topographic maps for the state of Arizona.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/arizona.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/arizona_2025-12.pmtiles", "size_mb": 400 }, { + "id": "colorado", + "version": "2025-12", "title": "Colorado", "description": "Topographic maps for the state of Colorado.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/colorado.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/colorado_2025-12.pmtiles", "size_mb": 450 }, { + "id": "idaho", + "version": "2025-12", "title": "Idaho", "description": "Topographic maps for the state of Idaho.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/idaho.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/idaho_2025-12.pmtiles", "size_mb": 220 }, { + "id": "montana", + "version": "2025-12", "title": "Montana", "description": "Topographic maps for the state of Montana.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/montana.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/montana_2025-12.pmtiles", "size_mb": 270 }, { + "id": "nevada", + "version": "2025-12", "title": "Nevada", "description": "Topographic maps for the state of Nevada.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/nevada.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/nevada_2025-12.pmtiles", "size_mb": 200 }, { + "id": "new_mexico", + "version": "2025-12", "title": "New Mexico", "description": "Topographic maps for the state of New Mexico.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_mexico.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_mexico_2025-12.pmtiles", "size_mb": 230 }, { + "id": "utah", + "version": "2025-12", "title": "Utah", "description": "Topographic maps for the state of Utah.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/utah.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/utah_2025-12.pmtiles", "size_mb": 240 }, { + "id": "wyoming", + "version": "2025-12", "title": "Wyoming", "description": "Topographic maps for the state of Wyoming.", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/wyoming.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/wyoming_2025-12.pmtiles", "size_mb": 210 } ] @@ -104,27 +131,35 @@ "language": "en", "resources": [ { + "id": "arkansas", + "version": "2025-12", "title": "Arkansas", "description": "Topographic maps for the state of Arkansas", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/arkansas.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/arkansas_2025-12.pmtiles", "size_mb": 400 }, { + "id": "oklahoma", + "version": "2025-12", "title": "Oklahoma", "description": "Topographic maps for the state of Oklahoma", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/oklahoma.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/oklahoma_2025-12.pmtiles", "size_mb": 400 }, { + "id": "louisiana", + "version": "2025-12", "title": "Louisiana", "description": "Topographic maps for the state of Louisiana", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/louisiana.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/louisiana_2025-12.pmtiles", "size_mb": 400 }, { + "id": "texas", + "version": "2025-12", "title": "Texas", "description": "Topographic maps for the state of Texas", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/texas.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/texas_2025-12.pmtiles", "size_mb": 400 } ] @@ -137,27 +172,35 @@ "language": "en", "resources": [ { + "id": "alabama", + "version": "2025-12", "title": "Alabama", "description": "Topographic maps for the state of Alabama", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/alabama.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/alabama_2025-12.pmtiles", "size_mb": 400 }, { + "id": "kentucky", + "version": "2025-12", "title": "Kentucky", "description": "Topographic maps for the state of Kentucky", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/kentucky.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/kentucky_2025-12.pmtiles", "size_mb": 400 }, { + "id": "mississippi", + "version": "2025-12", "title": "Mississippi", "description": "Topographic maps for the state of Mississippi", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/mississippi.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/mississippi_2025-12.pmtiles", "size_mb": 400 }, { + "id": "tennessee", + "version": "2025-12", "title": "Tennessee", "description": "Topographic maps for the state of Tennessee", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/tennessee.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/tennessee_2025-12.pmtiles", "size_mb": 400 } ] @@ -170,57 +213,75 @@ "language": "en", "resources": [ { + "id": "delaware", + "version": "2025-12", "title": "Delaware", "description": "Topographic maps for the state of Delaware", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/delaware.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/delaware_2025-12.pmtiles", "size_mb": 400 }, { + "id": "district_of_columbia", + "version": "2025-12", "title": "District_Of_Columbia", "description": "Topographic maps for the state of District_Of_Columbia", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/district_of_columbia.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/district_of_columbia_2025-12.pmtiles", "size_mb": 400 }, { + "id": "florida", + "version": "2025-12", "title": "Florida", "description": "Topographic maps for the state of Florida", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/florida.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/florida_2025-12.pmtiles", "size_mb": 400 }, { + "id": "georgia", + "version": "2025-12", "title": "Georgia", "description": "Topographic maps for the state of Georgia", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/georgia.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/georgia_2025-12.pmtiles", "size_mb": 400 }, { + "id": "maryland", + "version": "2025-12", "title": "Maryland", "description": "Topographic maps for the state of Maryland", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/maryland.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/maryland_2025-12.pmtiles", "size_mb": 400 }, { + "id": "north_carolina", + "version": "2025-12", "title": "North_Carolina", "description": "Topographic maps for the state of North_Carolina", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/north_carolina.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/north_carolina_2025-12.pmtiles", "size_mb": 400 }, { + "id": "south_carolina", + "version": "2025-12", "title": "South_Carolina", "description": "Topographic maps for the state of South_Carolina", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/south_carolina.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/south_carolina_2025-12.pmtiles", "size_mb": 400 }, { + "id": "virginia", + "version": "2025-12", "title": "Virginia", "description": "Topographic maps for the state of Virginia", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/virginia.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/virginia_2025-12.pmtiles", "size_mb": 400 }, { + "id": "west_virginia", + "version": "2025-12", "title": "West_Virginia", "description": "Topographic maps for the state of West_Virginia", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/west_virginia.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/west_virginia_2025-12.pmtiles", "size_mb": 400 } ] @@ -233,45 +294,59 @@ "language": "en", "resources": [ { + "id": "iowa", + "version": "2025-12", "title": "Iowa", "description": "Topographic maps for the state of Iowa", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/iowa.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/iowa_2025-12.pmtiles", "size_mb": 400 }, { + "id": "kansas", + "version": "2025-12", "title": "Kansas", "description": "Topographic maps for the state of Kansas", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/kansas.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/kansas_2025-12.pmtiles", "size_mb": 400 }, { + "id": "minnesota", + "version": "2025-12", "title": "Minnesota", "description": "Topographic maps for the state of Minnesota", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/minnesota.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/minnesota_2025-12.pmtiles", "size_mb": 400 }, { + "id": "missouri", + "version": "2025-12", "title": "Missouri", "description": "Topographic maps for the state of Missouri", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/missouri.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/missouri_2025-12.pmtiles", "size_mb": 400 }, { + "id": "nebraska", + "version": "2025-12", "title": "Nebraska", "description": "Topographic maps for the state of Nebraska", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/nebraska.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/nebraska_2025-12.pmtiles", "size_mb": 400 }, { + "id": "north_dakota", + "version": "2025-12", "title": "North_Dakota", "description": "Topographic maps for the state of North_Dakota", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/north_dakota.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/north_dakota_2025-12.pmtiles", "size_mb": 400 }, { + "id": "south_dakota", + "version": "2025-12", "title": "South_Dakota", "description": "Topographic maps for the state of South_Dakota", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/south_dakota.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/south_dakota_2025-12.pmtiles", "size_mb": 400 } ] @@ -284,27 +359,35 @@ "language": "en", "resources": [ { + "id": "illinois", + "version": "2025-12", "title": "Illinois", "description": "Topographic maps for the state of Illinois", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/illinois.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/illinois_2025-12.pmtiles", "size_mb": 400 }, { + "id": "indianamichigan", + "version": "2025-12", "title": "Indianamichigan", "description": "Topographic maps for the state of Indianamichigan", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/indianamichigan.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/indianamichigan_2025-12.pmtiles", "size_mb": 400 }, { + "id": "ohio", + "version": "2025-12", "title": "Ohio", "description": "Topographic maps for the state of Ohio", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/ohio.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/ohio_2025-12.pmtiles", "size_mb": 400 }, { + "id": "wisconsin", + "version": "2025-12", "title": "Wisconsin", "description": "Topographic maps for the state of Wisconsin", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/wisconsin.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/wisconsin_2025-12.pmtiles", "size_mb": 400 } ] @@ -317,21 +400,27 @@ "language": "en", "resources": [ { + "id": "new_jersey", + "version": "2025-12", "title": "New_Jersey", "description": "Topographic maps for the state of New_Jersey", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_jersey.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_jersey_2025-12.pmtiles", "size_mb": 400 }, { + "id": "new_york", + "version": "2025-12", "title": "New_York", "description": "Topographic maps for the state of New_York", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_york.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_york_2025-12.pmtiles", "size_mb": 400 }, { + "id": "pennsylvania", + "version": "2025-12", "title": "Pennsylvania", "description": "Topographic maps for the state of Pennsylvania", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/pennsylvania.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/pennsylvania_2025-12.pmtiles", "size_mb": 400 } ] @@ -344,39 +433,51 @@ "language": "en", "resources": [ { + "id": "connecticut", + "version": "2025-12", "title": "Connecticut", "description": "Topographic maps for the state of Connecticut", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/connecticut.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/connecticut_2025-12.pmtiles", "size_mb": 400 }, { + "id": "maine", + "version": "2025-12", "title": "Maine", "description": "Topographic maps for the state of Maine", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/maine.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/maine_2025-12.pmtiles", "size_mb": 400 }, { + "id": "massachusetts", + "version": "2025-12", "title": "Massachusetts", "description": "Topographic maps for the state of Massachusetts", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/massachusetts.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/massachusetts_2025-12.pmtiles", "size_mb": 400 }, { + "id": "new_hampshire", + "version": "2025-12", "title": "New_Hampshire", "description": "Topographic maps for the state of New_Hampshire", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_hampshire.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/new_hampshire_2025-12.pmtiles", "size_mb": 400 }, { + "id": "rhode_island", + "version": "2025-12", "title": "Rhode_Island", "description": "Topographic maps for the state of Rhode_Island", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/rhode_island.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/rhode_island_2025-12.pmtiles", "size_mb": 400 }, { + "id": "vermont", + "version": "2025-12", "title": "Vermont", "description": "Topographic maps for the state of Vermont", - "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/vermont.pmtiles", + "url": "https://github.com/Crosstalk-Solutions/project-nomad-maps/raw/refs/heads/master/pmtiles/vermont_2025-12.pmtiles", "size_mb": 400 } ] diff --git a/collections/wikipedia.json b/collections/wikipedia.json index 4350629..2acd6a5 100644 --- a/collections/wikipedia.json +++ b/collections/wikipedia.json @@ -1,46 +1,53 @@ { + "spec_version": "2026-02-11", "options": [ { "id": "none", "name": "No Wikipedia", "description": "Skip Wikipedia installation", "size_mb": 0, - "url": null + "url": null, + "version": null }, { "id": "top-mini", "name": "Quick Reference", "description": "Top 100,000 articles with minimal images. Great for quick lookups.", "size_mb": 313, - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_mini_2025-12.zim" + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_mini_2025-12.zim", + "version": "2025-12" }, { "id": "top-nopic", "name": "Popular Articles", "description": "Top articles without images. Good balance of content and size.", "size_mb": 2100, - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim" + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim", + "version": "2025-12" }, { "id": "all-mini", "name": "Complete Wikipedia (Compact)", "description": "All 6+ million articles in condensed format.", "size_mb": 11400, - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim" + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim", + "version": "2025-12" }, { "id": "all-nopic", "name": "Complete Wikipedia (No Images)", "description": "All articles without images. Comprehensive offline reference.", "size_mb": 25000, - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_nopic_2025-12.zim" + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_nopic_2025-12.zim", + "version": "2025-12" }, { "id": "all-maxi", "name": "Complete Wikipedia (Full)", "description": "The complete experience with all images and media.", "size_mb": 102000, - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_maxi_2024-01.zim" + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_maxi_2024-01.zim", + "version": "2024-01" } ] }