From b6e6e10328882b8eeecbfa31e865b135ed296ba0 Mon Sep 17 00:00:00 2001 From: Jake Turner Date: Mon, 19 Jan 2026 14:37:35 -0800 Subject: [PATCH] fix(CuratedCategories): improve fetching from Github --- .../app/controllers/easy_setup_controller.ts | 10 ++++++- admin/app/services/zim_service.ts | 23 ++++++++++++++-- admin/app/validators/curated_collections.ts | 26 +++++++++++++++++++ admin/inertia/hooks/useSystemInfo.ts | 6 ++--- admin/inertia/lib/api.ts | 11 +++++++- admin/inertia/pages/easy-setup/index.tsx | 26 ++++--------------- admin/start/routes.ts | 1 + admin/types/downloads.ts | 2 +- 8 files changed, 76 insertions(+), 29 deletions(-) diff --git a/admin/app/controllers/easy_setup_controller.ts b/admin/app/controllers/easy_setup_controller.ts index 4329106..b5a61b3 100644 --- a/admin/app/controllers/easy_setup_controller.ts +++ b/admin/app/controllers/easy_setup_controller.ts @@ -1,10 +1,14 @@ import { SystemService } from '#services/system_service' +import { ZimService } from '#services/zim_service' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @inject() export default class EasySetupController { - constructor(private systemService: SystemService) {} + constructor( + private systemService: SystemService, + private zimService: ZimService + ) {} async index({ inertia }: HttpContext) { const services = await this.systemService.getServices({ installedOnly: false }) @@ -18,4 +22,8 @@ export default class EasySetupController { async complete({ inertia }: HttpContext) { return inertia.render('easy-setup/complete') } + + async listCuratedCategories({}: HttpContext) { + return await this.zimService.listCuratedCategories() + } } diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 9cb1cc1..8cfb800 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -17,19 +17,21 @@ import { ZIM_STORAGE_PATH, } from '../utils/fs.js' import { join } from 'path' -import { CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js' +import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js' import vine from '@vinejs/vine' -import { curatedCollectionsFileSchema } from '#validators/curated_collections' +import { curatedCategoriesFileSchema, curatedCollectionsFileSchema } from '#validators/curated_collections' import CuratedCollection from '#models/curated_collection' import CuratedCollectionResource from '#models/curated_collection_resource' import { RunDownloadJob } from '#jobs/run_download_job' import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] +const CATEGORIES_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json' const COLLECTIONS_URL = 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json' + interface IZimService { downloadCollection: DownloadCollectionOperation downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback @@ -245,6 +247,23 @@ export class ZimService implements IZimService { } } + async listCuratedCategories(): Promise { + try { + const response = await axios.get(CATEGORIES_URL) + const data = response.data + + const validated = await vine.validate({ + schema: curatedCategoriesFileSchema, + data, + }); + + return validated.categories + } catch (error) { + logger.error(`[ZimService] Failed to fetch curated categories:`, error) + throw new Error('Failed to fetch curated categories or invalid format was received') + } + } + async listCuratedCollections(): Promise { const collections = await CuratedCollection.query().where('type', 'zim').preload('resources') return collections.map((collection) => ({ diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index f6bfe33..2780c92 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -19,3 +19,29 @@ export const curatedCollectionValidator = vine.object({ export const curatedCollectionsFileSchema = vine.object({ collections: vine.array(curatedCollectionValidator).minLength(1), }) + +/** + * For validating the categories file, which has a different structure than the collections file + * since it includes tiers within each category. + */ +export const curatedCategoriesFileSchema = vine.object({ + categories: vine.array( + vine.object({ + name: vine.string(), + slug: vine.string(), + icon: vine.string(), + description: vine.string(), + language: vine.string().minLength(2).maxLength(5), + tiers: vine.array( + vine.object({ + name: vine.string(), + slug: vine.string(), + description: vine.string(), + recommended: vine.boolean().optional(), + includesTier: vine.string().optional(), + resources: vine.array(curatedCollectionResourceValidator), + }) + ), + }) + ), +}) diff --git a/admin/inertia/hooks/useSystemInfo.ts b/admin/inertia/hooks/useSystemInfo.ts index d188f11..6bc1d7c 100644 --- a/admin/inertia/hooks/useSystemInfo.ts +++ b/admin/inertia/hooks/useSystemInfo.ts @@ -3,15 +3,15 @@ import { SystemInformationResponse } from '../../types/system' import api from '~/lib/api' export type UseSystemInfoProps = Omit< - UseQueryOptions, + UseQueryOptions, 'queryKey' | 'queryFn' > & {} export const useSystemInfo = (props: UseSystemInfoProps) => { - const queryData = useQuery({ + const queryData = useQuery({ ...props, queryKey: ['system-info'], - queryFn: () => api.getSystemInfo(), + queryFn: async () => await api.getSystemInfo(), refetchInterval: 45000, // Refetch every 45 seconds }) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index c367094..b161471 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -3,7 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { SystemInformationResponse, SystemUpdateStatus } from '../../types/system' -import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads' +import { CuratedCategory, CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads' import { catchInternal } from './util' class API { @@ -167,6 +167,15 @@ class API { })() } + async listCuratedCategories() { + return catchInternal(async () => { + const response = await this.client.get( + '/easy-setup/curated-categories' + ) + return response.data + })() + } + async listDocs() { return catchInternal(async () => { const response = await this.client.get>('/docs/list') diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index d25ea90..ab4044c 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -108,8 +108,6 @@ type WizardStep = 1 | 2 | 3 | 4 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections' const CURATED_ZIM_COLLECTIONS_KEY = 'curated-zim-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' -const CATEGORIES_URL = - 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/feature/tiered-collections/collections/kiwix-categories.json' // Helper to get all resources for a tier (including inherited resources) const getAllResourcesForTier = (tier: CategoryTier, allTiers: CategoryTier[]): CategoryResource[] => { @@ -159,24 +157,16 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim refetchOnWindowFocus: false, }) - // All services for display purposes - const allServices = props.system.services - - // Services that can still be installed (not already installed) // Fetch curated categories with tiers const { data: categories, isLoading: isLoadingCategories } = useQuery({ queryKey: [CURATED_CATEGORIES_KEY], - queryFn: async () => { - const response = await fetch(CATEGORIES_URL) - if (!response.ok) { - throw new Error('Failed to fetch categories') - } - const data = await response.json() - return data.categories as CuratedCategory[] - }, + queryFn: () => api.listCuratedCategories(), refetchOnWindowFocus: false, }) + // All services for display purposes + const allServices = props.system.services + const availableServices = props.system.services.filter( (service) => !service.installed && service.installation_status !== 'installing' ) @@ -186,12 +176,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim (service) => service.installed ) - const toggleServiceSelection = (serviceName: string) => { - setSelectedServices((prev) => - prev.includes(serviceName) ? prev.filter((s) => s !== serviceName) : [...prev, serviceName] - ) - } - const toggleMapCollection = (slug: string) => { setSelectedMapCollections((prev) => prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug] @@ -248,7 +232,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim // Add tier resources const tierResources = getSelectedTierResources() - totalBytes += tierResources.reduce((sum, r) => sum + r.size_mb * 1024 * 1024, 0) + totalBytes += tierResources.reduce((sum, r) => sum + (r.size_mb ?? 0) * 1024 * 1024, 0) // Add map collections if (mapCollections) { diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 06d05a8..6152e66 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -25,6 +25,7 @@ router.on('/about').renderInertia('about') router.get('/easy-setup', [EasySetupController, 'index']) router.get('/easy-setup/complete', [EasySetupController, 'complete']) +router.get('/api/easy-setup/curated-categories', [EasySetupController, 'listCuratedCategories']) router .group(() => { diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index 6828214..298f1a1 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -64,7 +64,7 @@ export type DownloadJobWithProgress = { export type CategoryResource = { title: string description: string - size_mb: number + size_mb?: number url: string }