fix(CuratedCategories): improve fetching from Github

admin/app/controllers/easy_setup_controller.ts

@@ -1,10 +1,14 @@
 import { SystemService } from '#services/system_service'
+import { ZimService } from '#services/zim_service'
 import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 
 @inject()
 export default class EasySetupController {
-  constructor(private systemService: SystemService) {}
+  constructor(
+    private systemService: SystemService,
+    private zimService: ZimService
+  ) {}
 
   async index({ inertia }: HttpContext) {
     const services = await this.systemService.getServices({ installedOnly: false })
@@ -18,4 +22,8 @@ export default class EasySetupController {
   async complete({ inertia }: HttpContext) {
     return inertia.render('easy-setup/complete')
   }
+
+  async listCuratedCategories({}: HttpContext) {
+    return await this.zimService.listCuratedCategories()
+  }
 }

admin/app/services/zim_service.ts

@@ -17,19 +17,21 @@ import {
   ZIM_STORAGE_PATH,
 } from '../utils/fs.js'
 import { join } from 'path'
-import { CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js'
+import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js'
 import vine from '@vinejs/vine'
-import { curatedCollectionsFileSchema } from '#validators/curated_collections'
+import { curatedCategoriesFileSchema, curatedCollectionsFileSchema } from '#validators/curated_collections'
 import CuratedCollection from '#models/curated_collection'
 import CuratedCollectionResource from '#models/curated_collection_resource'
 import { RunDownloadJob } from '#jobs/run_download_job'
 import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js'
 
 const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
+const CATEGORIES_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json'
 const COLLECTIONS_URL =
   'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json'
 
 
+
 interface IZimService {
   downloadCollection: DownloadCollectionOperation
   downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback
@@ -245,6 +247,23 @@ export class ZimService implements IZimService {
     }
   }
 
+  async listCuratedCategories(): Promise<CuratedCategory[]> {
+    try {
+      const response = await axios.get(CATEGORIES_URL)
+      const data = response.data
+
+      const validated = await vine.validate({
+        schema: curatedCategoriesFileSchema,
+        data,
+      });
+
+      return validated.categories
+    } catch (error) {
+      logger.error(`[ZimService] Failed to fetch curated categories:`, error)
+      throw new Error('Failed to fetch curated categories or invalid format was received')
+    }
+  }
+
   async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
     const collections = await CuratedCollection.query().where('type', 'zim').preload('resources')
     return collections.map((collection) => ({

admin/app/validators/curated_collections.ts

@@ -19,3 +19,29 @@ export const curatedCollectionValidator = vine.object({
 export const curatedCollectionsFileSchema = vine.object({
   collections: vine.array(curatedCollectionValidator).minLength(1),
 })
+
+/**
+ * For validating the categories file, which has a different structure than the collections file
+ * since it includes tiers within each category. 
+ */
+export const curatedCategoriesFileSchema = vine.object({
+  categories: vine.array(
+    vine.object({
+      name: vine.string(),
+      slug: vine.string(),
+      icon: vine.string(),
+      description: vine.string(),
+      language: vine.string().minLength(2).maxLength(5),
+      tiers: vine.array(
+        vine.object({
+          name: vine.string(),
+          slug: vine.string(),
+          description: vine.string(),
+          recommended: vine.boolean().optional(),
+          includesTier: vine.string().optional(),
+          resources: vine.array(curatedCollectionResourceValidator),
+        })
+      ),
+    })
+  ),
+})

admin/inertia/hooks/useSystemInfo.ts

@@ -3,15 +3,15 @@ import { SystemInformationResponse } from '../../types/system'
 import api from '~/lib/api'
 
 export type UseSystemInfoProps = Omit<
-  UseQueryOptions<SystemInformationResponse>,
+  UseQueryOptions<SystemInformationResponse | undefined>,
   'queryKey' | 'queryFn'
 > & {}
 
 export const useSystemInfo = (props: UseSystemInfoProps) => {
-  const queryData = useQuery<SystemInformationResponse>({
+  const queryData = useQuery<SystemInformationResponse | undefined>({
     ...props,
     queryKey: ['system-info'],
-    queryFn: () => api.getSystemInfo(),
+    queryFn: async () => await api.getSystemInfo(),
     refetchInterval: 45000, // Refetch every 45 seconds
   })
 

admin/inertia/lib/api.ts

@@ -3,7 +3,7 @@ import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zi
 import { ServiceSlim } from '../../types/services'
 import { FileEntry } from '../../types/files'
 import { SystemInformationResponse, SystemUpdateStatus } from '../../types/system'
-import { CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads'
+import { CuratedCategory, CuratedCollectionWithStatus, DownloadJobWithProgress } from '../../types/downloads'
 import { catchInternal } from './util'
 
 class API {
@@ -167,6 +167,15 @@ class API {
     })()
   }
 
+  async listCuratedCategories() {
+    return catchInternal(async () => {
+      const response = await this.client.get<CuratedCategory[]>(
+        '/easy-setup/curated-categories'
+      )
+      return response.data
+    })()
+  }
+
   async listDocs() {
     return catchInternal(async () => {
       const response = await this.client.get<Array<{ title: string; slug: string }>>('/docs/list')

admin/inertia/pages/easy-setup/index.tsx

@@ -108,8 +108,6 @@ type WizardStep = 1 | 2 | 3 | 4
 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections'
 const CURATED_ZIM_COLLECTIONS_KEY = 'curated-zim-collections'
 const CURATED_CATEGORIES_KEY = 'curated-categories'
-const CATEGORIES_URL =
-  'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/feature/tiered-collections/collections/kiwix-categories.json'
 
 // Helper to get all resources for a tier (including inherited resources)
 const getAllResourcesForTier = (tier: CategoryTier, allTiers: CategoryTier[]): CategoryResource[] => {
@@ -159,24 +157,16 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
     refetchOnWindowFocus: false,
   })
 
-  // All services for display purposes
-  const allServices = props.system.services
-
-  // Services that can still be installed (not already installed)
   // Fetch curated categories with tiers
   const { data: categories, isLoading: isLoadingCategories } = useQuery({
     queryKey: [CURATED_CATEGORIES_KEY],
-    queryFn: async () => {
+    queryFn: () => api.listCuratedCategories(),
-      const response = await fetch(CATEGORIES_URL)
-      if (!response.ok) {
-        throw new Error('Failed to fetch categories')
-      }
-      const data = await response.json()
-      return data.categories as CuratedCategory[]
-    },
     refetchOnWindowFocus: false,
   })
 
+  // All services for display purposes
+  const allServices = props.system.services
+
   const availableServices = props.system.services.filter(
     (service) => !service.installed && service.installation_status !== 'installing'
   )
@@ -186,12 +176,6 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
     (service) => service.installed
   )
 
-  const toggleServiceSelection = (serviceName: string) => {
-    setSelectedServices((prev) =>
-      prev.includes(serviceName) ? prev.filter((s) => s !== serviceName) : [...prev, serviceName]
-    )
-  }
-
   const toggleMapCollection = (slug: string) => {
     setSelectedMapCollections((prev) =>
       prev.includes(slug) ? prev.filter((s) => s !== slug) : [...prev, slug]
@@ -248,7 +232,7 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
 
     // Add tier resources
     const tierResources = getSelectedTierResources()
-    totalBytes += tierResources.reduce((sum, r) => sum + r.size_mb * 1024 * 1024, 0)
+    totalBytes += tierResources.reduce((sum, r) => sum + (r.size_mb ?? 0) * 1024 * 1024, 0)
 
     // Add map collections
     if (mapCollections) {

admin/start/routes.ts

@@ -25,6 +25,7 @@ router.on('/about').renderInertia('about')
 
 router.get('/easy-setup', [EasySetupController, 'index'])
 router.get('/easy-setup/complete', [EasySetupController, 'complete'])
+router.get('/api/easy-setup/curated-categories', [EasySetupController, 'listCuratedCategories'])
 
 router
   .group(() => {

admin/types/downloads.ts

@@ -64,7 +64,7 @@ export type DownloadJobWithProgress = {
 export type CategoryResource = {
   title: string
   description: string
-  size_mb: number
+  size_mb?: number
   url: string
 }