diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 72de28f..df58a1d 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -4,6 +4,7 @@ import { filenameParamValidator, remoteDownloadValidator, saveInstalledTierValidator, + selectWikipediaValidator, } from '#validators/common' import { listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' @@ -79,4 +80,15 @@ export default class ZimController { message: 'ZIM file deleted successfully', } } + + // Wikipedia selector endpoints + + async getWikipediaState({}: HttpContext) { + return this.zimService.getWikipediaState() + } + + async selectWikipedia({ request }: HttpContext) { + const payload = await request.validateUsing(selectWikipediaValidator) + return this.zimService.selectWikipedia(payload.optionId) + } } diff --git a/admin/app/models/wikipedia_selection.ts b/admin/app/models/wikipedia_selection.ts new file mode 100644 index 0000000..589113b --- /dev/null +++ b/admin/app/models/wikipedia_selection.ts @@ -0,0 +1,27 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' + +export default class WikipediaSelection extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare option_id: string + + @column() + declare url: string | null + + @column() + declare filename: string | null + + @column() + declare status: 'none' | 'downloading' | 'installed' | 'failed' + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c020c66..27f4297 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -17,12 +17,13 @@ import { ZIM_STORAGE_PATH, } from '../utils/fs.js' import { join } from 'path' -import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js' +import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile, WikipediaOption, WikipediaState } from '../../types/downloads.js' import vine from '@vinejs/vine' -import { curatedCategoriesFileSchema, curatedCollectionsFileSchema } from '#validators/curated_collections' +import { curatedCategoriesFileSchema, curatedCollectionsFileSchema, wikipediaOptionsFileSchema } from '#validators/curated_collections' import CuratedCollection from '#models/curated_collection' import CuratedCollectionResource from '#models/curated_collection_resource' import InstalledTier from '#models/installed_tier' +import WikipediaSelection from '#models/wikipedia_selection' import { RunDownloadJob } from '#jobs/run_download_job' import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js' @@ -30,6 +31,7 @@ const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'applicati const CATEGORIES_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json' const COLLECTIONS_URL = 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json' +const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/wikipedia.json' @@ -231,6 +233,13 @@ export class ZimService implements IZimService { } async downloadRemoteSuccessCallback(urls: string[], restart = true) { + // Check if any URL is a Wikipedia download and handle it + for (const url of urls) { + if (url.includes('wikipedia_en_')) { + await this.onWikipediaDownloadComplete(url, true) + } + } + // Restart KIWIX container to pick up new ZIM file if (restart) { await this.dockerService @@ -338,4 +347,206 @@ export class ZimService implements IZimService { await deleteFileIfExists(fullPath) } + + // Wikipedia selector methods + + async getWikipediaOptions(): Promise { + try { + const response = await axios.get(WIKIPEDIA_OPTIONS_URL) + const data = response.data + + const validated = await vine.validate({ + schema: wikipediaOptionsFileSchema, + data, + }) + + return validated.options + } catch (error) { + logger.error(`[ZimService] Failed to fetch Wikipedia options:`, error) + throw new Error('Failed to fetch Wikipedia options') + } + } + + async getWikipediaSelection(): Promise { + // Get the single row from wikipedia_selections (there should only ever be one) + return WikipediaSelection.query().first() + } + + async getWikipediaState(): Promise { + const options = await this.getWikipediaOptions() + const selection = await this.getWikipediaSelection() + + return { + options, + currentSelection: selection + ? { + optionId: selection.option_id, + status: selection.status, + filename: selection.filename, + url: selection.url, + } + : null, + } + } + + async selectWikipedia(optionId: string): Promise<{ success: boolean; jobId?: string; message?: string }> { + const options = await this.getWikipediaOptions() + const selectedOption = options.find((opt) => opt.id === optionId) + + if (!selectedOption) { + throw new Error(`Invalid Wikipedia option: ${optionId}`) + } + + const currentSelection = await this.getWikipediaSelection() + + // If same as currently installed, no action needed + if (currentSelection?.option_id === optionId && currentSelection.status === 'installed') { + return { success: true, message: 'Already installed' } + } + + // Handle "none" option - delete current Wikipedia file and update DB + if (optionId === 'none') { + if (currentSelection?.filename) { + try { + await this.delete(currentSelection.filename) + logger.info(`[ZimService] Deleted Wikipedia file: ${currentSelection.filename}`) + } catch (error) { + // File might already be deleted, that's OK + logger.warn(`[ZimService] Could not delete Wikipedia file (may already be gone): ${currentSelection.filename}`) + } + } + + // Update or create the selection record (always use first record) + if (currentSelection) { + currentSelection.option_id = 'none' + currentSelection.url = null + currentSelection.filename = null + currentSelection.status = 'none' + await currentSelection.save() + } else { + await WikipediaSelection.create({ + option_id: 'none', + url: null, + filename: null, + status: 'none', + }) + } + + // Restart Kiwix to reflect the change + await this.dockerService + .affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart') + .catch((error) => { + logger.error(`[ZimService] Failed to restart Kiwix after Wikipedia removal:`, error) + }) + + return { success: true, message: 'Wikipedia removed' } + } + + // Start download for the new Wikipedia option + if (!selectedOption.url) { + throw new Error('Selected Wikipedia option has no download URL') + } + + // Check if already downloading + const existingJob = await RunDownloadJob.getByUrl(selectedOption.url) + if (existingJob) { + return { success: false, message: 'Download already in progress' } + } + + // Extract filename from URL + const filename = selectedOption.url.split('/').pop() + if (!filename) { + throw new Error('Could not determine filename from URL') + } + + const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename) + + // Update or create selection record to show downloading status + let selection: WikipediaSelection + if (currentSelection) { + currentSelection.option_id = optionId + currentSelection.url = selectedOption.url + currentSelection.filename = filename + currentSelection.status = 'downloading' + await currentSelection.save() + selection = currentSelection + } else { + selection = await WikipediaSelection.create({ + option_id: optionId, + url: selectedOption.url, + filename: filename, + status: 'downloading', + }) + } + + // Dispatch download job + const result = await RunDownloadJob.dispatch({ + url: selectedOption.url, + filepath, + timeout: 30000, + allowedMimeTypes: ZIM_MIME_TYPES, + forceNew: true, + filetype: 'zim', + }) + + if (!result || !result.job) { + // Revert status on failure to dispatch + selection.option_id = currentSelection?.option_id || 'none' + selection.url = currentSelection?.url || null + selection.filename = currentSelection?.filename || null + selection.status = currentSelection?.status || 'none' + await selection.save() + throw new Error('Failed to dispatch download job') + } + + logger.info(`[ZimService] Started Wikipedia download for ${optionId}: ${filename}`) + + return { + success: true, + jobId: result.job.id, + message: 'Download started', + } + } + + async onWikipediaDownloadComplete(url: string, success: boolean): Promise { + const selection = await this.getWikipediaSelection() + + if (!selection || selection.url !== url) { + logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`) + return + } + + if (success) { + // Get the old filename before updating (if there was a previous Wikipedia installed) + const options = await this.getWikipediaOptions() + const previousOption = options.find((opt) => opt.id !== selection.option_id && opt.id !== 'none') + + // Update status to installed + selection.status = 'installed' + await selection.save() + + logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`) + + // Delete the old Wikipedia file if it exists and is different + // We need to find what was previously installed + const existingFiles = await this.list() + const wikipediaFiles = existingFiles.files.filter((f) => + f.name.startsWith('wikipedia_en_') && f.name !== selection.filename + ) + + for (const oldFile of wikipediaFiles) { + try { + await this.delete(oldFile.name) + logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile.name}`) + } catch (error) { + logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile.name}`, error) + } + } + } else { + // Download failed - keep the selection record but mark as failed + selection.status = 'failed' + await selection.save() + logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`) + } + } } diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index fb4a79d..49436dc 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -43,3 +43,9 @@ export const saveInstalledTierValidator = vine.compile( tierSlug: vine.string().trim().minLength(1), }) ) + +export const selectWikipediaValidator = vine.compile( + vine.object({ + optionId: vine.string().trim().minLength(1), + }) +) diff --git a/admin/app/validators/curated_collections.ts b/admin/app/validators/curated_collections.ts index 2780c92..fad987c 100644 --- a/admin/app/validators/curated_collections.ts +++ b/admin/app/validators/curated_collections.ts @@ -45,3 +45,18 @@ export const curatedCategoriesFileSchema = vine.object({ }) ), }) + +/** + * For validating the Wikipedia options file + */ +export const wikipediaOptionSchema = vine.object({ + id: vine.string(), + name: vine.string(), + description: vine.string(), + size_mb: vine.number().min(0), + url: vine.string().url().nullable(), +}) + +export const wikipediaOptionsFileSchema = vine.object({ + options: vine.array(wikipediaOptionSchema).minLength(1), +}) diff --git a/admin/database/migrations/1769500000001_create_wikipedia_selection_table.ts b/admin/database/migrations/1769500000001_create_wikipedia_selection_table.ts new file mode 100644 index 0000000..0e96ce4 --- /dev/null +++ b/admin/database/migrations/1769500000001_create_wikipedia_selection_table.ts @@ -0,0 +1,21 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'wikipedia_selections' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + table.string('option_id').notNullable() + table.string('url').nullable() + table.string('filename').nullable() + table.enum('status', ['none', 'downloading', 'installed', 'failed']).defaultTo('none') + table.timestamp('created_at') + table.timestamp('updated_at') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/inertia/components/WikipediaSelector.tsx b/admin/inertia/components/WikipediaSelector.tsx new file mode 100644 index 0000000..53ba92b --- /dev/null +++ b/admin/inertia/components/WikipediaSelector.tsx @@ -0,0 +1,156 @@ +import { formatBytes } from '~/lib/util' +import { WikipediaOption, WikipediaCurrentSelection } from '../../types/downloads' +import classNames from 'classnames' +import { IconCheck, IconDownload, IconWorld } from '@tabler/icons-react' +import StyledButton from './StyledButton' +import LoadingSpinner from './LoadingSpinner' + +export interface WikipediaSelectorProps { + options: WikipediaOption[] + currentSelection: WikipediaCurrentSelection | null + selectedOptionId: string | null // for wizard (pending selection) + onSelect: (optionId: string) => void + disabled?: boolean + showSubmitButton?: boolean // true for Content Explorer, false for wizard + onSubmit?: () => void + isSubmitting?: boolean +} + +const WikipediaSelector: React.FC = ({ + options, + currentSelection, + selectedOptionId, + onSelect, + disabled = false, + showSubmitButton = false, + onSubmit, + isSubmitting = false, +}) => { + // Determine which option to highlight + const highlightedOptionId = selectedOptionId ?? currentSelection?.optionId ?? null + + // Check if current selection is downloading + const isDownloading = currentSelection?.status === 'downloading' + + return ( +
+ {/* Header with Wikipedia branding */} +
+
+ +
+
+

Wikipedia

+

Select your preferred Wikipedia package

+
+
+ + {/* Downloading status message */} + {isDownloading && ( +
+ + + Downloading Wikipedia... This may take a while for larger packages. + +
+ )} + + {/* Options grid */} +
+ {options.map((option) => { + const isSelected = highlightedOptionId === option.id + const isInstalled = + currentSelection?.optionId === option.id && currentSelection?.status === 'installed' + const isCurrentDownloading = + currentSelection?.optionId === option.id && currentSelection?.status === 'downloading' + const isPending = selectedOptionId === option.id && selectedOptionId !== currentSelection?.optionId + + return ( +
!disabled && !isCurrentDownloading && onSelect(option.id)} + className={classNames( + 'relative p-4 rounded-lg border-2 transition-all', + disabled || isCurrentDownloading + ? 'opacity-60 cursor-not-allowed' + : 'cursor-pointer hover:shadow-md', + isInstalled + ? 'border-desert-green bg-desert-green/10' + : isSelected + ? 'border-lime-500 bg-lime-50' + : 'border-gray-200 bg-white hover:border-gray-300' + )} + > + {/* Status badges */} +
+ {isInstalled && ( + + + Installed + + )} + {isPending && !isInstalled && ( + + Selected + + )} + {isCurrentDownloading && ( + + + Downloading + + )} +
+ + {/* Option content */} +
+

{option.name}

+

{option.description}

+
+ {/* Radio indicator */} +
+ {isSelected && } +
+ + {option.size_mb === 0 ? 'No download' : formatBytes(option.size_mb * 1024 * 1024, 1)} + +
+
+
+ ) + })} +
+ + {/* Submit button for Content Explorer mode */} + {showSubmitButton && selectedOptionId && selectedOptionId !== currentSelection?.optionId && ( +
+ + {selectedOptionId === 'none' ? 'Remove Wikipedia' : 'Download Selected'} + +
+ )} +
+ ) +} + +export default WikipediaSelector diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index b8f867f..393e6ba 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -7,6 +7,7 @@ import { CuratedCategory, CuratedCollectionWithStatus, DownloadJobWithProgress, + WikipediaState, } from '../../types/downloads' import { catchInternal } from './util' import { NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -402,6 +403,28 @@ class API { })() } + // Wikipedia selector methods + + async getWikipediaState(): Promise { + return catchInternal(async () => { + const response = await this.client.get('/zim/wikipedia') + return response.data + })() + } + + async selectWikipedia( + optionId: string + ): Promise<{ success: boolean; jobId?: string; message?: string } | undefined> { + return catchInternal(async () => { + const response = await this.client.post<{ + success: boolean + jobId?: string + message?: string + }>('/zim/wikipedia/select', { optionId }) + return response.data + })() + } + async uploadDocument(file: File) { return catchInternal(async () => { const formData = new FormData() diff --git a/admin/inertia/pages/easy-setup/index.tsx b/admin/inertia/pages/easy-setup/index.tsx index a3d7df4..7135152 100644 --- a/admin/inertia/pages/easy-setup/index.tsx +++ b/admin/inertia/pages/easy-setup/index.tsx @@ -8,15 +8,16 @@ import { ServiceSlim } from '../../../types/services' import CuratedCollectionCard from '~/components/CuratedCollectionCard' import CategoryCard from '~/components/CategoryCard' import TierSelectionModal from '~/components/TierSelectionModal' +import WikipediaSelector from '~/components/WikipediaSelector' import LoadingSpinner from '~/components/LoadingSpinner' import Alert from '~/components/Alert' -import { IconCheck, IconChevronDown, IconChevronUp } from '@tabler/icons-react' +import { IconCheck, IconChevronDown, IconChevronUp, IconArrowRight, IconCpu, IconBooks } from '@tabler/icons-react' import StorageProjectionBar from '~/components/StorageProjectionBar' import { useNotifications } from '~/context/NotificationContext' import useInternetStatus from '~/hooks/useInternetStatus' import { useSystemInfo } from '~/hooks/useSystemInfo' import classNames from 'classnames' -import { CuratedCategory, CategoryTier, CategoryResource } from '../../../types/downloads' +import { CuratedCategory, CategoryTier, CategoryResource, WikipediaState } from '../../../types/downloads' // Capability definitions - maps user-friendly categories to services interface Capability { @@ -105,6 +106,7 @@ type WizardStep = 1 | 2 | 3 | 4 const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections' const CURATED_ZIM_COLLECTIONS_KEY = 'curated-zim-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' +const WIKIPEDIA_STATE_KEY = 'wikipedia-state' // Helper to get all resources for a tier (including inherited resources) const getAllResourcesForTier = ( @@ -135,6 +137,9 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim const [tierModalOpen, setTierModalOpen] = useState(false) const [activeCategory, setActiveCategory] = useState(null) + // Wikipedia selection state + const [selectedWikipedia, setSelectedWikipedia] = useState(null) + const { addNotification } = useNotifications() const { isOnline } = useInternetStatus() const queryClient = useQueryClient() @@ -145,7 +150,8 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim selectedMapCollections.length > 0 || selectedZimCollections.length > 0 || selectedTiers.size > 0 || - selectedAiModels.length > 0 + selectedAiModels.length > 0 || + (selectedWikipedia !== null && selectedWikipedia !== 'none') const { data: mapCollections, isLoading: isLoadingMaps } = useQuery({ queryKey: [CURATED_MAP_COLLECTIONS_KEY], @@ -172,6 +178,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim refetchOnWindowFocus: false, }) + // Fetch Wikipedia options and current state + const { data: wikipediaState, isLoading: isLoadingWikipedia } = useQuery({ + queryKey: [WIKIPEDIA_STATE_KEY], + queryFn: () => api.getWikipediaState(), + refetchOnWindowFocus: false, + }) + // All services for display purposes const allServices = props.system.services @@ -289,16 +302,26 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim }) } + // Add Wikipedia selection + if (selectedWikipedia && wikipediaState) { + const option = wikipediaState.options.find((o) => o.id === selectedWikipedia) + if (option && option.size_mb > 0) { + totalBytes += option.size_mb * 1024 * 1024 + } + } + return totalBytes }, [ selectedTiers, selectedMapCollections, selectedZimCollections, selectedAiModels, + selectedWikipedia, categories, mapCollections, zimCollections, recommendedModels, + wikipediaState, ]) // Get primary disk/filesystem info for storage projection @@ -385,6 +408,11 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim ) await Promise.all(tierSavePromises) + // Select Wikipedia option if one was chosen + if (selectedWikipedia && selectedWikipedia !== wikipediaState?.currentSelection?.optionId) { + await api.selectWikipedia(selectedWikipedia) + } + addNotification({ type: 'success', message: 'Setup wizard completed! Your selections are being processed.', @@ -799,11 +827,14 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim {/* AI Model Selection - Only show if AI capability is selected */} {isAiSelected && (
-
-

Choose AI Models

-

- Select AI models to download. We've recommended some smaller, popular models to get you started. You'll need at least one to use AI features, but you can always add more later. -

+
+
+ +
+
+

AI Models

+

Select models to download for offline AI

+
{isLoadingRecommendedModels ? ( @@ -879,9 +910,46 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
)} + {/* Wikipedia Selection - Only show if Information capability is selected */} + {isInformationSelected && ( + <> + {/* Divider between AI Models and Wikipedia */} + {isAiSelected &&
} + +
+ {isLoadingWikipedia ? ( +
+ +
+ ) : wikipediaState && wikipediaState.options.length > 0 ? ( + isOnline && setSelectedWikipedia(optionId)} + disabled={!isOnline} + /> + ) : null} +
+ + )} + {/* Curated Categories with Tiers - Only show if Information capability is selected */} {isInformationSelected && ( <> + {/* Divider between Wikipedia and Additional Content */} +
+ +
+
+ +
+
+

Additional Content

+

Curated collections for offline reference

+
+
+ {isLoadingCategories ? (
@@ -979,7 +1047,8 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim selectedMapCollections.length > 0 || selectedZimCollections.length > 0 || selectedTiers.size > 0 || - selectedAiModels.length > 0 + selectedAiModels.length > 0 || + (selectedWikipedia !== null && selectedWikipedia !== 'none') return (
@@ -1091,6 +1160,28 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
)} + {selectedWikipedia && selectedWikipedia !== 'none' && ( +
+

Wikipedia

+ {(() => { + const option = wikipediaState?.options.find((o) => o.id === selectedWikipedia) + return option ? ( +
+
+ + {option.name} +
+ + {option.size_mb > 0 + ? `${(option.size_mb / 1024).toFixed(1)} GB` + : 'No download'} + +
+ ) : null + })()} +
+ )} + {selectedAiModels.length > 0 && (

diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index 79c4c61..10d586a 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -21,23 +21,26 @@ import useInternetStatus from '~/hooks/useInternetStatus' import Alert from '~/components/Alert' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Input from '~/components/inputs/Input' -import { IconSearch } from '@tabler/icons-react' +import { IconSearch, IconBooks } from '@tabler/icons-react' import useDebounce from '~/hooks/useDebounce' import CuratedCollectionCard from '~/components/CuratedCollectionCard' import CategoryCard from '~/components/CategoryCard' import TierSelectionModal from '~/components/TierSelectionModal' +import WikipediaSelector from '~/components/WikipediaSelector' import StyledSectionHeader from '~/components/StyledSectionHeader' import { CuratedCollectionWithStatus, CuratedCategory, CategoryTier, CategoryResource, + WikipediaState, } from '../../../../types/downloads' import useDownloads from '~/hooks/useDownloads' import ActiveDownloads from '~/components/ActiveDownloads' const CURATED_COLLECTIONS_KEY = 'curated-zim-collections' const CURATED_CATEGORIES_KEY = 'curated-categories' +const WIKIPEDIA_STATE_KEY = 'wikipedia-state' // Helper to get all resources for a tier (including inherited resources) const getAllResourcesForTier = (tier: CategoryTier, allTiers: CategoryTier[]): CategoryResource[] => { @@ -68,6 +71,10 @@ export default function ZimRemoteExplorer() { const [tierModalOpen, setTierModalOpen] = useState(false) const [activeCategory, setActiveCategory] = useState(null) + // Wikipedia selection state + const [selectedWikipedia, setSelectedWikipedia] = useState(null) + const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false) + const debouncedSetQuery = debounce((val: string) => { setQuery(val) }, 400) @@ -85,6 +92,13 @@ export default function ZimRemoteExplorer() { refetchOnWindowFocus: false, }) + // Fetch Wikipedia options and state + const { data: wikipediaState, isLoading: isLoadingWikipedia } = useQuery({ + queryKey: [WIKIPEDIA_STATE_KEY], + queryFn: () => api.getWikipediaState(), + refetchOnWindowFocus: false, + }) + const { data: downloads, invalidate: invalidateDownloads } = useDownloads({ filetype: 'zim', enabled: true, @@ -253,6 +267,46 @@ export default function ZimRemoteExplorer() { setActiveCategory(null) } + // Wikipedia selection handlers + const handleWikipediaSelect = (optionId: string) => { + if (!isOnline) return + setSelectedWikipedia(optionId) + } + + const handleWikipediaSubmit = async () => { + if (!selectedWikipedia) return + + setIsSubmittingWikipedia(true) + try { + const result = await api.selectWikipedia(selectedWikipedia) + if (result?.success) { + addNotification({ + message: + selectedWikipedia === 'none' + ? 'Wikipedia removed successfully' + : 'Wikipedia download started', + type: 'success', + }) + invalidateDownloads() + queryClient.invalidateQueries({ queryKey: [WIKIPEDIA_STATE_KEY] }) + setSelectedWikipedia(null) + } else { + addNotification({ + message: result?.message || 'Failed to change Wikipedia selection', + type: 'error', + }) + } + } catch (error) { + console.error('Error selecting Wikipedia:', error) + addNotification({ + message: 'An error occurred while changing Wikipedia selection', + type: 'error', + }) + } finally { + setIsSubmittingWikipedia(false) + } + } + const fetchLatestCollections = useMutation({ mutationFn: () => api.fetchLatestZimCollections(), onSuccess: () => { @@ -308,7 +362,38 @@ export default function ZimRemoteExplorer() { Fetch Latest Collections + {/* Wikipedia Selector */} + {isLoadingWikipedia ? ( +
+
+
+
+
+ ) : wikipediaState && wikipediaState.options.length > 0 ? ( +
+ +
+ ) : null} + {/* Tiered Category Collections - matches Easy Setup Wizard */} +
+
+ +
+
+

Additional Content

+

Curated collections for offline reference

+
+
{categories && categories.length > 0 ? ( <>
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 21e0f5e..84dade9 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -145,6 +145,8 @@ router router.post('/download-remote', [ZimController, 'downloadRemote']) router.post('/download-collection', [ZimController, 'downloadCollection']) router.post('/save-installed-tier', [ZimController, 'saveInstalledTier']) + router.get('/wikipedia', [ZimController, 'getWikipediaState']) + router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) router.delete('/:filename', [ZimController, 'delete']) }) .prefix('/api/zim') diff --git a/admin/types/downloads.ts b/admin/types/downloads.ts index beedd84..b7b3ab3 100644 --- a/admin/types/downloads.ts +++ b/admin/types/downloads.ts @@ -90,3 +90,28 @@ export type CuratedCategory = { export type CuratedCategoriesFile = { categories: CuratedCategory[] } + +// Wikipedia selector types +export type WikipediaOption = { + id: string + name: string + description: string + size_mb: number + url: string | null +} + +export type WikipediaOptionsFile = { + options: WikipediaOption[] +} + +export type WikipediaCurrentSelection = { + optionId: string + status: 'none' | 'downloading' | 'installed' | 'failed' + filename: string | null + url: string | null +} + +export type WikipediaState = { + options: WikipediaOption[] + currentSelection: WikipediaCurrentSelection | null +} diff --git a/collections/kiwix-categories.json b/collections/kiwix-categories.json index f9afab1..efb199e 100644 --- a/collections/kiwix-categories.json +++ b/collections/kiwix-categories.json @@ -154,15 +154,9 @@ { "name": "Essential", "slug": "education-essential", - "description": "Core reference materials - Wikipedia's best articles and open textbooks. Lightweight, text-focused.", + "description": "Core reference materials - open textbooks and essential educational content. Lightweight, text-focused.", "recommended": true, "resources": [ - { - "title": "Wikipedia Top 45,000 Articles", - "description": "The 45,000 best Wikipedia articles, optimized for size (no images)", - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim", - "size_mb": 1880 - }, { "title": "Wikibooks", "description": "Open-content textbooks covering math, science, computing, and more", @@ -224,15 +218,9 @@ { "name": "Comprehensive", "slug": "education-comprehensive", - "description": "Complete educational library with full Wikipedia, enhanced textbooks, and TED talks. Includes Standard.", + "description": "Complete educational library with enhanced textbooks and TED talks. Includes Standard.", "includesTier": "education-standard", "resources": [ - { - "title": "Wikipedia (Full, No Images)", - "description": "Complete English Wikipedia - over 6 million articles", - "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim", - "size_mb": 11400 - }, { "title": "Wikibooks (With Images)", "description": "Open textbooks with full illustrations and diagrams", diff --git a/collections/wikipedia.json b/collections/wikipedia.json new file mode 100644 index 0000000..4350629 --- /dev/null +++ b/collections/wikipedia.json @@ -0,0 +1,46 @@ +{ + "options": [ + { + "id": "none", + "name": "No Wikipedia", + "description": "Skip Wikipedia installation", + "size_mb": 0, + "url": null + }, + { + "id": "top-mini", + "name": "Quick Reference", + "description": "Top 100,000 articles with minimal images. Great for quick lookups.", + "size_mb": 313, + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_mini_2025-12.zim" + }, + { + "id": "top-nopic", + "name": "Popular Articles", + "description": "Top articles without images. Good balance of content and size.", + "size_mb": 2100, + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim" + }, + { + "id": "all-mini", + "name": "Complete Wikipedia (Compact)", + "description": "All 6+ million articles in condensed format.", + "size_mb": 11400, + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim" + }, + { + "id": "all-nopic", + "name": "Complete Wikipedia (No Images)", + "description": "All articles without images. Comprehensive offline reference.", + "size_mb": 25000, + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_nopic_2025-12.zim" + }, + { + "id": "all-maxi", + "name": "Complete Wikipedia (Full)", + "description": "The complete experience with all images and media.", + "size_mb": 102000, + "url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_maxi_2024-01.zim" + } + ] +}