mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-28 03:29:25 +01:00
feat: Add dedicated Wikipedia Selector with smart package management
Adds a standalone Wikipedia selection section that appears prominently in both the Easy Setup Wizard and Content Explorer. Features include: - Six Wikipedia package options ranging from Quick Reference (313MB) to Complete Wikipedia with Full Media (99.6GB) - Card-based radio selection UI with clear size indicators - Smart replacement: downloads new package before deleting old one - Status tracking: shows Installed, Selected, or Downloading badges - "No Wikipedia" option for users who want to skip or remove Wikipedia Technical changes: - New wikipedia_selections database table and model - New /api/zim/wikipedia and /api/zim/wikipedia/select endpoints - WikipediaSelector component with consistent styling - Integration with existing download queue system - Callback updates status to 'installed' on successful download - Wikipedia removed from tiered category system to avoid duplication UI improvements: - Added section dividers and icons (AI Models, Wikipedia, Additional Content) - Consistent spacing between major sections in Easy Setup Wizard - Content Explorer gets matching Wikipedia section with submit button Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood
Jake Turner
parent
80aa556b42
commit
68f374e3a8
|
|
@ -4,6 +4,7 @@ import {
|
|||
filenameParamValidator,
|
||||
remoteDownloadValidator,
|
||||
saveInstalledTierValidator,
|
||||
selectWikipediaValidator,
|
||||
} from '#validators/common'
|
||||
import { listRemoteZimValidator } from '#validators/zim'
|
||||
import { inject } from '@adonisjs/core'
|
||||
|
|
@ -79,4 +80,15 @@ export default class ZimController {
|
|||
message: 'ZIM file deleted successfully',
|
||||
}
|
||||
}
|
||||
|
||||
// Wikipedia selector endpoints
|
||||
|
||||
async getWikipediaState({}: HttpContext) {
|
||||
return this.zimService.getWikipediaState()
|
||||
}
|
||||
|
||||
async selectWikipedia({ request }: HttpContext) {
|
||||
const payload = await request.validateUsing(selectWikipediaValidator)
|
||||
return this.zimService.selectWikipedia(payload.optionId)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
27
admin/app/models/wikipedia_selection.ts
Normal file
27
admin/app/models/wikipedia_selection.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class WikipediaSelection extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare option_id: string
|
||||
|
||||
@column()
|
||||
declare url: string | null
|
||||
|
||||
@column()
|
||||
declare filename: string | null
|
||||
|
||||
@column()
|
||||
declare status: 'none' | 'downloading' | 'installed' | 'failed'
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
}
|
||||
|
|
@ -17,12 +17,13 @@ import {
|
|||
ZIM_STORAGE_PATH,
|
||||
} from '../utils/fs.js'
|
||||
import { join } from 'path'
|
||||
import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile } from '../../types/downloads.js'
|
||||
import { CuratedCategory, CuratedCollectionWithStatus, CuratedCollectionsFile, WikipediaOption, WikipediaState } from '../../types/downloads.js'
|
||||
import vine from '@vinejs/vine'
|
||||
import { curatedCategoriesFileSchema, curatedCollectionsFileSchema } from '#validators/curated_collections'
|
||||
import { curatedCategoriesFileSchema, curatedCollectionsFileSchema, wikipediaOptionsFileSchema } from '#validators/curated_collections'
|
||||
import CuratedCollection from '#models/curated_collection'
|
||||
import CuratedCollectionResource from '#models/curated_collection_resource'
|
||||
import InstalledTier from '#models/installed_tier'
|
||||
import WikipediaSelection from '#models/wikipedia_selection'
|
||||
import { RunDownloadJob } from '#jobs/run_download_job'
|
||||
import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js'
|
||||
|
||||
|
|
@ -30,6 +31,7 @@ const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'applicati
|
|||
const CATEGORIES_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/kiwix-categories.json'
|
||||
const COLLECTIONS_URL =
|
||||
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json'
|
||||
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/master/collections/wikipedia.json'
|
||||
|
||||
|
||||
|
||||
|
|
@ -231,6 +233,13 @@ export class ZimService implements IZimService {
|
|||
}
|
||||
|
||||
async downloadRemoteSuccessCallback(urls: string[], restart = true) {
|
||||
// Check if any URL is a Wikipedia download and handle it
|
||||
for (const url of urls) {
|
||||
if (url.includes('wikipedia_en_')) {
|
||||
await this.onWikipediaDownloadComplete(url, true)
|
||||
}
|
||||
}
|
||||
|
||||
// Restart KIWIX container to pick up new ZIM file
|
||||
if (restart) {
|
||||
await this.dockerService
|
||||
|
|
@ -338,4 +347,206 @@ export class ZimService implements IZimService {
|
|||
|
||||
await deleteFileIfExists(fullPath)
|
||||
}
|
||||
|
||||
// Wikipedia selector methods
|
||||
|
||||
async getWikipediaOptions(): Promise<WikipediaOption[]> {
|
||||
try {
|
||||
const response = await axios.get(WIKIPEDIA_OPTIONS_URL)
|
||||
const data = response.data
|
||||
|
||||
const validated = await vine.validate({
|
||||
schema: wikipediaOptionsFileSchema,
|
||||
data,
|
||||
})
|
||||
|
||||
return validated.options
|
||||
} catch (error) {
|
||||
logger.error(`[ZimService] Failed to fetch Wikipedia options:`, error)
|
||||
throw new Error('Failed to fetch Wikipedia options')
|
||||
}
|
||||
}
|
||||
|
||||
async getWikipediaSelection(): Promise<WikipediaSelection | null> {
|
||||
// Get the single row from wikipedia_selections (there should only ever be one)
|
||||
return WikipediaSelection.query().first()
|
||||
}
|
||||
|
||||
async getWikipediaState(): Promise<WikipediaState> {
|
||||
const options = await this.getWikipediaOptions()
|
||||
const selection = await this.getWikipediaSelection()
|
||||
|
||||
return {
|
||||
options,
|
||||
currentSelection: selection
|
||||
? {
|
||||
optionId: selection.option_id,
|
||||
status: selection.status,
|
||||
filename: selection.filename,
|
||||
url: selection.url,
|
||||
}
|
||||
: null,
|
||||
}
|
||||
}
|
||||
|
||||
async selectWikipedia(optionId: string): Promise<{ success: boolean; jobId?: string; message?: string }> {
|
||||
const options = await this.getWikipediaOptions()
|
||||
const selectedOption = options.find((opt) => opt.id === optionId)
|
||||
|
||||
if (!selectedOption) {
|
||||
throw new Error(`Invalid Wikipedia option: ${optionId}`)
|
||||
}
|
||||
|
||||
const currentSelection = await this.getWikipediaSelection()
|
||||
|
||||
// If same as currently installed, no action needed
|
||||
if (currentSelection?.option_id === optionId && currentSelection.status === 'installed') {
|
||||
return { success: true, message: 'Already installed' }
|
||||
}
|
||||
|
||||
// Handle "none" option - delete current Wikipedia file and update DB
|
||||
if (optionId === 'none') {
|
||||
if (currentSelection?.filename) {
|
||||
try {
|
||||
await this.delete(currentSelection.filename)
|
||||
logger.info(`[ZimService] Deleted Wikipedia file: ${currentSelection.filename}`)
|
||||
} catch (error) {
|
||||
// File might already be deleted, that's OK
|
||||
logger.warn(`[ZimService] Could not delete Wikipedia file (may already be gone): ${currentSelection.filename}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Update or create the selection record (always use first record)
|
||||
if (currentSelection) {
|
||||
currentSelection.option_id = 'none'
|
||||
currentSelection.url = null
|
||||
currentSelection.filename = null
|
||||
currentSelection.status = 'none'
|
||||
await currentSelection.save()
|
||||
} else {
|
||||
await WikipediaSelection.create({
|
||||
option_id: 'none',
|
||||
url: null,
|
||||
filename: null,
|
||||
status: 'none',
|
||||
})
|
||||
}
|
||||
|
||||
// Restart Kiwix to reflect the change
|
||||
await this.dockerService
|
||||
.affectContainer(DockerService.KIWIX_SERVICE_NAME, 'restart')
|
||||
.catch((error) => {
|
||||
logger.error(`[ZimService] Failed to restart Kiwix after Wikipedia removal:`, error)
|
||||
})
|
||||
|
||||
return { success: true, message: 'Wikipedia removed' }
|
||||
}
|
||||
|
||||
// Start download for the new Wikipedia option
|
||||
if (!selectedOption.url) {
|
||||
throw new Error('Selected Wikipedia option has no download URL')
|
||||
}
|
||||
|
||||
// Check if already downloading
|
||||
const existingJob = await RunDownloadJob.getByUrl(selectedOption.url)
|
||||
if (existingJob) {
|
||||
return { success: false, message: 'Download already in progress' }
|
||||
}
|
||||
|
||||
// Extract filename from URL
|
||||
const filename = selectedOption.url.split('/').pop()
|
||||
if (!filename) {
|
||||
throw new Error('Could not determine filename from URL')
|
||||
}
|
||||
|
||||
const filepath = join(process.cwd(), ZIM_STORAGE_PATH, filename)
|
||||
|
||||
// Update or create selection record to show downloading status
|
||||
let selection: WikipediaSelection
|
||||
if (currentSelection) {
|
||||
currentSelection.option_id = optionId
|
||||
currentSelection.url = selectedOption.url
|
||||
currentSelection.filename = filename
|
||||
currentSelection.status = 'downloading'
|
||||
await currentSelection.save()
|
||||
selection = currentSelection
|
||||
} else {
|
||||
selection = await WikipediaSelection.create({
|
||||
option_id: optionId,
|
||||
url: selectedOption.url,
|
||||
filename: filename,
|
||||
status: 'downloading',
|
||||
})
|
||||
}
|
||||
|
||||
// Dispatch download job
|
||||
const result = await RunDownloadJob.dispatch({
|
||||
url: selectedOption.url,
|
||||
filepath,
|
||||
timeout: 30000,
|
||||
allowedMimeTypes: ZIM_MIME_TYPES,
|
||||
forceNew: true,
|
||||
filetype: 'zim',
|
||||
})
|
||||
|
||||
if (!result || !result.job) {
|
||||
// Revert status on failure to dispatch
|
||||
selection.option_id = currentSelection?.option_id || 'none'
|
||||
selection.url = currentSelection?.url || null
|
||||
selection.filename = currentSelection?.filename || null
|
||||
selection.status = currentSelection?.status || 'none'
|
||||
await selection.save()
|
||||
throw new Error('Failed to dispatch download job')
|
||||
}
|
||||
|
||||
logger.info(`[ZimService] Started Wikipedia download for ${optionId}: ${filename}`)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobId: result.job.id,
|
||||
message: 'Download started',
|
||||
}
|
||||
}
|
||||
|
||||
async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
|
||||
const selection = await this.getWikipediaSelection()
|
||||
|
||||
if (!selection || selection.url !== url) {
|
||||
logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`)
|
||||
return
|
||||
}
|
||||
|
||||
if (success) {
|
||||
// Get the old filename before updating (if there was a previous Wikipedia installed)
|
||||
const options = await this.getWikipediaOptions()
|
||||
const previousOption = options.find((opt) => opt.id !== selection.option_id && opt.id !== 'none')
|
||||
|
||||
// Update status to installed
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`)
|
||||
|
||||
// Delete the old Wikipedia file if it exists and is different
|
||||
// We need to find what was previously installed
|
||||
const existingFiles = await this.list()
|
||||
const wikipediaFiles = existingFiles.files.filter((f) =>
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== selection.filename
|
||||
)
|
||||
|
||||
for (const oldFile of wikipediaFiles) {
|
||||
try {
|
||||
await this.delete(oldFile.name)
|
||||
logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile.name}`)
|
||||
} catch (error) {
|
||||
logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile.name}`, error)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Download failed - keep the selection record but mark as failed
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,3 +43,9 @@ export const saveInstalledTierValidator = vine.compile(
|
|||
tierSlug: vine.string().trim().minLength(1),
|
||||
})
|
||||
)
|
||||
|
||||
export const selectWikipediaValidator = vine.compile(
|
||||
vine.object({
|
||||
optionId: vine.string().trim().minLength(1),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -45,3 +45,18 @@ export const curatedCategoriesFileSchema = vine.object({
|
|||
})
|
||||
),
|
||||
})
|
||||
|
||||
/**
|
||||
* For validating the Wikipedia options file
|
||||
*/
|
||||
export const wikipediaOptionSchema = vine.object({
|
||||
id: vine.string(),
|
||||
name: vine.string(),
|
||||
description: vine.string(),
|
||||
size_mb: vine.number().min(0),
|
||||
url: vine.string().url().nullable(),
|
||||
})
|
||||
|
||||
export const wikipediaOptionsFileSchema = vine.object({
|
||||
options: vine.array(wikipediaOptionSchema).minLength(1),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'wikipedia_selections'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('option_id').notNullable()
|
||||
table.string('url').nullable()
|
||||
table.string('filename').nullable()
|
||||
table.enum('status', ['none', 'downloading', 'installed', 'failed']).defaultTo('none')
|
||||
table.timestamp('created_at')
|
||||
table.timestamp('updated_at')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
156
admin/inertia/components/WikipediaSelector.tsx
Normal file
156
admin/inertia/components/WikipediaSelector.tsx
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import { formatBytes } from '~/lib/util'
|
||||
import { WikipediaOption, WikipediaCurrentSelection } from '../../types/downloads'
|
||||
import classNames from 'classnames'
|
||||
import { IconCheck, IconDownload, IconWorld } from '@tabler/icons-react'
|
||||
import StyledButton from './StyledButton'
|
||||
import LoadingSpinner from './LoadingSpinner'
|
||||
|
||||
export interface WikipediaSelectorProps {
|
||||
options: WikipediaOption[]
|
||||
currentSelection: WikipediaCurrentSelection | null
|
||||
selectedOptionId: string | null // for wizard (pending selection)
|
||||
onSelect: (optionId: string) => void
|
||||
disabled?: boolean
|
||||
showSubmitButton?: boolean // true for Content Explorer, false for wizard
|
||||
onSubmit?: () => void
|
||||
isSubmitting?: boolean
|
||||
}
|
||||
|
||||
const WikipediaSelector: React.FC<WikipediaSelectorProps> = ({
|
||||
options,
|
||||
currentSelection,
|
||||
selectedOptionId,
|
||||
onSelect,
|
||||
disabled = false,
|
||||
showSubmitButton = false,
|
||||
onSubmit,
|
||||
isSubmitting = false,
|
||||
}) => {
|
||||
// Determine which option to highlight
|
||||
const highlightedOptionId = selectedOptionId ?? currentSelection?.optionId ?? null
|
||||
|
||||
// Check if current selection is downloading
|
||||
const isDownloading = currentSelection?.status === 'downloading'
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
{/* Header with Wikipedia branding */}
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div className="w-10 h-10 rounded-full bg-white flex items-center justify-center shadow-sm">
|
||||
<IconWorld className="w-6 h-6 text-gray-700" />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-xl font-semibold text-gray-900">Wikipedia</h3>
|
||||
<p className="text-sm text-gray-500">Select your preferred Wikipedia package</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Downloading status message */}
|
||||
{isDownloading && (
|
||||
<div className="mb-4 p-3 bg-blue-50 border border-blue-200 rounded-lg flex items-center gap-2">
|
||||
<LoadingSpinner size="sm" />
|
||||
<span className="text-sm text-blue-700">
|
||||
Downloading Wikipedia... This may take a while for larger packages.
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Options grid */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
{options.map((option) => {
|
||||
const isSelected = highlightedOptionId === option.id
|
||||
const isInstalled =
|
||||
currentSelection?.optionId === option.id && currentSelection?.status === 'installed'
|
||||
const isCurrentDownloading =
|
||||
currentSelection?.optionId === option.id && currentSelection?.status === 'downloading'
|
||||
const isPending = selectedOptionId === option.id && selectedOptionId !== currentSelection?.optionId
|
||||
|
||||
return (
|
||||
<div
|
||||
key={option.id}
|
||||
onClick={() => !disabled && !isCurrentDownloading && onSelect(option.id)}
|
||||
className={classNames(
|
||||
'relative p-4 rounded-lg border-2 transition-all',
|
||||
disabled || isCurrentDownloading
|
||||
? 'opacity-60 cursor-not-allowed'
|
||||
: 'cursor-pointer hover:shadow-md',
|
||||
isInstalled
|
||||
? 'border-desert-green bg-desert-green/10'
|
||||
: isSelected
|
||||
? 'border-lime-500 bg-lime-50'
|
||||
: 'border-gray-200 bg-white hover:border-gray-300'
|
||||
)}
|
||||
>
|
||||
{/* Status badges */}
|
||||
<div className="absolute top-2 right-2 flex gap-1">
|
||||
{isInstalled && (
|
||||
<span className="text-xs bg-desert-green text-white px-2 py-0.5 rounded-full flex items-center gap-1">
|
||||
<IconCheck size={12} />
|
||||
Installed
|
||||
</span>
|
||||
)}
|
||||
{isPending && !isInstalled && (
|
||||
<span className="text-xs bg-lime-500 text-white px-2 py-0.5 rounded-full">
|
||||
Selected
|
||||
</span>
|
||||
)}
|
||||
{isCurrentDownloading && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded-full flex items-center gap-1">
|
||||
<IconDownload size={12} />
|
||||
Downloading
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Option content */}
|
||||
<div className="pr-16 flex flex-col h-full">
|
||||
<h4 className="text-lg font-semibold text-gray-900 mb-1">{option.name}</h4>
|
||||
<p className="text-sm text-gray-600 mb-3 flex-grow">{option.description}</p>
|
||||
<div className="flex items-center gap-3">
|
||||
{/* Radio indicator */}
|
||||
<div
|
||||
className={classNames(
|
||||
'w-5 h-5 rounded-full border-2 flex items-center justify-center transition-all flex-shrink-0',
|
||||
isSelected
|
||||
? isInstalled
|
||||
? 'border-desert-green bg-desert-green'
|
||||
: 'border-lime-500 bg-lime-500'
|
||||
: 'border-gray-300'
|
||||
)}
|
||||
>
|
||||
{isSelected && <IconCheck size={12} className="text-white" />}
|
||||
</div>
|
||||
<span
|
||||
className={classNames(
|
||||
'text-sm font-medium px-2 py-1 rounded',
|
||||
option.size_mb === 0 ? 'bg-gray-100 text-gray-500' : 'bg-gray-100 text-gray-700'
|
||||
)}
|
||||
>
|
||||
{option.size_mb === 0 ? 'No download' : formatBytes(option.size_mb * 1024 * 1024, 1)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Submit button for Content Explorer mode */}
|
||||
{showSubmitButton && selectedOptionId && selectedOptionId !== currentSelection?.optionId && (
|
||||
<div className="mt-4 flex justify-end">
|
||||
<StyledButton
|
||||
variant="primary"
|
||||
onClick={onSubmit}
|
||||
disabled={isSubmitting || disabled}
|
||||
loading={isSubmitting}
|
||||
icon="ArrowDownTrayIcon"
|
||||
>
|
||||
{selectedOptionId === 'none' ? 'Remove Wikipedia' : 'Download Selected'}
|
||||
</StyledButton>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default WikipediaSelector
|
||||
|
|
@ -7,6 +7,7 @@ import {
|
|||
CuratedCategory,
|
||||
CuratedCollectionWithStatus,
|
||||
DownloadJobWithProgress,
|
||||
WikipediaState,
|
||||
} from '../../types/downloads'
|
||||
import { catchInternal } from './util'
|
||||
import { NomadOllamaModel, OllamaChatRequest } from '../../types/ollama'
|
||||
|
|
@ -402,6 +403,28 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
// Wikipedia selector methods
|
||||
|
||||
async getWikipediaState(): Promise<WikipediaState | undefined> {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<WikipediaState>('/zim/wikipedia')
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async selectWikipedia(
|
||||
optionId: string
|
||||
): Promise<{ success: boolean; jobId?: string; message?: string } | undefined> {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
success: boolean
|
||||
jobId?: string
|
||||
message?: string
|
||||
}>('/zim/wikipedia/select', { optionId })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async uploadDocument(file: File) {
|
||||
return catchInternal(async () => {
|
||||
const formData = new FormData()
|
||||
|
|
|
|||
|
|
@ -8,15 +8,16 @@ import { ServiceSlim } from '../../../types/services'
|
|||
import CuratedCollectionCard from '~/components/CuratedCollectionCard'
|
||||
import CategoryCard from '~/components/CategoryCard'
|
||||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
import WikipediaSelector from '~/components/WikipediaSelector'
|
||||
import LoadingSpinner from '~/components/LoadingSpinner'
|
||||
import Alert from '~/components/Alert'
|
||||
import { IconCheck, IconChevronDown, IconChevronUp } from '@tabler/icons-react'
|
||||
import { IconCheck, IconChevronDown, IconChevronUp, IconArrowRight, IconCpu, IconBooks } from '@tabler/icons-react'
|
||||
import StorageProjectionBar from '~/components/StorageProjectionBar'
|
||||
import { useNotifications } from '~/context/NotificationContext'
|
||||
import useInternetStatus from '~/hooks/useInternetStatus'
|
||||
import { useSystemInfo } from '~/hooks/useSystemInfo'
|
||||
import classNames from 'classnames'
|
||||
import { CuratedCategory, CategoryTier, CategoryResource } from '../../../types/downloads'
|
||||
import { CuratedCategory, CategoryTier, CategoryResource, WikipediaState } from '../../../types/downloads'
|
||||
|
||||
// Capability definitions - maps user-friendly categories to services
|
||||
interface Capability {
|
||||
|
|
@ -105,6 +106,7 @@ type WizardStep = 1 | 2 | 3 | 4
|
|||
const CURATED_MAP_COLLECTIONS_KEY = 'curated-map-collections'
|
||||
const CURATED_ZIM_COLLECTIONS_KEY = 'curated-zim-collections'
|
||||
const CURATED_CATEGORIES_KEY = 'curated-categories'
|
||||
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
|
||||
|
||||
// Helper to get all resources for a tier (including inherited resources)
|
||||
const getAllResourcesForTier = (
|
||||
|
|
@ -135,6 +137,9 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
const [tierModalOpen, setTierModalOpen] = useState(false)
|
||||
const [activeCategory, setActiveCategory] = useState<CuratedCategory | null>(null)
|
||||
|
||||
// Wikipedia selection state
|
||||
const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
|
||||
|
||||
const { addNotification } = useNotifications()
|
||||
const { isOnline } = useInternetStatus()
|
||||
const queryClient = useQueryClient()
|
||||
|
|
@ -145,7 +150,8 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
selectedMapCollections.length > 0 ||
|
||||
selectedZimCollections.length > 0 ||
|
||||
selectedTiers.size > 0 ||
|
||||
selectedAiModels.length > 0
|
||||
selectedAiModels.length > 0 ||
|
||||
(selectedWikipedia !== null && selectedWikipedia !== 'none')
|
||||
|
||||
const { data: mapCollections, isLoading: isLoadingMaps } = useQuery({
|
||||
queryKey: [CURATED_MAP_COLLECTIONS_KEY],
|
||||
|
|
@ -172,6 +178,13 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// Fetch Wikipedia options and current state
|
||||
const { data: wikipediaState, isLoading: isLoadingWikipedia } = useQuery({
|
||||
queryKey: [WIKIPEDIA_STATE_KEY],
|
||||
queryFn: () => api.getWikipediaState(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// All services for display purposes
|
||||
const allServices = props.system.services
|
||||
|
||||
|
|
@ -289,16 +302,26 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
})
|
||||
}
|
||||
|
||||
// Add Wikipedia selection
|
||||
if (selectedWikipedia && wikipediaState) {
|
||||
const option = wikipediaState.options.find((o) => o.id === selectedWikipedia)
|
||||
if (option && option.size_mb > 0) {
|
||||
totalBytes += option.size_mb * 1024 * 1024
|
||||
}
|
||||
}
|
||||
|
||||
return totalBytes
|
||||
}, [
|
||||
selectedTiers,
|
||||
selectedMapCollections,
|
||||
selectedZimCollections,
|
||||
selectedAiModels,
|
||||
selectedWikipedia,
|
||||
categories,
|
||||
mapCollections,
|
||||
zimCollections,
|
||||
recommendedModels,
|
||||
wikipediaState,
|
||||
])
|
||||
|
||||
// Get primary disk/filesystem info for storage projection
|
||||
|
|
@ -385,6 +408,11 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
)
|
||||
await Promise.all(tierSavePromises)
|
||||
|
||||
// Select Wikipedia option if one was chosen
|
||||
if (selectedWikipedia && selectedWikipedia !== wikipediaState?.currentSelection?.optionId) {
|
||||
await api.selectWikipedia(selectedWikipedia)
|
||||
}
|
||||
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message: 'Setup wizard completed! Your selections are being processed.',
|
||||
|
|
@ -799,11 +827,14 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
{/* AI Model Selection - Only show if AI capability is selected */}
|
||||
{isAiSelected && (
|
||||
<div className="mb-8">
|
||||
<div className="mb-4">
|
||||
<h3 className="text-2xl font-semibold text-gray-900 mb-2">Choose AI Models</h3>
|
||||
<p className="text-gray-600">
|
||||
Select AI models to download. We've recommended some smaller, popular models to get you started. You'll need at least one to use AI features, but you can always add more later.
|
||||
</p>
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div className="w-10 h-10 rounded-full bg-white border border-gray-200 flex items-center justify-center shadow-sm">
|
||||
<IconCpu className="w-6 h-6 text-gray-700" />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-xl font-semibold text-gray-900">AI Models</h3>
|
||||
<p className="text-sm text-gray-500">Select models to download for offline AI</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isLoadingRecommendedModels ? (
|
||||
|
|
@ -879,9 +910,46 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Wikipedia Selection - Only show if Information capability is selected */}
|
||||
{isInformationSelected && (
|
||||
<>
|
||||
{/* Divider between AI Models and Wikipedia */}
|
||||
{isAiSelected && <hr className="my-8 border-gray-200" />}
|
||||
|
||||
<div className="mb-8">
|
||||
{isLoadingWikipedia ? (
|
||||
<div className="flex justify-center py-12">
|
||||
<LoadingSpinner />
|
||||
</div>
|
||||
) : wikipediaState && wikipediaState.options.length > 0 ? (
|
||||
<WikipediaSelector
|
||||
options={wikipediaState.options}
|
||||
currentSelection={wikipediaState.currentSelection}
|
||||
selectedOptionId={selectedWikipedia}
|
||||
onSelect={(optionId) => isOnline && setSelectedWikipedia(optionId)}
|
||||
disabled={!isOnline}
|
||||
/>
|
||||
) : null}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Curated Categories with Tiers - Only show if Information capability is selected */}
|
||||
{isInformationSelected && (
|
||||
<>
|
||||
{/* Divider between Wikipedia and Additional Content */}
|
||||
<hr className="my-8 border-gray-200" />
|
||||
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div className="w-10 h-10 rounded-full bg-white border border-gray-200 flex items-center justify-center shadow-sm">
|
||||
<IconBooks className="w-6 h-6 text-gray-700" />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-xl font-semibold text-gray-900">Additional Content</h3>
|
||||
<p className="text-sm text-gray-500">Curated collections for offline reference</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isLoadingCategories ? (
|
||||
<div className="flex justify-center py-12">
|
||||
<LoadingSpinner />
|
||||
|
|
@ -979,7 +1047,8 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
selectedMapCollections.length > 0 ||
|
||||
selectedZimCollections.length > 0 ||
|
||||
selectedTiers.size > 0 ||
|
||||
selectedAiModels.length > 0
|
||||
selectedAiModels.length > 0 ||
|
||||
(selectedWikipedia !== null && selectedWikipedia !== 'none')
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
|
|
@ -1091,6 +1160,28 @@ export default function EasySetupWizard(props: { system: { services: ServiceSlim
|
|||
</div>
|
||||
)}
|
||||
|
||||
{selectedWikipedia && selectedWikipedia !== 'none' && (
|
||||
<div className="bg-white rounded-lg border-2 border-desert-stone-light p-6">
|
||||
<h3 className="text-xl font-semibold text-gray-900 mb-4">Wikipedia</h3>
|
||||
{(() => {
|
||||
const option = wikipediaState?.options.find((o) => o.id === selectedWikipedia)
|
||||
return option ? (
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center">
|
||||
<IconCheck size={20} className="text-desert-green mr-2" />
|
||||
<span className="text-gray-700">{option.name}</span>
|
||||
</div>
|
||||
<span className="text-gray-500 text-sm">
|
||||
{option.size_mb > 0
|
||||
? `${(option.size_mb / 1024).toFixed(1)} GB`
|
||||
: 'No download'}
|
||||
</span>
|
||||
</div>
|
||||
) : null
|
||||
})()}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selectedAiModels.length > 0 && (
|
||||
<div className="bg-white rounded-lg border-2 border-desert-stone-light p-6">
|
||||
<h3 className="text-xl font-semibold text-gray-900 mb-4">
|
||||
|
|
|
|||
|
|
@ -21,23 +21,26 @@ import useInternetStatus from '~/hooks/useInternetStatus'
|
|||
import Alert from '~/components/Alert'
|
||||
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||
import Input from '~/components/inputs/Input'
|
||||
import { IconSearch } from '@tabler/icons-react'
|
||||
import { IconSearch, IconBooks } from '@tabler/icons-react'
|
||||
import useDebounce from '~/hooks/useDebounce'
|
||||
import CuratedCollectionCard from '~/components/CuratedCollectionCard'
|
||||
import CategoryCard from '~/components/CategoryCard'
|
||||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
import WikipediaSelector from '~/components/WikipediaSelector'
|
||||
import StyledSectionHeader from '~/components/StyledSectionHeader'
|
||||
import {
|
||||
CuratedCollectionWithStatus,
|
||||
CuratedCategory,
|
||||
CategoryTier,
|
||||
CategoryResource,
|
||||
WikipediaState,
|
||||
} from '../../../../types/downloads'
|
||||
import useDownloads from '~/hooks/useDownloads'
|
||||
import ActiveDownloads from '~/components/ActiveDownloads'
|
||||
|
||||
const CURATED_COLLECTIONS_KEY = 'curated-zim-collections'
|
||||
const CURATED_CATEGORIES_KEY = 'curated-categories'
|
||||
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
|
||||
|
||||
// Helper to get all resources for a tier (including inherited resources)
|
||||
const getAllResourcesForTier = (tier: CategoryTier, allTiers: CategoryTier[]): CategoryResource[] => {
|
||||
|
|
@ -68,6 +71,10 @@ export default function ZimRemoteExplorer() {
|
|||
const [tierModalOpen, setTierModalOpen] = useState(false)
|
||||
const [activeCategory, setActiveCategory] = useState<CuratedCategory | null>(null)
|
||||
|
||||
// Wikipedia selection state
|
||||
const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
|
||||
const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
|
||||
|
||||
const debouncedSetQuery = debounce((val: string) => {
|
||||
setQuery(val)
|
||||
}, 400)
|
||||
|
|
@ -85,6 +92,13 @@ export default function ZimRemoteExplorer() {
|
|||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// Fetch Wikipedia options and state
|
||||
const { data: wikipediaState, isLoading: isLoadingWikipedia } = useQuery({
|
||||
queryKey: [WIKIPEDIA_STATE_KEY],
|
||||
queryFn: () => api.getWikipediaState(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
const { data: downloads, invalidate: invalidateDownloads } = useDownloads({
|
||||
filetype: 'zim',
|
||||
enabled: true,
|
||||
|
|
@ -253,6 +267,46 @@ export default function ZimRemoteExplorer() {
|
|||
setActiveCategory(null)
|
||||
}
|
||||
|
||||
// Wikipedia selection handlers
|
||||
const handleWikipediaSelect = (optionId: string) => {
|
||||
if (!isOnline) return
|
||||
setSelectedWikipedia(optionId)
|
||||
}
|
||||
|
||||
const handleWikipediaSubmit = async () => {
|
||||
if (!selectedWikipedia) return
|
||||
|
||||
setIsSubmittingWikipedia(true)
|
||||
try {
|
||||
const result = await api.selectWikipedia(selectedWikipedia)
|
||||
if (result?.success) {
|
||||
addNotification({
|
||||
message:
|
||||
selectedWikipedia === 'none'
|
||||
? 'Wikipedia removed successfully'
|
||||
: 'Wikipedia download started',
|
||||
type: 'success',
|
||||
})
|
||||
invalidateDownloads()
|
||||
queryClient.invalidateQueries({ queryKey: [WIKIPEDIA_STATE_KEY] })
|
||||
setSelectedWikipedia(null)
|
||||
} else {
|
||||
addNotification({
|
||||
message: result?.message || 'Failed to change Wikipedia selection',
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error selecting Wikipedia:', error)
|
||||
addNotification({
|
||||
message: 'An error occurred while changing Wikipedia selection',
|
||||
type: 'error',
|
||||
})
|
||||
} finally {
|
||||
setIsSubmittingWikipedia(false)
|
||||
}
|
||||
}
|
||||
|
||||
const fetchLatestCollections = useMutation({
|
||||
mutationFn: () => api.fetchLatestZimCollections(),
|
||||
onSuccess: () => {
|
||||
|
|
@ -308,7 +362,38 @@ export default function ZimRemoteExplorer() {
|
|||
Fetch Latest Collections
|
||||
</StyledButton>
|
||||
|
||||
{/* Wikipedia Selector */}
|
||||
{isLoadingWikipedia ? (
|
||||
<div className="mt-8 bg-white rounded-lg border border-gray-200 p-6">
|
||||
<div className="flex justify-center py-6">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-desert-green"></div>
|
||||
</div>
|
||||
</div>
|
||||
) : wikipediaState && wikipediaState.options.length > 0 ? (
|
||||
<div className="mt-8 bg-white rounded-lg border border-gray-200 p-6">
|
||||
<WikipediaSelector
|
||||
options={wikipediaState.options}
|
||||
currentSelection={wikipediaState.currentSelection}
|
||||
selectedOptionId={selectedWikipedia}
|
||||
onSelect={handleWikipediaSelect}
|
||||
disabled={!isOnline}
|
||||
showSubmitButton
|
||||
onSubmit={handleWikipediaSubmit}
|
||||
isSubmitting={isSubmittingWikipedia}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Tiered Category Collections - matches Easy Setup Wizard */}
|
||||
<div className="flex items-center gap-3 mt-8 mb-4">
|
||||
<div className="w-10 h-10 rounded-full bg-white border border-gray-200 flex items-center justify-center shadow-sm">
|
||||
<IconBooks className="w-6 h-6 text-gray-700" />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-xl font-semibold text-gray-900">Additional Content</h3>
|
||||
<p className="text-sm text-gray-500">Curated collections for offline reference</p>
|
||||
</div>
|
||||
</div>
|
||||
{categories && categories.length > 0 ? (
|
||||
<>
|
||||
<div className="mt-4 grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6">
|
||||
|
|
|
|||
|
|
@ -145,6 +145,8 @@ router
|
|||
router.post('/download-remote', [ZimController, 'downloadRemote'])
|
||||
router.post('/download-collection', [ZimController, 'downloadCollection'])
|
||||
router.post('/save-installed-tier', [ZimController, 'saveInstalledTier'])
|
||||
router.get('/wikipedia', [ZimController, 'getWikipediaState'])
|
||||
router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])
|
||||
router.delete('/:filename', [ZimController, 'delete'])
|
||||
})
|
||||
.prefix('/api/zim')
|
||||
|
|
|
|||
|
|
@ -90,3 +90,28 @@ export type CuratedCategory = {
|
|||
export type CuratedCategoriesFile = {
|
||||
categories: CuratedCategory[]
|
||||
}
|
||||
|
||||
// Wikipedia selector types
|
||||
export type WikipediaOption = {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
size_mb: number
|
||||
url: string | null
|
||||
}
|
||||
|
||||
export type WikipediaOptionsFile = {
|
||||
options: WikipediaOption[]
|
||||
}
|
||||
|
||||
export type WikipediaCurrentSelection = {
|
||||
optionId: string
|
||||
status: 'none' | 'downloading' | 'installed' | 'failed'
|
||||
filename: string | null
|
||||
url: string | null
|
||||
}
|
||||
|
||||
export type WikipediaState = {
|
||||
options: WikipediaOption[]
|
||||
currentSelection: WikipediaCurrentSelection | null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,15 +154,9 @@
|
|||
{
|
||||
"name": "Essential",
|
||||
"slug": "education-essential",
|
||||
"description": "Core reference materials - Wikipedia's best articles and open textbooks. Lightweight, text-focused.",
|
||||
"description": "Core reference materials - open textbooks and essential educational content. Lightweight, text-focused.",
|
||||
"recommended": true,
|
||||
"resources": [
|
||||
{
|
||||
"title": "Wikipedia Top 45,000 Articles",
|
||||
"description": "The 45,000 best Wikipedia articles, optimized for size (no images)",
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim",
|
||||
"size_mb": 1880
|
||||
},
|
||||
{
|
||||
"title": "Wikibooks",
|
||||
"description": "Open-content textbooks covering math, science, computing, and more",
|
||||
|
|
@ -224,15 +218,9 @@
|
|||
{
|
||||
"name": "Comprehensive",
|
||||
"slug": "education-comprehensive",
|
||||
"description": "Complete educational library with full Wikipedia, enhanced textbooks, and TED talks. Includes Standard.",
|
||||
"description": "Complete educational library with enhanced textbooks and TED talks. Includes Standard.",
|
||||
"includesTier": "education-standard",
|
||||
"resources": [
|
||||
{
|
||||
"title": "Wikipedia (Full, No Images)",
|
||||
"description": "Complete English Wikipedia - over 6 million articles",
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim",
|
||||
"size_mb": 11400
|
||||
},
|
||||
{
|
||||
"title": "Wikibooks (With Images)",
|
||||
"description": "Open textbooks with full illustrations and diagrams",
|
||||
|
|
|
|||
46
collections/wikipedia.json
Normal file
46
collections/wikipedia.json
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"options": [
|
||||
{
|
||||
"id": "none",
|
||||
"name": "No Wikipedia",
|
||||
"description": "Skip Wikipedia installation",
|
||||
"size_mb": 0,
|
||||
"url": null
|
||||
},
|
||||
{
|
||||
"id": "top-mini",
|
||||
"name": "Quick Reference",
|
||||
"description": "Top 100,000 articles with minimal images. Great for quick lookups.",
|
||||
"size_mb": 313,
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_mini_2025-12.zim"
|
||||
},
|
||||
{
|
||||
"id": "top-nopic",
|
||||
"name": "Popular Articles",
|
||||
"description": "Top articles without images. Good balance of content and size.",
|
||||
"size_mb": 2100,
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_top_nopic_2025-12.zim"
|
||||
},
|
||||
{
|
||||
"id": "all-mini",
|
||||
"name": "Complete Wikipedia (Compact)",
|
||||
"description": "All 6+ million articles in condensed format.",
|
||||
"size_mb": 11400,
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_2025-12.zim"
|
||||
},
|
||||
{
|
||||
"id": "all-nopic",
|
||||
"name": "Complete Wikipedia (No Images)",
|
||||
"description": "All articles without images. Comprehensive offline reference.",
|
||||
"size_mb": 25000,
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_nopic_2025-12.zim"
|
||||
},
|
||||
{
|
||||
"id": "all-maxi",
|
||||
"name": "Complete Wikipedia (Full)",
|
||||
"description": "The complete experience with all images and media.",
|
||||
"size_mb": 102000,
|
||||
"url": "https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_maxi_2024-01.zim"
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user