From 3b26f3cf1a93110dd60ca97e2e11c03b354fa0a4 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Sun, 29 Mar 2026 20:31:58 -0700 Subject: [PATCH 1/2] feat(content): add custom ZIM library sources with pre-seeded mirrors Users reported slow download speeds from the default Kiwix CDN. This adds the ability to browse and download ZIM files from alternative Kiwix mirrors or self-hosted repositories, all through the GUI. - Add "Custom Libraries" button next to "Browse the Kiwix Library" - Source dropdown to switch between Default (Kiwix) and custom libraries - Browsable directory structure with breadcrumb navigation - 5 pre-seeded official Kiwix mirrors (US, DE, DK, UK, Global CDN) - Built-in mirrors protected from deletion - Downloads use existing pipeline (progress, cancel, Kiwix restart) - Source selection persists across page loads via localStorage - Scrollable directory browser (600px max) with sticky header - SSRF protection on all custom library URLs Closes #576 Co-Authored-By: Claude Opus 4.6 (1M context) --- admin/app/controllers/zim_controller.ts | 49 +- admin/app/models/custom_library_source.ts | 24 + admin/app/services/zim_service.ts | 132 +++++ admin/app/validators/zim.ts | 27 + ...001_create_custom_library_sources_table.ts | 42 ++ admin/inertia/lib/api.ts | 36 ++ .../pages/settings/zim/remote-explorer.tsx | 545 +++++++++++++++--- admin/start/routes.ts | 6 + 8 files changed, 780 insertions(+), 81 deletions(-) create mode 100644 admin/app/models/custom_library_source.ts create mode 100644 admin/database/migrations/1775100000001_create_custom_library_sources_table.ts diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index 4bb00e3..a075d30 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -6,7 +6,7 @@ import { remoteDownloadWithMetadataValidator, selectWikipediaValidator, } from '#validators/common' -import { listRemoteZimValidator } from '#validators/zim' +import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' @@ -85,4 +85,51 @@ export default class ZimController { const payload = await request.validateUsing(selectWikipediaValidator) return this.zimService.selectWikipedia(payload.optionId) } + + // Custom library endpoints + + async listCustomLibraries({}: HttpContext) { + return this.zimService.listCustomLibraries() + } + + async addCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(addCustomLibraryValidator) + assertNotPrivateUrl(payload.base_url) + try { + const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url) + return { message: 'Custom library added', library: source } + } catch (error) { + if (error.message === 'Maximum of 10 custom libraries allowed') { + return response.status(400).send({ message: error.message }) + } + throw error + } + } + + async removeCustomLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(idParamValidator) + try { + await this.zimService.removeCustomLibrary(payload.params.id) + return { message: 'Custom library removed' } + } catch (error) { + if (error.message === 'Custom library not found') { + return response.status(404).send({ message: error.message }) + } + throw error + } + } + + async browseLibrary({ request, response }: HttpContext) { + const payload = await request.validateUsing(browseLibraryValidator) + try { + return await this.zimService.browseLibraryUrl(payload.url) + } catch (error) { + if (error.message?.includes('loopback or link-local')) { + return response.status(400).send({ message: error.message }) + } + return response.status(502).send({ + message: 'Could not fetch directory listing from the provided URL', + }) + } + } } diff --git a/admin/app/models/custom_library_source.ts b/admin/app/models/custom_library_source.ts new file mode 100644 index 0000000..478d9a0 --- /dev/null +++ b/admin/app/models/custom_library_source.ts @@ -0,0 +1,24 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' + +export default class CustomLibrarySource extends BaseModel { + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare name: string + + @column() + declare base_url: string + + @column() + declare is_default: boolean + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime +} diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 3eee1cb..93e3203 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -26,6 +26,8 @@ import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' import { CollectionManifestService } from './collection_manifest_service.js' import type { CategoryWithStatus } from '../../types/collections.js' +import CustomLibrarySource from '#models/custom_library_source' +import { assertNotPrivateUrl } from '#validators/common' const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream'] const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json' @@ -555,4 +557,134 @@ export class ZimService { logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`) } } + + // Custom library source management + + async listCustomLibraries(): Promise { + return CustomLibrarySource.all() + } + + async addCustomLibrary(name: string, baseUrl: string): Promise { + const count = await CustomLibrarySource.query().count('* as total') + const total = Number(count[0].$extras.total) + if (total >= 10) { + throw new Error('Maximum of 10 custom libraries allowed') + } + + // Ensure URL ends with / + const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/' + + return CustomLibrarySource.create({ + name, + base_url: normalizedUrl, + }) + } + + async removeCustomLibrary(id: number): Promise { + const source = await CustomLibrarySource.find(id) + if (!source) { + throw new Error('Custom library not found') + } + if (source.is_default) { + throw new Error('Cannot remove a built-in mirror') + } + await source.delete() + } + + async browseLibraryUrl(url: string): Promise<{ + directories: { name: string; url: string }[] + files: { name: string; url: string; size_bytes: number | null }[] + }> { + assertNotPrivateUrl(url) + + const normalizedUrl = url.endsWith('/') ? url : url + '/' + + const res = await axios.get(normalizedUrl, { + responseType: 'text', + timeout: 15000, + headers: { + 'Accept': 'text/html', + }, + }) + + const html: string = res.data + const directories: { name: string; url: string }[] = [] + const files: { name: string; url: string; size_bytes: number | null }[] = [] + + // Parse links from HTML directory listings + // Works with Apache, Nginx, and most HTTP directory indexes + const linkRegex = /]*href="([^"]+)"[^>]*>([^<]*)<\/a>/gi + let match: RegExpExecArray | null + + while ((match = linkRegex.exec(html)) !== null) { + const href = match[1] + + // Skip parent directory, self, sorting links, absolute paths, and absolute URLs + if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) { + continue + } + + // Skip absolute paths (e.g., /mirror/kiwix.org/) and absolute URLs + if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) { + continue + } + + // Directory (ends with /) + if (href.endsWith('/')) { + const dirName = decodeURIComponent(href.replace(/\/$/, '')) + directories.push({ + name: dirName, + url: normalizedUrl + href, + }) + continue + } + + // ZIM file + if (href.endsWith('.zim')) { + const fileName = decodeURIComponent(href) + const sizeBytes = this._extractSizeFromListing(html, href) + + files.push({ + name: fileName, + url: normalizedUrl + href, + size_bytes: sizeBytes, + }) + } + } + + // Sort directories alphabetically, files alphabetically + directories.sort((a, b) => a.name.localeCompare(b.name)) + files.sort((a, b) => a.name.localeCompare(b.name)) + + return { directories, files } + } + + /** + * Try to extract file size from HTML directory listing. + * Apache and Nginx directory listings typically show size near the filename. + * Returns bytes or null if not parseable. + */ + private _extractSizeFromListing(html: string, href: string): number | null { + // Apache style: file.zim 2024-01-15 10:30 5.1G + // Nginx style: file.zim 15-Jan-2024 10:30 5368709120 + const escapedHref = href.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const sizePattern = new RegExp( + escapedHref + `"[^<]*\\s+[\\d-]+\\s+[\\d:]+\\s+([\\d.]+[KMGT]?)\\b`, + 'i' + ) + const sizeMatch = sizePattern.exec(html) + if (!sizeMatch) return null + + const sizeStr = sizeMatch[1] + const num = parseFloat(sizeStr) + if (isNaN(num)) return null + + // If it's a plain number (Nginx shows raw bytes) + if (/^\d+$/.test(sizeStr)) return num + + // Apache uses K, M, G, T suffixes + const suffix = sizeStr.slice(-1).toUpperCase() + const multipliers: Record = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 } + return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null + } } diff --git a/admin/app/validators/zim.ts b/admin/app/validators/zim.ts index 2c18271..5463e52 100644 --- a/admin/app/validators/zim.ts +++ b/admin/app/validators/zim.ts @@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile( query: vine.string().optional(), }) ) + +export const addCustomLibraryValidator = vine.compile( + vine.object({ + name: vine.string().trim().minLength(1).maxLength(100), + base_url: vine + .string() + .url({ require_tld: false }) + .trim(), + }) +) + +export const browseLibraryValidator = vine.compile( + vine.object({ + url: vine + .string() + .url({ require_tld: false }) + .trim(), + }) +) + +export const idParamValidator = vine.compile( + vine.object({ + params: vine.object({ + id: vine.number(), + }), + }) +) diff --git a/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts new file mode 100644 index 0000000..baa3c77 --- /dev/null +++ b/admin/database/migrations/1775100000001_create_custom_library_sources_table.ts @@ -0,0 +1,42 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'custom_library_sources' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + table.string('name', 100).notNullable() + table.string('base_url', 2048).notNullable() + table.boolean('is_default').notNullable().defaultTo(false) + table.timestamp('created_at').notNullable() + table.timestamp('updated_at').notNullable() + }) + + // Seed default Kiwix mirrors + const now = new Date().toISOString().slice(0, 19).replace('T', ' ') + const defaults = [ + { name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' }, + { name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' }, + { name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' }, + { name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' }, + { name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' }, + ] + + for (const d of defaults) { + await this.defer(async (db) => { + await db.table(this.tableName).insert({ + name: d.name, + base_url: d.base_url, + is_default: true, + created_at: now, + updated_at: now, + }) + }) + } + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index a47f326..06c9908 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -536,6 +536,42 @@ class API { })() } + async listCustomLibraries() { + return catchInternal(async () => { + const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>( + '/zim/custom-libraries' + ) + return response.data + })() + } + + async addCustomLibrary(name: string, base_url: string) { + return catchInternal(async () => { + const response = await this.client.post<{ + message: string + library: { id: number; name: string; base_url: string } + }>('/zim/custom-libraries', { name, base_url }) + return response.data + })() + } + + async removeCustomLibrary(id: number) { + return catchInternal(async () => { + const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`) + return response.data + })() + } + + async browseLibrary(url: string) { + return catchInternal(async () => { + const response = await this.client.get<{ + directories: { name: string; url: string }[] + files: { name: string; url: string; size_bytes: number | null }[] + }>('/zim/browse-library', { params: { url } }) + return response.data + })() + } + async deleteZimFile(filename: string) { return catchInternal(async () => { const response = await this.client.delete<{ message: string }>(`/zim/${filename}`) diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index b515a50..bf0c42a 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus' import Alert from '~/components/Alert' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Input from '~/components/inputs/Input' -import { IconSearch, IconBooks } from '@tabler/icons-react' +import { + IconSearch, + IconBooks, + IconFolder, + IconFileDownload, + IconChevronRight, + IconPlus, + IconTrash, + IconLibrary, +} from '@tabler/icons-react' import useDebounce from '~/hooks/useDebounce' import CategoryCard from '~/components/CategoryCard' import TierSelectionModal from '~/components/TierSelectionModal' @@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names' const CURATED_CATEGORIES_KEY = 'curated-categories' const WIKIPEDIA_STATE_KEY = 'wikipedia-state' +const CUSTOM_LIBRARIES_KEY = 'custom-libraries' + +type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean } +type BrowseResult = { + directories: { name: string; url: string }[] + files: { name: string; url: string; size_bytes: number | null }[] +} export default function ZimRemoteExplorer() { const queryClient = useQueryClient() @@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() { const [selectedWikipedia, setSelectedWikipedia] = useState(null) const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false) + // Custom library state - persist selection to localStorage + const [selectedSource, setSelectedSource] = useState<'default' | number>(() => { + try { + const saved = localStorage.getItem('nomad:zim-library-source') + if (saved && saved !== 'default') return parseInt(saved, 10) + } catch {} + return 'default' + }) + const [browseUrl, setBrowseUrl] = useState(null) + const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([]) + const [manageModalOpen, setManageModalOpen] = useState(false) + const [newLibraryName, setNewLibraryName] = useState('') + const [newLibraryUrl, setNewLibraryUrl] = useState('') + const debouncedSetQuery = debounce((val: string) => { setQuery(val) }, 400) @@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() { enabled: true, }) + // Fetch custom libraries + const { data: customLibraries } = useQuery({ + queryKey: [CUSTOM_LIBRARIES_KEY], + queryFn: () => api.listCustomLibraries(), + refetchOnWindowFocus: false, + }) + + // Browse custom library directory + const { + data: browseData, + isLoading: isBrowsing, + error: browseError, + } = useQuery({ + queryKey: ['browse-library', browseUrl], + queryFn: () => api.browseLibrary(browseUrl!) as Promise, + enabled: !!browseUrl && selectedSource !== 'default', + refetchOnWindowFocus: false, + retry: false, + }) + const { data, fetchNextPage, isFetching, isLoading } = useInfiniteQuery({ queryKey: ['remote-zim-files', query], @@ -100,6 +150,7 @@ export default function ZimRemoteExplorer() { }, refetchOnWindowFocus: false, placeholderData: keepPreviousData, + enabled: selectedSource === 'default', }) const flatData = useMemo(() => { @@ -145,6 +196,50 @@ export default function ZimRemoteExplorer() { fetchOnBottomReached(tableParentRef.current) }, [fetchOnBottomReached]) + // Restore custom library selection on mount when data loads + useEffect(() => { + if (selectedSource !== 'default' && customLibraries) { + const lib = customLibraries.find((l) => l.id === selectedSource) + if (lib && !browseUrl) { + setBrowseUrl(lib.base_url) + setBreadcrumbs([{ name: lib.name, url: lib.base_url }]) + } else if (!lib) { + // Saved library was deleted + setSelectedSource('default') + localStorage.setItem('nomad:zim-library-source', 'default') + } + } + }, [customLibraries, selectedSource]) + + // When selecting a custom library, navigate to its root + const handleSourceChange = (value: string) => { + localStorage.setItem('nomad:zim-library-source', value) + if (value === 'default') { + setSelectedSource('default') + setBrowseUrl(null) + setBreadcrumbs([]) + } else { + const id = parseInt(value, 10) + const lib = customLibraries?.find((l) => l.id === id) + if (lib) { + setSelectedSource(id) + setBrowseUrl(lib.base_url) + setBreadcrumbs([{ name: lib.name, url: lib.base_url }]) + } + } + } + + const navigateToDirectory = (name: string, url: string) => { + setBrowseUrl(url) + setBreadcrumbs((prev) => [...prev, { name, url }]) + } + + const navigateToBreadcrumb = (index: number) => { + const crumb = breadcrumbs[index] + setBrowseUrl(crumb.url) + setBreadcrumbs((prev) => prev.slice(0, index + 1)) + } + async function confirmDownload(record: RemoteZimFileEntry) { openModal( { + downloadCustomFile(file) + closeAllModals() + }} + onCancel={closeAllModals} + open={true} + confirmText="Download" + cancelText="Cancel" + confirmVariant="primary" + > +

+ Are you sure you want to download{' '} + {file.name} + {file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix + application will be restarted after the download is complete. +

+ , + 'confirm-download-custom-modal' + ) + } + async function downloadFile(record: RemoteZimFileEntry) { try { await api.downloadRemoteZimFile(record.download_url, { @@ -184,6 +304,26 @@ export default function ZimRemoteExplorer() { } } + async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) { + try { + await api.downloadRemoteZimFile(file.url, { + title: file.name.replace(/\.zim$/, ''), + size_bytes: file.size_bytes ?? undefined, + }) + addNotification({ + message: `Started downloading "${file.name}"`, + type: 'success', + }) + invalidateDownloads() + } catch (error) { + console.error('Error downloading file:', error) + addNotification({ + message: 'Failed to start download.', + type: 'error', + }) + } + } + // Category/tier handlers const handleCategoryClick = (category: CategoryWithStatus) => { if (!isOnline) return @@ -269,6 +409,35 @@ export default function ZimRemoteExplorer() { }, }) + // Custom library management + const addLibraryMutation = useMutation({ + mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()), + onSuccess: () => { + addNotification({ message: 'Custom library added.', type: 'success' }) + queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] }) + setNewLibraryName('') + setNewLibraryUrl('') + }, + onError: () => { + addNotification({ message: 'Failed to add custom library.', type: 'error' }) + }, + }) + + const removeLibraryMutation = useMutation({ + mutationFn: (id: number) => api.removeCustomLibrary(id), + onSuccess: (_data, id) => { + addNotification({ message: 'Custom library removed.', type: 'success' }) + queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] }) + if (selectedSource === id) { + setSelectedSource('default') + setBrowseUrl(null) + setBreadcrumbs([]) + } + }, + }) + + const hasCustomLibraries = customLibraries && customLibraries.length > 0 + return ( @@ -307,7 +476,7 @@ export default function ZimRemoteExplorer() { Force Refresh Collections - + {/* Wikipedia Selector */} {isLoadingWikipedia ? (
@@ -365,87 +534,303 @@ export default function ZimRemoteExplorer() { ) : (

No curated content categories available.

)} - -
- { - setQueryUI(e.target.value) - debouncedSetQuery(e.target.value) - }} - className="w-1/3" - leftIcon={} - /> + + {/* Kiwix Library / Custom Library Browser */} +
+ + setManageModalOpen(true)} + disabled={!isOnline} + icon="IconLibrary" + > + {hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'} +
- - data={flatData.map((i, idx) => { - const row = virtualizer.getVirtualItems().find((v) => v.index === idx) - return { - ...i, - height: `${row?.size || 48}px`, // Use the size from the virtualizer - translateY: row?.start || 0, - } - })} - ref={tableParentRef} - loading={isLoading} - columns={[ - { - accessor: 'title', - }, - { - accessor: 'author', - }, - { - accessor: 'summary', - }, - { - accessor: 'updated', - render(record) { - return new Intl.DateTimeFormat('en-US', { - dateStyle: 'medium', - }).format(new Date(record.updated)) - }, - }, - { - accessor: 'size_bytes', - title: 'Size', - render(record) { - return formatBytes(record.size_bytes) - }, - }, - { - accessor: 'actions', - render(record) { - return ( -
- { - confirmDownload(record) - }} + + {/* Source selector dropdown */} + {hasCustomLibraries && ( +
+ + +
+ )} + + {/* Default Kiwix library browser */} + {selectedSource === 'default' && ( + <> +
+ { + setQueryUI(e.target.value) + debouncedSetQuery(e.target.value) + }} + className="w-1/3" + leftIcon={} + /> +
+ + data={flatData.map((i, idx) => { + const row = virtualizer.getVirtualItems().find((v) => v.index === idx) + return { + ...i, + height: `${row?.size || 48}px`, + translateY: row?.start || 0, + } + })} + ref={tableParentRef} + loading={isLoading} + columns={[ + { + accessor: 'title', + }, + { + accessor: 'author', + }, + { + accessor: 'summary', + }, + { + accessor: 'updated', + render(record) { + return new Intl.DateTimeFormat('en-US', { + dateStyle: 'medium', + }).format(new Date(record.updated)) + }, + }, + { + accessor: 'size_bytes', + title: 'Size', + render(record) { + return formatBytes(record.size_bytes) + }, + }, + { + accessor: 'actions', + render(record) { + return ( +
+ { + confirmDownload(record) + }} + > + Download + +
+ ) + }, + }, + ]} + className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" + tableBodyStyle={{ + position: 'relative', + height: `${virtualizer.getTotalSize()}px`, + }} + containerProps={{ + onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), + }} + compact + rowLines + /> + + )} + + {/* Custom library directory browser */} + {selectedSource !== 'default' && ( +
+ {/* Breadcrumb navigation */} +
- ) - }, - }, - ]} - className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4" - tableBodyStyle={{ - position: 'relative', - height: `${virtualizer.getTotalSize()}px`, - }} - containerProps={{ - onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement), - }} - compact - rowLines - /> + {crumb.name} + + ) : ( + {crumb.name} + )} + + ))} + + + {isBrowsing && ( +
+
+
+ )} + + {browseError && ( + + )} + + {!isBrowsing && !browseError && browseData && ( +
+ {browseData.directories.length === 0 && browseData.files.length === 0 ? ( +

+ No directories or ZIM files found at this location. +

+ ) : ( + + + + + + + + + + {browseData.directories.map((dir) => ( + navigateToDirectory(dir.name, dir.url)} + > + + + + + ))} + {browseData.files.map((file) => ( + + + + + + ))} + +
NameSize
+ + + {dir.name} + + -- + +
+ + + {file.name} + + + {file.size_bytes ? formatBytes(file.size_bytes) : '--'} + + confirmCustomDownload(file)} + > + Download + +
+ )} +
+ )} +
+ )} + + + {/* Manage Custom Libraries Modal */} + setManageModalOpen(false)} + cancelText="Close" + > +
+
+

+ Add Kiwix mirrors or other ZIM file sources for faster downloads. +

+ + {/* Existing libraries */} + {customLibraries && customLibraries.length > 0 && ( +
+ {customLibraries.map((lib) => ( +
+
+

+ {lib.name} + {lib.is_default && ( + (built-in) + )} +

+

{lib.base_url}

+
+ {!lib.is_default && ( + + )} +
+ ))} +
+ )} + + {/* Add new library form */} +
+ setNewLibraryName(e.target.value)} + /> + setNewLibraryUrl(e.target.value)} + /> + addLibraryMutation.mutate()} + disabled={ + !newLibraryName.trim() || + !newLibraryUrl.trim() || + addLibraryMutation.isPending + } + > + Add Library + +
+
+
+
diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 631c528..b140132 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -167,6 +167,12 @@ router router.get('/wikipedia', [ZimController, 'getWikipediaState']) router.post('/wikipedia/select', [ZimController, 'selectWikipedia']) + + router.get('/custom-libraries', [ZimController, 'listCustomLibraries']) + router.post('/custom-libraries', [ZimController, 'addCustomLibrary']) + router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary']) + router.get('/browse-library', [ZimController, 'browseLibrary']) + router.delete('/:filename', [ZimController, 'delete']) }) .prefix('/api/zim') From 38304d9012856285e4ebea8c480f136ab24fe686 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Tue, 31 Mar 2026 09:50:43 -0700 Subject: [PATCH 2/2] fix(content): recognize Wikipedia downloads from mirror sources When Wikipedia is downloaded via a custom mirror instead of the default Kiwix server, the completion callback now matches by filename instead of exact URL. This ensures the Wikipedia selector correctly shows "Installed" status and triggers old-version cleanup regardless of which mirror was used. Also handles the case where no Wikipedia selection exists yet (file downloaded before visiting the selector), creating the record automatically. Co-Authored-By: Claude Opus 4.6 (1M context) --- admin/app/services/zim_service.ts | 54 +++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 93e3203..1ec5e63 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -521,25 +521,47 @@ export class ZimService { } async onWikipediaDownloadComplete(url: string, success: boolean): Promise { + const filename = url.split('/').pop() || '' const selection = await this.getWikipediaSelection() - if (!selection || selection.url !== url) { - logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`) - return + // Determine which Wikipedia option this file belongs to by matching filename + let matchedOptionId: string | null = null + try { + const options = await this.getWikipediaOptions() + for (const opt of options) { + if (opt.url && opt.url.split('/').pop() === filename) { + matchedOptionId = opt.id + break + } + } + } catch { + // If we can't fetch options, try to continue with existing selection } if (success) { - // Update status to installed - selection.status = 'installed' - await selection.save() + // Update or create the selection record + // Match by filename (not URL) so mirror downloads are recognized + if (selection) { + selection.option_id = matchedOptionId || selection.option_id + selection.url = url + selection.filename = filename + selection.status = 'installed' + await selection.save() + } else { + await WikipediaSelection.create({ + option_id: matchedOptionId || 'unknown', + url: url, + filename: filename, + status: 'installed', + }) + } - logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`) + logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`) - // Delete the old Wikipedia file if it exists and is different - // We need to find what was previously installed + // Delete old Wikipedia files (keep only the newly installed one) const existingFiles = await this.list() const wikipediaFiles = existingFiles.files.filter((f) => - f.name.startsWith('wikipedia_en_') && f.name !== selection.filename + f.name.startsWith('wikipedia_en_') && f.name !== filename ) for (const oldFile of wikipediaFiles) { @@ -551,10 +573,14 @@ export class ZimService { } } } else { - // Download failed - keep the selection record but mark as failed - selection.status = 'failed' - await selection.save() - logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`) + // Download failed - update selection if it matches this file + if (selection && (!selection.filename || selection.filename === filename)) { + selection.status = 'failed' + await selection.save() + logger.error(`[ZimService] Wikipedia download failed for: ${filename}`) + } else { + logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`) + } } }