mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-05-12 16:10:11 +02:00
feat(Content): custom ZIM library sources with pre-seeded mirrors (#593)
* feat(content): add custom ZIM library sources with pre-seeded mirrors Users reported slow download speeds from the default Kiwix CDN. This adds the ability to browse and download ZIM files from alternative Kiwix mirrors or self-hosted repositories, all through the GUI. - Add "Custom Libraries" button next to "Browse the Kiwix Library" - Source dropdown to switch between Default (Kiwix) and custom libraries - Browsable directory structure with breadcrumb navigation - 5 pre-seeded official Kiwix mirrors (US, DE, DK, UK, Global CDN) - Built-in mirrors protected from deletion - Downloads use existing pipeline (progress, cancel, Kiwix restart) - Source selection persists across page loads via localStorage - Scrollable directory browser (600px max) with sticky header - SSRF protection on all custom library URLs Closes #576 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(content): recognize Wikipedia downloads from mirror sources When Wikipedia is downloaded via a custom mirror instead of the default Kiwix server, the completion callback now matches by filename instead of exact URL. This ensures the Wikipedia selector correctly shows "Installed" status and triggers old-version cleanup regardless of which mirror was used. Also handles the case where no Wikipedia selection exists yet (file downloaded before visiting the selector), creating the record automatically. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ZIM): use cheerio for custom mirror directory parsing * fix(ZIM): use URL constructor for more robust joining --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Jake Turner <jturner@cosmistack.com>
This commit is contained in:
chriscrosstalk
GitHub
parent
d66eaa3d42
commit
a7dbee55c4
|
|
@ -6,7 +6,7 @@ import {
|
|||
remoteDownloadWithMetadataValidator,
|
||||
selectWikipediaValidator,
|
||||
} from '#validators/common'
|
||||
import { listRemoteZimValidator } from '#validators/zim'
|
||||
import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
|
||||
import { inject } from '@adonisjs/core'
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
|
||||
|
|
@ -85,4 +85,51 @@ export default class ZimController {
|
|||
const payload = await request.validateUsing(selectWikipediaValidator)
|
||||
return this.zimService.selectWikipedia(payload.optionId)
|
||||
}
|
||||
|
||||
// Custom library endpoints
|
||||
|
||||
async listCustomLibraries({}: HttpContext) {
|
||||
return this.zimService.listCustomLibraries()
|
||||
}
|
||||
|
||||
async addCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(addCustomLibraryValidator)
|
||||
assertNotPrivateUrl(payload.base_url)
|
||||
try {
|
||||
const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
|
||||
return { message: 'Custom library added', library: source }
|
||||
} catch (error) {
|
||||
if (error.message === 'Maximum of 10 custom libraries allowed') {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async removeCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(idParamValidator)
|
||||
try {
|
||||
await this.zimService.removeCustomLibrary(payload.params.id)
|
||||
return { message: 'Custom library removed' }
|
||||
} catch (error) {
|
||||
if (error.message === 'Custom library not found') {
|
||||
return response.status(404).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async browseLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(browseLibraryValidator)
|
||||
try {
|
||||
return await this.zimService.browseLibraryUrl(payload.url)
|
||||
} catch (error) {
|
||||
if (error.message?.includes('loopback or link-local')) {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
return response.status(502).send({
|
||||
message: 'Could not fetch directory listing from the provided URL',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
24
admin/app/models/custom_library_source.ts
Normal file
24
admin/app/models/custom_library_source.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class CustomLibrarySource extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare name: string
|
||||
|
||||
@column()
|
||||
declare base_url: string
|
||||
|
||||
@column()
|
||||
declare is_default: boolean
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
RemoteZimFileEntry,
|
||||
} from '../../types/zim.js'
|
||||
import axios from 'axios'
|
||||
import * as cheerio from 'cheerio'
|
||||
import { XMLParser } from 'fast-xml-parser'
|
||||
import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
|
@ -27,6 +28,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js'
|
|||
import { CollectionManifestService } from './collection_manifest_service.js'
|
||||
import { KiwixLibraryService } from './kiwix_library_service.js'
|
||||
import type { CategoryWithStatus } from '../../types/collections.js'
|
||||
import CustomLibrarySource from '#models/custom_library_source'
|
||||
import { assertNotPrivateUrl } from '#validators/common'
|
||||
|
||||
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
|
||||
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
|
||||
|
|
@ -587,25 +590,47 @@ export class ZimService {
|
|||
}
|
||||
|
||||
async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
|
||||
const filename = url.split('/').pop() || ''
|
||||
const selection = await this.getWikipediaSelection()
|
||||
|
||||
if (!selection || selection.url !== url) {
|
||||
logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`)
|
||||
return
|
||||
// Determine which Wikipedia option this file belongs to by matching filename
|
||||
let matchedOptionId: string | null = null
|
||||
try {
|
||||
const options = await this.getWikipediaOptions()
|
||||
for (const opt of options) {
|
||||
if (opt.url && opt.url.split('/').pop() === filename) {
|
||||
matchedOptionId = opt.id
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// If we can't fetch options, try to continue with existing selection
|
||||
}
|
||||
|
||||
if (success) {
|
||||
// Update status to installed
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
// Update or create the selection record
|
||||
// Match by filename (not URL) so mirror downloads are recognized
|
||||
if (selection) {
|
||||
selection.option_id = matchedOptionId || selection.option_id
|
||||
selection.url = url
|
||||
selection.filename = filename
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
} else {
|
||||
await WikipediaSelection.create({
|
||||
option_id: matchedOptionId || 'unknown',
|
||||
url: url,
|
||||
filename: filename,
|
||||
status: 'installed',
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`)
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`)
|
||||
|
||||
// Delete the old Wikipedia file if it exists and is different
|
||||
// We need to find what was previously installed
|
||||
// Delete old Wikipedia files (keep only the newly installed one)
|
||||
const existingFiles = await this.list()
|
||||
const wikipediaFiles = existingFiles.files.filter((f) =>
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== selection.filename
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== filename
|
||||
)
|
||||
|
||||
for (const oldFile of wikipediaFiles) {
|
||||
|
|
@ -617,10 +642,137 @@ export class ZimService {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
// Download failed - keep the selection record but mark as failed
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`)
|
||||
// Download failed - update selection if it matches this file
|
||||
if (selection && (!selection.filename || selection.filename === filename)) {
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename}`)
|
||||
} else {
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Custom library source management
|
||||
|
||||
async listCustomLibraries(): Promise<CustomLibrarySource[]> {
|
||||
return CustomLibrarySource.all()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, baseUrl: string): Promise<CustomLibrarySource> {
|
||||
const count = await CustomLibrarySource.query().count('* as total')
|
||||
const total = Number(count[0].$extras.total)
|
||||
if (total >= 10) {
|
||||
throw new Error('Maximum of 10 custom libraries allowed')
|
||||
}
|
||||
|
||||
// Ensure URL ends with /
|
||||
const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'
|
||||
|
||||
return CustomLibrarySource.create({
|
||||
name,
|
||||
base_url: normalizedUrl,
|
||||
})
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number): Promise<void> {
|
||||
const source = await CustomLibrarySource.find(id)
|
||||
if (!source) {
|
||||
throw new Error('Custom library not found')
|
||||
}
|
||||
if (source.is_default) {
|
||||
throw new Error('Cannot remove a built-in mirror')
|
||||
}
|
||||
await source.delete()
|
||||
}
|
||||
|
||||
async browseLibraryUrl(url: string): Promise<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}> {
|
||||
assertNotPrivateUrl(url)
|
||||
|
||||
const normalizedUrl = url.endsWith('/') ? url : url + '/'
|
||||
|
||||
const res = await axios.get(normalizedUrl, {
|
||||
responseType: 'text',
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'Accept': 'text/html',
|
||||
},
|
||||
})
|
||||
|
||||
const html: string = res.data
|
||||
const directories: { name: string; url: string }[] = []
|
||||
const files: { name: string; url: string; size_bytes: number | null }[] = []
|
||||
|
||||
const $ = cheerio.load(html)
|
||||
|
||||
$('a').each((_, el) => {
|
||||
const href = el.attribs?.href
|
||||
if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) {
|
||||
return
|
||||
}
|
||||
if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) {
|
||||
return
|
||||
}
|
||||
|
||||
if (href.endsWith('/')) {
|
||||
const dirName = decodeURIComponent(href.replace(/\/$/, ''))
|
||||
directories.push({
|
||||
name: dirName,
|
||||
url: new URL(href, normalizedUrl).toString(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (href.endsWith('.zim')) {
|
||||
const fileName = decodeURIComponent(href)
|
||||
|
||||
// Apache/Nginx autoindex put the date + size in the text node directly
|
||||
// following </a> within a <pre>. Walk forward across text siblings until
|
||||
// we find a parseable size token.
|
||||
let trailingText = ''
|
||||
let sibling = el.next
|
||||
while (sibling && sibling.type === 'text') {
|
||||
trailingText += sibling.data
|
||||
if (/\n/.test(sibling.data)) break
|
||||
sibling = sibling.next
|
||||
}
|
||||
|
||||
files.push({
|
||||
name: fileName,
|
||||
url: new URL(href, normalizedUrl).toString(),
|
||||
size_bytes: this._parseListingSize(trailingText),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
directories.sort((a, b) => a.name.localeCompare(b.name))
|
||||
files.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
return { directories, files }
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a directory-listing size token out of the text that follows an anchor.
|
||||
* Apache renders e.g. ` 2024-01-15 10:30 5.1G`; Nginx renders raw bytes.
|
||||
* Returns bytes or null if no size token is found.
|
||||
*/
|
||||
private _parseListingSize(text: string): number | null {
|
||||
// Skip the date/time columns; grab the last numeric token (with optional suffix)
|
||||
// before a newline. Matches `5.1G`, `5368709120`, `1.2T`, etc.
|
||||
const sizeMatch = /([\d.]+\s*[KMGT]?B?|\d+)\s*$/i.exec(text.split('\n')[0].trim())
|
||||
if (!sizeMatch) return null
|
||||
|
||||
const sizeStr = sizeMatch[1].replace(/\s|B$/gi, '')
|
||||
const num = parseFloat(sizeStr)
|
||||
if (isNaN(num)) return null
|
||||
|
||||
if (/^\d+$/.test(sizeStr)) return num
|
||||
|
||||
const suffix = sizeStr.slice(-1).toUpperCase()
|
||||
const multipliers: Record<string, number> = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
|
||||
return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
|
|||
query: vine.string().optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const addCustomLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
name: vine.string().trim().minLength(1).maxLength(100),
|
||||
base_url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const browseLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const idParamValidator = vine.compile(
|
||||
vine.object({
|
||||
params: vine.object({
|
||||
id: vine.number(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'custom_library_sources'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('name', 100).notNullable()
|
||||
table.string('base_url', 2048).notNullable()
|
||||
table.boolean('is_default').notNullable().defaultTo(false)
|
||||
table.timestamp('created_at').notNullable()
|
||||
table.timestamp('updated_at').notNullable()
|
||||
})
|
||||
|
||||
// Seed default Kiwix mirrors
|
||||
const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
|
||||
const defaults = [
|
||||
{ name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
|
||||
{ name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
|
||||
{ name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
|
||||
{ name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
|
||||
{ name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
|
||||
]
|
||||
|
||||
for (const d of defaults) {
|
||||
await this.defer(async (db) => {
|
||||
await db.table(this.tableName).insert({
|
||||
name: d.name,
|
||||
base_url: d.base_url,
|
||||
is_default: true,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
|
|
@ -681,6 +681,42 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async listCustomLibraries() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
|
||||
'/zim/custom-libraries'
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, base_url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
message: string
|
||||
library: { id: number; name: string; base_url: string }
|
||||
}>('/zim/custom-libraries', { name, base_url })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async browseLibrary(url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}>('/zim/browse-library', { params: { url } })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async deleteZimFile(filename: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
|
|||
import Alert from '~/components/Alert'
|
||||
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||
import Input from '~/components/inputs/Input'
|
||||
import { IconSearch, IconBooks } from '@tabler/icons-react'
|
||||
import {
|
||||
IconSearch,
|
||||
IconBooks,
|
||||
IconFolder,
|
||||
IconFileDownload,
|
||||
IconChevronRight,
|
||||
IconPlus,
|
||||
IconTrash,
|
||||
IconLibrary,
|
||||
} from '@tabler/icons-react'
|
||||
import useDebounce from '~/hooks/useDebounce'
|
||||
import CategoryCard from '~/components/CategoryCard'
|
||||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
|
|
@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
|
|||
|
||||
const CURATED_CATEGORIES_KEY = 'curated-categories'
|
||||
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
|
||||
const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
|
||||
|
||||
type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
|
||||
type BrowseResult = {
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}
|
||||
|
||||
export default function ZimRemoteExplorer() {
|
||||
const queryClient = useQueryClient()
|
||||
|
|
@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
|
|||
const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
|
||||
const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
|
||||
|
||||
// Custom library state - persist selection to localStorage
|
||||
const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem('nomad:zim-library-source')
|
||||
if (saved && saved !== 'default') return parseInt(saved, 10)
|
||||
} catch {}
|
||||
return 'default'
|
||||
})
|
||||
const [browseUrl, setBrowseUrl] = useState<string | null>(null)
|
||||
const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
|
||||
const [manageModalOpen, setManageModalOpen] = useState(false)
|
||||
const [newLibraryName, setNewLibraryName] = useState('')
|
||||
const [newLibraryUrl, setNewLibraryUrl] = useState('')
|
||||
|
||||
const debouncedSetQuery = debounce((val: string) => {
|
||||
setQuery(val)
|
||||
}, 400)
|
||||
|
|
@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
|
|||
enabled: true,
|
||||
})
|
||||
|
||||
// Fetch custom libraries
|
||||
const { data: customLibraries } = useQuery({
|
||||
queryKey: [CUSTOM_LIBRARIES_KEY],
|
||||
queryFn: () => api.listCustomLibraries(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// Browse custom library directory
|
||||
const {
|
||||
data: browseData,
|
||||
isLoading: isBrowsing,
|
||||
error: browseError,
|
||||
} = useQuery<BrowseResult>({
|
||||
queryKey: ['browse-library', browseUrl],
|
||||
queryFn: () => api.browseLibrary(browseUrl!) as Promise<BrowseResult>,
|
||||
enabled: !!browseUrl && selectedSource !== 'default',
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
})
|
||||
|
||||
const { data, fetchNextPage, isFetching, isLoading } =
|
||||
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
||||
queryKey: ['remote-zim-files', query],
|
||||
|
|
@ -97,6 +147,7 @@ export default function ZimRemoteExplorer() {
|
|||
getNextPageParam: (lastPage) => (lastPage.has_more ? lastPage.next_start : undefined),
|
||||
refetchOnWindowFocus: false,
|
||||
placeholderData: keepPreviousData,
|
||||
enabled: selectedSource === 'default',
|
||||
})
|
||||
|
||||
const flatData = useMemo(() => {
|
||||
|
|
@ -140,6 +191,50 @@ export default function ZimRemoteExplorer() {
|
|||
fetchOnBottomReached(tableParentRef.current)
|
||||
}, [fetchOnBottomReached])
|
||||
|
||||
// Restore custom library selection on mount when data loads
|
||||
useEffect(() => {
|
||||
if (selectedSource !== 'default' && customLibraries) {
|
||||
const lib = customLibraries.find((l) => l.id === selectedSource)
|
||||
if (lib && !browseUrl) {
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
} else if (!lib) {
|
||||
// Saved library was deleted
|
||||
setSelectedSource('default')
|
||||
localStorage.setItem('nomad:zim-library-source', 'default')
|
||||
}
|
||||
}
|
||||
}, [customLibraries, selectedSource])
|
||||
|
||||
// When selecting a custom library, navigate to its root
|
||||
const handleSourceChange = (value: string) => {
|
||||
localStorage.setItem('nomad:zim-library-source', value)
|
||||
if (value === 'default') {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
} else {
|
||||
const id = parseInt(value, 10)
|
||||
const lib = customLibraries?.find((l) => l.id === id)
|
||||
if (lib) {
|
||||
setSelectedSource(id)
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const navigateToDirectory = (name: string, url: string) => {
|
||||
setBrowseUrl(url)
|
||||
setBreadcrumbs((prev) => [...prev, { name, url }])
|
||||
}
|
||||
|
||||
const navigateToBreadcrumb = (index: number) => {
|
||||
const crumb = breadcrumbs[index]
|
||||
setBrowseUrl(crumb.url)
|
||||
setBreadcrumbs((prev) => prev.slice(0, index + 1))
|
||||
}
|
||||
|
||||
async function confirmDownload(record: RemoteZimFileEntry) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
|
|
@ -165,6 +260,31 @@ export default function ZimRemoteExplorer() {
|
|||
)
|
||||
}
|
||||
|
||||
async function confirmCustomDownload(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
title="Confirm Download?"
|
||||
onConfirm={() => {
|
||||
downloadCustomFile(file)
|
||||
closeAllModals()
|
||||
}}
|
||||
onCancel={closeAllModals}
|
||||
open={true}
|
||||
confirmText="Download"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="primary"
|
||||
>
|
||||
<p className="text-text-primary">
|
||||
Are you sure you want to download{' '}
|
||||
<strong>{file.name}</strong>
|
||||
{file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix
|
||||
application will be restarted after the download is complete.
|
||||
</p>
|
||||
</StyledModal>,
|
||||
'confirm-download-custom-modal'
|
||||
)
|
||||
}
|
||||
|
||||
async function downloadFile(record: RemoteZimFileEntry) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(record.download_url, {
|
||||
|
|
@ -179,6 +299,26 @@ export default function ZimRemoteExplorer() {
|
|||
}
|
||||
}
|
||||
|
||||
async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(file.url, {
|
||||
title: file.name.replace(/\.zim$/, ''),
|
||||
size_bytes: file.size_bytes ?? undefined,
|
||||
})
|
||||
addNotification({
|
||||
message: `Started downloading "${file.name}"`,
|
||||
type: 'success',
|
||||
})
|
||||
invalidateDownloads()
|
||||
} catch (error) {
|
||||
console.error('Error downloading file:', error)
|
||||
addNotification({
|
||||
message: 'Failed to start download.',
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Category/tier handlers
|
||||
const handleCategoryClick = (category: CategoryWithStatus) => {
|
||||
if (!isOnline) return
|
||||
|
|
@ -264,6 +404,35 @@ export default function ZimRemoteExplorer() {
|
|||
},
|
||||
})
|
||||
|
||||
// Custom library management
|
||||
const addLibraryMutation = useMutation({
|
||||
mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()),
|
||||
onSuccess: () => {
|
||||
addNotification({ message: 'Custom library added.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
setNewLibraryName('')
|
||||
setNewLibraryUrl('')
|
||||
},
|
||||
onError: () => {
|
||||
addNotification({ message: 'Failed to add custom library.', type: 'error' })
|
||||
},
|
||||
})
|
||||
|
||||
const removeLibraryMutation = useMutation({
|
||||
mutationFn: (id: number) => api.removeCustomLibrary(id),
|
||||
onSuccess: (_data, id) => {
|
||||
addNotification({ message: 'Custom library removed.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
if (selectedSource === id) {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const hasCustomLibraries = customLibraries && customLibraries.length > 0
|
||||
|
||||
return (
|
||||
<SettingsLayout>
|
||||
<Head title="Content Explorer | Project N.O.M.A.D." />
|
||||
|
|
@ -302,7 +471,7 @@ export default function ZimRemoteExplorer() {
|
|||
Force Refresh Collections
|
||||
</StyledButton>
|
||||
</div>
|
||||
|
||||
|
||||
{/* Wikipedia Selector */}
|
||||
{isLoadingWikipedia ? (
|
||||
<div className="mt-8 bg-surface-primary rounded-lg border border-border-subtle p-6">
|
||||
|
|
@ -360,87 +529,303 @@ export default function ZimRemoteExplorer() {
|
|||
) : (
|
||||
<p className="text-text-muted mt-4">No curated content categories available.</p>
|
||||
)}
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="mt-12 mb-4" />
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
|
||||
{/* Kiwix Library / Custom Library Browser */}
|
||||
<div className="mt-12 mb-4 flex items-center justify-between">
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="!mb-0" />
|
||||
<StyledButton
|
||||
onClick={() => setManageModalOpen(true)}
|
||||
disabled={!isOnline}
|
||||
icon="IconLibrary"
|
||||
>
|
||||
{hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'}
|
||||
</StyledButton>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`, // Use the size from the virtualizer
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
|
||||
{/* Source selector dropdown */}
|
||||
{hasCustomLibraries && (
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<label className="text-sm font-medium text-text-secondary">Source:</label>
|
||||
<select
|
||||
value={selectedSource === 'default' ? 'default' : String(selectedSource)}
|
||||
onChange={(e) => handleSourceChange(e.target.value)}
|
||||
className="rounded-md border border-border-default bg-surface-primary text-text-primary px-3 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
|
||||
>
|
||||
<option value="default">Default (Kiwix)</option>
|
||||
{customLibraries.map((lib) => (
|
||||
<option key={lib.id} value={String(lib.id)}>
|
||||
{lib.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Default Kiwix library browser */}
|
||||
{selectedSource === 'default' && (
|
||||
<>
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`,
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Custom library directory browser */}
|
||||
{selectedSource !== 'default' && (
|
||||
<div className="mt-4">
|
||||
{/* Breadcrumb navigation */}
|
||||
<nav className="flex items-center gap-1 text-sm text-text-muted mb-4 flex-wrap">
|
||||
{breadcrumbs.map((crumb, idx) => (
|
||||
<span key={idx} className="flex items-center gap-1">
|
||||
{idx > 0 && <IconChevronRight className="w-4 h-4" />}
|
||||
{idx < breadcrumbs.length - 1 ? (
|
||||
<button
|
||||
onClick={() => navigateToBreadcrumb(idx)}
|
||||
className="text-desert-green hover:underline"
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
{crumb.name}
|
||||
</button>
|
||||
) : (
|
||||
<span className="text-text-primary font-medium">{crumb.name}</span>
|
||||
)}
|
||||
</span>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
{isBrowsing && (
|
||||
<div className="flex justify-center py-12">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-desert-green"></div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{browseError && (
|
||||
<Alert
|
||||
title="Could not fetch directory listing from this URL."
|
||||
message="The server may not support directory browsing, or the URL may be incorrect."
|
||||
type="error"
|
||||
variant="solid"
|
||||
/>
|
||||
)}
|
||||
|
||||
{!isBrowsing && !browseError && browseData && (
|
||||
<div className="bg-surface-primary rounded-lg border border-border-subtle overflow-hidden relative" style={{ maxHeight: '600px', overflowY: 'auto' }}>
|
||||
{browseData.directories.length === 0 && browseData.files.length === 0 ? (
|
||||
<p className="text-text-muted p-6 text-center">
|
||||
No directories or ZIM files found at this location.
|
||||
</p>
|
||||
) : (
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-border-subtle bg-surface-secondary sticky top-0 z-10">
|
||||
<th className="text-left px-4 py-3 font-medium text-text-secondary">Name</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-32">Size</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-36"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{browseData.directories.map((dir) => (
|
||||
<tr
|
||||
key={dir.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary cursor-pointer transition-colors"
|
||||
onClick={() => navigateToDirectory(dir.name, dir.url)}
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFolder className="w-5 h-5 text-desert-orange" />
|
||||
{dir.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">--</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<IconChevronRight className="w-4 h-4 text-text-muted ml-auto" />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{browseData.files.map((file) => (
|
||||
<tr
|
||||
key={file.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary transition-colors"
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFileDownload className="w-5 h-5 text-desert-green" />
|
||||
{file.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">
|
||||
{file.size_bytes ? formatBytes(file.size_bytes) : '--'}
|
||||
</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<StyledButton
|
||||
icon="IconDownload"
|
||||
onClick={() => confirmCustomDownload(file)}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ActiveDownloads filetype="zim" withHeader />
|
||||
|
||||
{/* Manage Custom Libraries Modal */}
|
||||
<StyledModal
|
||||
title="Manage Custom Libraries"
|
||||
open={manageModalOpen}
|
||||
onCancel={() => setManageModalOpen(false)}
|
||||
cancelText="Close"
|
||||
>
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<p className="text-sm text-text-muted mb-4">
|
||||
Add Kiwix mirrors or other ZIM file sources for faster downloads.
|
||||
</p>
|
||||
|
||||
{/* Existing libraries */}
|
||||
{customLibraries && customLibraries.length > 0 && (
|
||||
<div className="space-y-2 mb-6">
|
||||
{customLibraries.map((lib) => (
|
||||
<div
|
||||
key={lib.id}
|
||||
className="flex items-center justify-between bg-surface-secondary rounded-lg px-4 py-3 border border-border-subtle"
|
||||
>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="font-medium text-text-primary truncate">
|
||||
{lib.name}
|
||||
{lib.is_default && (
|
||||
<span className="ml-2 text-xs text-text-muted font-normal">(built-in)</span>
|
||||
)}
|
||||
</p>
|
||||
<p className="text-xs text-text-muted truncate">{lib.base_url}</p>
|
||||
</div>
|
||||
{!lib.is_default && (
|
||||
<button
|
||||
onClick={() => removeLibraryMutation.mutate(lib.id)}
|
||||
className="ml-3 p-1.5 text-text-muted hover:text-red-500 transition-colors rounded"
|
||||
title="Remove library"
|
||||
>
|
||||
<IconTrash className="w-4 h-4" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Add new library form */}
|
||||
<div className="space-y-3">
|
||||
<Input
|
||||
name="library-name"
|
||||
label="Library Name"
|
||||
placeholder="e.g., Debian Mirror"
|
||||
value={newLibraryName}
|
||||
onChange={(e) => setNewLibraryName(e.target.value)}
|
||||
/>
|
||||
<Input
|
||||
name="library-url"
|
||||
label="Base URL"
|
||||
placeholder="e.g., https://cdimage.debian.org/mirror/kiwix.org/zim/"
|
||||
value={newLibraryUrl}
|
||||
onChange={(e) => setNewLibraryUrl(e.target.value)}
|
||||
/>
|
||||
<StyledButton
|
||||
icon="IconPlus"
|
||||
onClick={() => addLibraryMutation.mutate()}
|
||||
disabled={
|
||||
!newLibraryName.trim() ||
|
||||
!newLibraryUrl.trim() ||
|
||||
addLibraryMutation.isPending
|
||||
}
|
||||
>
|
||||
Add Library
|
||||
</StyledButton>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</StyledModal>
|
||||
</main>
|
||||
</div>
|
||||
</SettingsLayout>
|
||||
|
|
|
|||
|
|
@ -183,6 +183,12 @@ router
|
|||
|
||||
router.get('/wikipedia', [ZimController, 'getWikipediaState'])
|
||||
router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])
|
||||
|
||||
router.get('/custom-libraries', [ZimController, 'listCustomLibraries'])
|
||||
router.post('/custom-libraries', [ZimController, 'addCustomLibrary'])
|
||||
router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary'])
|
||||
router.get('/browse-library', [ZimController, 'browseLibrary'])
|
||||
|
||||
router.delete('/:filename', [ZimController, 'delete'])
|
||||
})
|
||||
.prefix('/api/zim')
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user