mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-04 15:56:16 +02:00
Merge 38304d9012 into d666b24598
This commit is contained in:
chriscrosstalk
GitHub
commit
878af385b7
|
|
@ -6,7 +6,7 @@ import {
|
|||
remoteDownloadWithMetadataValidator,
|
||||
selectWikipediaValidator,
|
||||
} from '#validators/common'
|
||||
import { listRemoteZimValidator } from '#validators/zim'
|
||||
import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
|
||||
import { inject } from '@adonisjs/core'
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
|
||||
|
|
@ -85,4 +85,51 @@ export default class ZimController {
|
|||
const payload = await request.validateUsing(selectWikipediaValidator)
|
||||
return this.zimService.selectWikipedia(payload.optionId)
|
||||
}
|
||||
|
||||
// Custom library endpoints
|
||||
|
||||
async listCustomLibraries({}: HttpContext) {
|
||||
return this.zimService.listCustomLibraries()
|
||||
}
|
||||
|
||||
async addCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(addCustomLibraryValidator)
|
||||
assertNotPrivateUrl(payload.base_url)
|
||||
try {
|
||||
const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
|
||||
return { message: 'Custom library added', library: source }
|
||||
} catch (error) {
|
||||
if (error.message === 'Maximum of 10 custom libraries allowed') {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async removeCustomLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(idParamValidator)
|
||||
try {
|
||||
await this.zimService.removeCustomLibrary(payload.params.id)
|
||||
return { message: 'Custom library removed' }
|
||||
} catch (error) {
|
||||
if (error.message === 'Custom library not found') {
|
||||
return response.status(404).send({ message: error.message })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async browseLibrary({ request, response }: HttpContext) {
|
||||
const payload = await request.validateUsing(browseLibraryValidator)
|
||||
try {
|
||||
return await this.zimService.browseLibraryUrl(payload.url)
|
||||
} catch (error) {
|
||||
if (error.message?.includes('loopback or link-local')) {
|
||||
return response.status(400).send({ message: error.message })
|
||||
}
|
||||
return response.status(502).send({
|
||||
message: 'Could not fetch directory listing from the provided URL',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
24
admin/app/models/custom_library_source.ts
Normal file
24
admin/app/models/custom_library_source.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { DateTime } from 'luxon'
|
||||
import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class CustomLibrarySource extends BaseModel {
|
||||
static namingStrategy = new SnakeCaseNamingStrategy()
|
||||
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare name: string
|
||||
|
||||
@column()
|
||||
declare base_url: string
|
||||
|
||||
@column()
|
||||
declare is_default: boolean
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare created_at: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updated_at: DateTime
|
||||
}
|
||||
|
|
@ -27,6 +27,8 @@ import { SERVICE_NAMES } from '../../constants/service_names.js'
|
|||
import { CollectionManifestService } from './collection_manifest_service.js'
|
||||
import { KiwixLibraryService } from './kiwix_library_service.js'
|
||||
import type { CategoryWithStatus } from '../../types/collections.js'
|
||||
import CustomLibrarySource from '#models/custom_library_source'
|
||||
import { assertNotPrivateUrl } from '#validators/common'
|
||||
|
||||
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
|
||||
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
|
||||
|
|
@ -552,25 +554,47 @@ export class ZimService {
|
|||
}
|
||||
|
||||
async onWikipediaDownloadComplete(url: string, success: boolean): Promise<void> {
|
||||
const filename = url.split('/').pop() || ''
|
||||
const selection = await this.getWikipediaSelection()
|
||||
|
||||
if (!selection || selection.url !== url) {
|
||||
logger.warn(`[ZimService] Wikipedia download complete callback for unknown URL: ${url}`)
|
||||
return
|
||||
// Determine which Wikipedia option this file belongs to by matching filename
|
||||
let matchedOptionId: string | null = null
|
||||
try {
|
||||
const options = await this.getWikipediaOptions()
|
||||
for (const opt of options) {
|
||||
if (opt.url && opt.url.split('/').pop() === filename) {
|
||||
matchedOptionId = opt.id
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// If we can't fetch options, try to continue with existing selection
|
||||
}
|
||||
|
||||
if (success) {
|
||||
// Update status to installed
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
// Update or create the selection record
|
||||
// Match by filename (not URL) so mirror downloads are recognized
|
||||
if (selection) {
|
||||
selection.option_id = matchedOptionId || selection.option_id
|
||||
selection.url = url
|
||||
selection.filename = filename
|
||||
selection.status = 'installed'
|
||||
await selection.save()
|
||||
} else {
|
||||
await WikipediaSelection.create({
|
||||
option_id: matchedOptionId || 'unknown',
|
||||
url: url,
|
||||
filename: filename,
|
||||
status: 'installed',
|
||||
})
|
||||
}
|
||||
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${selection.filename}`)
|
||||
logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`)
|
||||
|
||||
// Delete the old Wikipedia file if it exists and is different
|
||||
// We need to find what was previously installed
|
||||
// Delete old Wikipedia files (keep only the newly installed one)
|
||||
const existingFiles = await this.list()
|
||||
const wikipediaFiles = existingFiles.files.filter((f) =>
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== selection.filename
|
||||
f.name.startsWith('wikipedia_en_') && f.name !== filename
|
||||
)
|
||||
|
||||
for (const oldFile of wikipediaFiles) {
|
||||
|
|
@ -582,10 +606,144 @@ export class ZimService {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
// Download failed - keep the selection record but mark as failed
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${selection.filename}`)
|
||||
// Download failed - update selection if it matches this file
|
||||
if (selection && (!selection.filename || selection.filename === filename)) {
|
||||
selection.status = 'failed'
|
||||
await selection.save()
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename}`)
|
||||
} else {
|
||||
logger.error(`[ZimService] Wikipedia download failed for: ${filename} (no matching selection)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Custom library source management
|
||||
|
||||
async listCustomLibraries(): Promise<CustomLibrarySource[]> {
|
||||
return CustomLibrarySource.all()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, baseUrl: string): Promise<CustomLibrarySource> {
|
||||
const count = await CustomLibrarySource.query().count('* as total')
|
||||
const total = Number(count[0].$extras.total)
|
||||
if (total >= 10) {
|
||||
throw new Error('Maximum of 10 custom libraries allowed')
|
||||
}
|
||||
|
||||
// Ensure URL ends with /
|
||||
const normalizedUrl = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'
|
||||
|
||||
return CustomLibrarySource.create({
|
||||
name,
|
||||
base_url: normalizedUrl,
|
||||
})
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number): Promise<void> {
|
||||
const source = await CustomLibrarySource.find(id)
|
||||
if (!source) {
|
||||
throw new Error('Custom library not found')
|
||||
}
|
||||
if (source.is_default) {
|
||||
throw new Error('Cannot remove a built-in mirror')
|
||||
}
|
||||
await source.delete()
|
||||
}
|
||||
|
||||
async browseLibraryUrl(url: string): Promise<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}> {
|
||||
assertNotPrivateUrl(url)
|
||||
|
||||
const normalizedUrl = url.endsWith('/') ? url : url + '/'
|
||||
|
||||
const res = await axios.get(normalizedUrl, {
|
||||
responseType: 'text',
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'Accept': 'text/html',
|
||||
},
|
||||
})
|
||||
|
||||
const html: string = res.data
|
||||
const directories: { name: string; url: string }[] = []
|
||||
const files: { name: string; url: string; size_bytes: number | null }[] = []
|
||||
|
||||
// Parse <a href="..."> links from HTML directory listings
|
||||
// Works with Apache, Nginx, and most HTTP directory indexes
|
||||
const linkRegex = /<a\s+[^>]*href="([^"]+)"[^>]*>([^<]*)<\/a>/gi
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = linkRegex.exec(html)) !== null) {
|
||||
const href = match[1]
|
||||
|
||||
// Skip parent directory, self, sorting links, absolute paths, and absolute URLs
|
||||
if (!href || href === '../' || href === './' || href === '/' || href.startsWith('?') || href.startsWith('#')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip absolute paths (e.g., /mirror/kiwix.org/) and absolute URLs
|
||||
if (href.startsWith('/') || href.startsWith('http://') || href.startsWith('https://')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Directory (ends with /)
|
||||
if (href.endsWith('/')) {
|
||||
const dirName = decodeURIComponent(href.replace(/\/$/, ''))
|
||||
directories.push({
|
||||
name: dirName,
|
||||
url: normalizedUrl + href,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// ZIM file
|
||||
if (href.endsWith('.zim')) {
|
||||
const fileName = decodeURIComponent(href)
|
||||
const sizeBytes = this._extractSizeFromListing(html, href)
|
||||
|
||||
files.push({
|
||||
name: fileName,
|
||||
url: normalizedUrl + href,
|
||||
size_bytes: sizeBytes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort directories alphabetically, files alphabetically
|
||||
directories.sort((a, b) => a.name.localeCompare(b.name))
|
||||
files.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
return { directories, files }
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to extract file size from HTML directory listing.
|
||||
* Apache and Nginx directory listings typically show size near the filename.
|
||||
* Returns bytes or null if not parseable.
|
||||
*/
|
||||
private _extractSizeFromListing(html: string, href: string): number | null {
|
||||
// Apache style: <a href="file.zim">file.zim</a> 2024-01-15 10:30 5.1G
|
||||
// Nginx style: <a href="file.zim">file.zim</a> 15-Jan-2024 10:30 5368709120
|
||||
const escapedHref = href.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const sizePattern = new RegExp(
|
||||
escapedHref + `"[^<]*</a>\\s+[\\d-]+\\s+[\\d:]+\\s+([\\d.]+[KMGT]?)\\b`,
|
||||
'i'
|
||||
)
|
||||
const sizeMatch = sizePattern.exec(html)
|
||||
if (!sizeMatch) return null
|
||||
|
||||
const sizeStr = sizeMatch[1]
|
||||
const num = parseFloat(sizeStr)
|
||||
if (isNaN(num)) return null
|
||||
|
||||
// If it's a plain number (Nginx shows raw bytes)
|
||||
if (/^\d+$/.test(sizeStr)) return num
|
||||
|
||||
// Apache uses K, M, G, T suffixes
|
||||
const suffix = sizeStr.slice(-1).toUpperCase()
|
||||
const multipliers: Record<string, number> = { K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }
|
||||
return multipliers[suffix] ? Math.round(num * multipliers[suffix]) : null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,3 +7,30 @@ export const listRemoteZimValidator = vine.compile(
|
|||
query: vine.string().optional(),
|
||||
})
|
||||
)
|
||||
|
||||
export const addCustomLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
name: vine.string().trim().minLength(1).maxLength(100),
|
||||
base_url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const browseLibraryValidator = vine.compile(
|
||||
vine.object({
|
||||
url: vine
|
||||
.string()
|
||||
.url({ require_tld: false })
|
||||
.trim(),
|
||||
})
|
||||
)
|
||||
|
||||
export const idParamValidator = vine.compile(
|
||||
vine.object({
|
||||
params: vine.object({
|
||||
id: vine.number(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'custom_library_sources'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('name', 100).notNullable()
|
||||
table.string('base_url', 2048).notNullable()
|
||||
table.boolean('is_default').notNullable().defaultTo(false)
|
||||
table.timestamp('created_at').notNullable()
|
||||
table.timestamp('updated_at').notNullable()
|
||||
})
|
||||
|
||||
// Seed default Kiwix mirrors
|
||||
const now = new Date().toISOString().slice(0, 19).replace('T', ' ')
|
||||
const defaults = [
|
||||
{ name: 'Debian CDN (Global)', base_url: 'https://cdimage.debian.org/mirror/kiwix.org/zim/' },
|
||||
{ name: 'Your.org (US)', base_url: 'https://ftpmirror.your.org/pub/kiwix/zim/' },
|
||||
{ name: 'FAU Erlangen (DE)', base_url: 'https://ftp.fau.de/kiwix/zim/' },
|
||||
{ name: 'Dotsrc (DK)', base_url: 'https://mirrors.dotsrc.org/kiwix/zim/' },
|
||||
{ name: 'MirrorService (UK)', base_url: 'https://www.mirrorservice.org/sites/download.kiwix.org/zim/' },
|
||||
]
|
||||
|
||||
for (const d of defaults) {
|
||||
await this.defer(async (db) => {
|
||||
await db.table(this.tableName).insert({
|
||||
name: d.name,
|
||||
base_url: d.base_url,
|
||||
is_default: true,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
|
|
@ -624,6 +624,42 @@ class API {
|
|||
})()
|
||||
}
|
||||
|
||||
async listCustomLibraries() {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>(
|
||||
'/zim/custom-libraries'
|
||||
)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async addCustomLibrary(name: string, base_url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.post<{
|
||||
message: string
|
||||
library: { id: number; name: string; base_url: string }
|
||||
}>('/zim/custom-libraries', { name, base_url })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async removeCustomLibrary(id: number) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/custom-libraries/${id}`)
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async browseLibrary(url: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.get<{
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}>('/zim/browse-library', { params: { url } })
|
||||
return response.data
|
||||
})()
|
||||
}
|
||||
|
||||
async deleteZimFile(filename: string) {
|
||||
return catchInternal(async () => {
|
||||
const response = await this.client.delete<{ message: string }>(`/zim/${filename}`)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,16 @@ import useInternetStatus from '~/hooks/useInternetStatus'
|
|||
import Alert from '~/components/Alert'
|
||||
import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
|
||||
import Input from '~/components/inputs/Input'
|
||||
import { IconSearch, IconBooks } from '@tabler/icons-react'
|
||||
import {
|
||||
IconSearch,
|
||||
IconBooks,
|
||||
IconFolder,
|
||||
IconFileDownload,
|
||||
IconChevronRight,
|
||||
IconPlus,
|
||||
IconTrash,
|
||||
IconLibrary,
|
||||
} from '@tabler/icons-react'
|
||||
import useDebounce from '~/hooks/useDebounce'
|
||||
import CategoryCard from '~/components/CategoryCard'
|
||||
import TierSelectionModal from '~/components/TierSelectionModal'
|
||||
|
|
@ -34,6 +43,13 @@ import { SERVICE_NAMES } from '../../../../constants/service_names'
|
|||
|
||||
const CURATED_CATEGORIES_KEY = 'curated-categories'
|
||||
const WIKIPEDIA_STATE_KEY = 'wikipedia-state'
|
||||
const CUSTOM_LIBRARIES_KEY = 'custom-libraries'
|
||||
|
||||
type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean }
|
||||
type BrowseResult = {
|
||||
directories: { name: string; url: string }[]
|
||||
files: { name: string; url: string; size_bytes: number | null }[]
|
||||
}
|
||||
|
||||
export default function ZimRemoteExplorer() {
|
||||
const queryClient = useQueryClient()
|
||||
|
|
@ -56,6 +72,20 @@ export default function ZimRemoteExplorer() {
|
|||
const [selectedWikipedia, setSelectedWikipedia] = useState<string | null>(null)
|
||||
const [isSubmittingWikipedia, setIsSubmittingWikipedia] = useState(false)
|
||||
|
||||
// Custom library state - persist selection to localStorage
|
||||
const [selectedSource, setSelectedSource] = useState<'default' | number>(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem('nomad:zim-library-source')
|
||||
if (saved && saved !== 'default') return parseInt(saved, 10)
|
||||
} catch {}
|
||||
return 'default'
|
||||
})
|
||||
const [browseUrl, setBrowseUrl] = useState<string | null>(null)
|
||||
const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([])
|
||||
const [manageModalOpen, setManageModalOpen] = useState(false)
|
||||
const [newLibraryName, setNewLibraryName] = useState('')
|
||||
const [newLibraryUrl, setNewLibraryUrl] = useState('')
|
||||
|
||||
const debouncedSetQuery = debounce((val: string) => {
|
||||
setQuery(val)
|
||||
}, 400)
|
||||
|
|
@ -79,6 +109,26 @@ export default function ZimRemoteExplorer() {
|
|||
enabled: true,
|
||||
})
|
||||
|
||||
// Fetch custom libraries
|
||||
const { data: customLibraries } = useQuery({
|
||||
queryKey: [CUSTOM_LIBRARIES_KEY],
|
||||
queryFn: () => api.listCustomLibraries(),
|
||||
refetchOnWindowFocus: false,
|
||||
})
|
||||
|
||||
// Browse custom library directory
|
||||
const {
|
||||
data: browseData,
|
||||
isLoading: isBrowsing,
|
||||
error: browseError,
|
||||
} = useQuery<BrowseResult>({
|
||||
queryKey: ['browse-library', browseUrl],
|
||||
queryFn: () => api.browseLibrary(browseUrl!) as Promise<BrowseResult>,
|
||||
enabled: !!browseUrl && selectedSource !== 'default',
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
})
|
||||
|
||||
const { data, fetchNextPage, isFetching, isLoading } =
|
||||
useInfiniteQuery<ListRemoteZimFilesResponse>({
|
||||
queryKey: ['remote-zim-files', query],
|
||||
|
|
@ -100,6 +150,7 @@ export default function ZimRemoteExplorer() {
|
|||
},
|
||||
refetchOnWindowFocus: false,
|
||||
placeholderData: keepPreviousData,
|
||||
enabled: selectedSource === 'default',
|
||||
})
|
||||
|
||||
const flatData = useMemo(() => {
|
||||
|
|
@ -145,6 +196,50 @@ export default function ZimRemoteExplorer() {
|
|||
fetchOnBottomReached(tableParentRef.current)
|
||||
}, [fetchOnBottomReached])
|
||||
|
||||
// Restore custom library selection on mount when data loads
|
||||
useEffect(() => {
|
||||
if (selectedSource !== 'default' && customLibraries) {
|
||||
const lib = customLibraries.find((l) => l.id === selectedSource)
|
||||
if (lib && !browseUrl) {
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
} else if (!lib) {
|
||||
// Saved library was deleted
|
||||
setSelectedSource('default')
|
||||
localStorage.setItem('nomad:zim-library-source', 'default')
|
||||
}
|
||||
}
|
||||
}, [customLibraries, selectedSource])
|
||||
|
||||
// When selecting a custom library, navigate to its root
|
||||
const handleSourceChange = (value: string) => {
|
||||
localStorage.setItem('nomad:zim-library-source', value)
|
||||
if (value === 'default') {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
} else {
|
||||
const id = parseInt(value, 10)
|
||||
const lib = customLibraries?.find((l) => l.id === id)
|
||||
if (lib) {
|
||||
setSelectedSource(id)
|
||||
setBrowseUrl(lib.base_url)
|
||||
setBreadcrumbs([{ name: lib.name, url: lib.base_url }])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const navigateToDirectory = (name: string, url: string) => {
|
||||
setBrowseUrl(url)
|
||||
setBreadcrumbs((prev) => [...prev, { name, url }])
|
||||
}
|
||||
|
||||
const navigateToBreadcrumb = (index: number) => {
|
||||
const crumb = breadcrumbs[index]
|
||||
setBrowseUrl(crumb.url)
|
||||
setBreadcrumbs((prev) => prev.slice(0, index + 1))
|
||||
}
|
||||
|
||||
async function confirmDownload(record: RemoteZimFileEntry) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
|
|
@ -170,6 +265,31 @@ export default function ZimRemoteExplorer() {
|
|||
)
|
||||
}
|
||||
|
||||
async function confirmCustomDownload(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
openModal(
|
||||
<StyledModal
|
||||
title="Confirm Download?"
|
||||
onConfirm={() => {
|
||||
downloadCustomFile(file)
|
||||
closeAllModals()
|
||||
}}
|
||||
onCancel={closeAllModals}
|
||||
open={true}
|
||||
confirmText="Download"
|
||||
cancelText="Cancel"
|
||||
confirmVariant="primary"
|
||||
>
|
||||
<p className="text-text-primary">
|
||||
Are you sure you want to download{' '}
|
||||
<strong>{file.name}</strong>
|
||||
{file.size_bytes ? ` (${formatBytes(file.size_bytes)})` : ''}? The Kiwix
|
||||
application will be restarted after the download is complete.
|
||||
</p>
|
||||
</StyledModal>,
|
||||
'confirm-download-custom-modal'
|
||||
)
|
||||
}
|
||||
|
||||
async function downloadFile(record: RemoteZimFileEntry) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(record.download_url, {
|
||||
|
|
@ -184,6 +304,26 @@ export default function ZimRemoteExplorer() {
|
|||
}
|
||||
}
|
||||
|
||||
async function downloadCustomFile(file: { name: string; url: string; size_bytes: number | null }) {
|
||||
try {
|
||||
await api.downloadRemoteZimFile(file.url, {
|
||||
title: file.name.replace(/\.zim$/, ''),
|
||||
size_bytes: file.size_bytes ?? undefined,
|
||||
})
|
||||
addNotification({
|
||||
message: `Started downloading "${file.name}"`,
|
||||
type: 'success',
|
||||
})
|
||||
invalidateDownloads()
|
||||
} catch (error) {
|
||||
console.error('Error downloading file:', error)
|
||||
addNotification({
|
||||
message: 'Failed to start download.',
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Category/tier handlers
|
||||
const handleCategoryClick = (category: CategoryWithStatus) => {
|
||||
if (!isOnline) return
|
||||
|
|
@ -269,6 +409,35 @@ export default function ZimRemoteExplorer() {
|
|||
},
|
||||
})
|
||||
|
||||
// Custom library management
|
||||
const addLibraryMutation = useMutation({
|
||||
mutationFn: () => api.addCustomLibrary(newLibraryName.trim(), newLibraryUrl.trim()),
|
||||
onSuccess: () => {
|
||||
addNotification({ message: 'Custom library added.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
setNewLibraryName('')
|
||||
setNewLibraryUrl('')
|
||||
},
|
||||
onError: () => {
|
||||
addNotification({ message: 'Failed to add custom library.', type: 'error' })
|
||||
},
|
||||
})
|
||||
|
||||
const removeLibraryMutation = useMutation({
|
||||
mutationFn: (id: number) => api.removeCustomLibrary(id),
|
||||
onSuccess: (_data, id) => {
|
||||
addNotification({ message: 'Custom library removed.', type: 'success' })
|
||||
queryClient.invalidateQueries({ queryKey: [CUSTOM_LIBRARIES_KEY] })
|
||||
if (selectedSource === id) {
|
||||
setSelectedSource('default')
|
||||
setBrowseUrl(null)
|
||||
setBreadcrumbs([])
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const hasCustomLibraries = customLibraries && customLibraries.length > 0
|
||||
|
||||
return (
|
||||
<SettingsLayout>
|
||||
<Head title="Content Explorer | Project N.O.M.A.D." />
|
||||
|
|
@ -307,7 +476,7 @@ export default function ZimRemoteExplorer() {
|
|||
Force Refresh Collections
|
||||
</StyledButton>
|
||||
</div>
|
||||
|
||||
|
||||
{/* Wikipedia Selector */}
|
||||
{isLoadingWikipedia ? (
|
||||
<div className="mt-8 bg-surface-primary rounded-lg border border-border-subtle p-6">
|
||||
|
|
@ -365,87 +534,303 @@ export default function ZimRemoteExplorer() {
|
|||
) : (
|
||||
<p className="text-text-muted mt-4">No curated content categories available.</p>
|
||||
)}
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="mt-12 mb-4" />
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
|
||||
{/* Kiwix Library / Custom Library Browser */}
|
||||
<div className="mt-12 mb-4 flex items-center justify-between">
|
||||
<StyledSectionHeader title="Browse the Kiwix Library" className="!mb-0" />
|
||||
<StyledButton
|
||||
onClick={() => setManageModalOpen(true)}
|
||||
disabled={!isOnline}
|
||||
icon="IconLibrary"
|
||||
>
|
||||
{hasCustomLibraries ? 'Manage Custom Libraries' : 'Add Custom Library'}
|
||||
</StyledButton>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`, // Use the size from the virtualizer
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
|
||||
{/* Source selector dropdown */}
|
||||
{hasCustomLibraries && (
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<label className="text-sm font-medium text-text-secondary">Source:</label>
|
||||
<select
|
||||
value={selectedSource === 'default' ? 'default' : String(selectedSource)}
|
||||
onChange={(e) => handleSourceChange(e.target.value)}
|
||||
className="rounded-md border border-border-default bg-surface-primary text-text-primary px-3 py-1.5 text-sm focus:outline-none focus:ring-2 focus:ring-desert-green"
|
||||
>
|
||||
<option value="default">Default (Kiwix)</option>
|
||||
{customLibraries.map((lib) => (
|
||||
<option key={lib.id} value={String(lib.id)}>
|
||||
{lib.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Default Kiwix library browser */}
|
||||
{selectedSource === 'default' && (
|
||||
<>
|
||||
<div className="flex justify-start mt-4">
|
||||
<Input
|
||||
name="search"
|
||||
label=""
|
||||
placeholder="Search available ZIM files..."
|
||||
value={queryUI}
|
||||
onChange={(e) => {
|
||||
setQueryUI(e.target.value)
|
||||
debouncedSetQuery(e.target.value)
|
||||
}}
|
||||
className="w-1/3"
|
||||
leftIcon={<IconSearch className="w-5 h-5 text-text-muted" />}
|
||||
/>
|
||||
</div>
|
||||
<StyledTable<RemoteZimFileEntry & { actions?: any }>
|
||||
data={flatData.map((i, idx) => {
|
||||
const row = virtualizer.getVirtualItems().find((v) => v.index === idx)
|
||||
return {
|
||||
...i,
|
||||
height: `${row?.size || 48}px`,
|
||||
translateY: row?.start || 0,
|
||||
}
|
||||
})}
|
||||
ref={tableParentRef}
|
||||
loading={isLoading}
|
||||
columns={[
|
||||
{
|
||||
accessor: 'title',
|
||||
},
|
||||
{
|
||||
accessor: 'author',
|
||||
},
|
||||
{
|
||||
accessor: 'summary',
|
||||
},
|
||||
{
|
||||
accessor: 'updated',
|
||||
render(record) {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
dateStyle: 'medium',
|
||||
}).format(new Date(record.updated))
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'size_bytes',
|
||||
title: 'Size',
|
||||
render(record) {
|
||||
return formatBytes(record.size_bytes)
|
||||
},
|
||||
},
|
||||
{
|
||||
accessor: 'actions',
|
||||
render(record) {
|
||||
return (
|
||||
<div className="flex space-x-2">
|
||||
<StyledButton
|
||||
icon={'IconDownload'}
|
||||
onClick={() => {
|
||||
confirmDownload(record)
|
||||
}}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Custom library directory browser */}
|
||||
{selectedSource !== 'default' && (
|
||||
<div className="mt-4">
|
||||
{/* Breadcrumb navigation */}
|
||||
<nav className="flex items-center gap-1 text-sm text-text-muted mb-4 flex-wrap">
|
||||
{breadcrumbs.map((crumb, idx) => (
|
||||
<span key={idx} className="flex items-center gap-1">
|
||||
{idx > 0 && <IconChevronRight className="w-4 h-4" />}
|
||||
{idx < breadcrumbs.length - 1 ? (
|
||||
<button
|
||||
onClick={() => navigateToBreadcrumb(idx)}
|
||||
className="text-desert-green hover:underline"
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
},
|
||||
]}
|
||||
className="relative overflow-x-auto overflow-y-auto h-[600px] w-full mt-4"
|
||||
tableBodyStyle={{
|
||||
position: 'relative',
|
||||
height: `${virtualizer.getTotalSize()}px`,
|
||||
}}
|
||||
containerProps={{
|
||||
onScroll: (e) => fetchOnBottomReached(e.currentTarget as HTMLDivElement),
|
||||
}}
|
||||
compact
|
||||
rowLines
|
||||
/>
|
||||
{crumb.name}
|
||||
</button>
|
||||
) : (
|
||||
<span className="text-text-primary font-medium">{crumb.name}</span>
|
||||
)}
|
||||
</span>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
{isBrowsing && (
|
||||
<div className="flex justify-center py-12">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-desert-green"></div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{browseError && (
|
||||
<Alert
|
||||
title="Could not fetch directory listing from this URL."
|
||||
message="The server may not support directory browsing, or the URL may be incorrect."
|
||||
type="error"
|
||||
variant="solid"
|
||||
/>
|
||||
)}
|
||||
|
||||
{!isBrowsing && !browseError && browseData && (
|
||||
<div className="bg-surface-primary rounded-lg border border-border-subtle overflow-hidden relative" style={{ maxHeight: '600px', overflowY: 'auto' }}>
|
||||
{browseData.directories.length === 0 && browseData.files.length === 0 ? (
|
||||
<p className="text-text-muted p-6 text-center">
|
||||
No directories or ZIM files found at this location.
|
||||
</p>
|
||||
) : (
|
||||
<table className="w-full text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-border-subtle bg-surface-secondary sticky top-0 z-10">
|
||||
<th className="text-left px-4 py-3 font-medium text-text-secondary">Name</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-32">Size</th>
|
||||
<th className="text-right px-4 py-3 font-medium text-text-secondary w-36"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{browseData.directories.map((dir) => (
|
||||
<tr
|
||||
key={dir.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary cursor-pointer transition-colors"
|
||||
onClick={() => navigateToDirectory(dir.name, dir.url)}
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFolder className="w-5 h-5 text-desert-orange" />
|
||||
{dir.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">--</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<IconChevronRight className="w-4 h-4 text-text-muted ml-auto" />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{browseData.files.map((file) => (
|
||||
<tr
|
||||
key={file.url}
|
||||
className="border-b border-border-subtle hover:bg-surface-secondary transition-colors"
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<span className="flex items-center gap-2 text-text-primary">
|
||||
<IconFileDownload className="w-5 h-5 text-desert-green" />
|
||||
{file.name}
|
||||
</span>
|
||||
</td>
|
||||
<td className="text-right px-4 py-3 text-text-muted">
|
||||
{file.size_bytes ? formatBytes(file.size_bytes) : '--'}
|
||||
</td>
|
||||
<td className="text-right px-4 py-3">
|
||||
<StyledButton
|
||||
icon="IconDownload"
|
||||
onClick={() => confirmCustomDownload(file)}
|
||||
>
|
||||
Download
|
||||
</StyledButton>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ActiveDownloads filetype="zim" withHeader />
|
||||
|
||||
{/* Manage Custom Libraries Modal */}
|
||||
<StyledModal
|
||||
title="Manage Custom Libraries"
|
||||
open={manageModalOpen}
|
||||
onCancel={() => setManageModalOpen(false)}
|
||||
cancelText="Close"
|
||||
>
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<p className="text-sm text-text-muted mb-4">
|
||||
Add Kiwix mirrors or other ZIM file sources for faster downloads.
|
||||
</p>
|
||||
|
||||
{/* Existing libraries */}
|
||||
{customLibraries && customLibraries.length > 0 && (
|
||||
<div className="space-y-2 mb-6">
|
||||
{customLibraries.map((lib) => (
|
||||
<div
|
||||
key={lib.id}
|
||||
className="flex items-center justify-between bg-surface-secondary rounded-lg px-4 py-3 border border-border-subtle"
|
||||
>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="font-medium text-text-primary truncate">
|
||||
{lib.name}
|
||||
{lib.is_default && (
|
||||
<span className="ml-2 text-xs text-text-muted font-normal">(built-in)</span>
|
||||
)}
|
||||
</p>
|
||||
<p className="text-xs text-text-muted truncate">{lib.base_url}</p>
|
||||
</div>
|
||||
{!lib.is_default && (
|
||||
<button
|
||||
onClick={() => removeLibraryMutation.mutate(lib.id)}
|
||||
className="ml-3 p-1.5 text-text-muted hover:text-red-500 transition-colors rounded"
|
||||
title="Remove library"
|
||||
>
|
||||
<IconTrash className="w-4 h-4" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Add new library form */}
|
||||
<div className="space-y-3">
|
||||
<Input
|
||||
name="library-name"
|
||||
label="Library Name"
|
||||
placeholder="e.g., Debian Mirror"
|
||||
value={newLibraryName}
|
||||
onChange={(e) => setNewLibraryName(e.target.value)}
|
||||
/>
|
||||
<Input
|
||||
name="library-url"
|
||||
label="Base URL"
|
||||
placeholder="e.g., https://cdimage.debian.org/mirror/kiwix.org/zim/"
|
||||
value={newLibraryUrl}
|
||||
onChange={(e) => setNewLibraryUrl(e.target.value)}
|
||||
/>
|
||||
<StyledButton
|
||||
icon="IconPlus"
|
||||
onClick={() => addLibraryMutation.mutate()}
|
||||
disabled={
|
||||
!newLibraryName.trim() ||
|
||||
!newLibraryUrl.trim() ||
|
||||
addLibraryMutation.isPending
|
||||
}
|
||||
>
|
||||
Add Library
|
||||
</StyledButton>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</StyledModal>
|
||||
</main>
|
||||
</div>
|
||||
</SettingsLayout>
|
||||
|
|
|
|||
|
|
@ -178,6 +178,12 @@ router
|
|||
|
||||
router.get('/wikipedia', [ZimController, 'getWikipediaState'])
|
||||
router.post('/wikipedia/select', [ZimController, 'selectWikipedia'])
|
||||
|
||||
router.get('/custom-libraries', [ZimController, 'listCustomLibraries'])
|
||||
router.post('/custom-libraries', [ZimController, 'addCustomLibrary'])
|
||||
router.delete('/custom-libraries/:id', [ZimController, 'removeCustomLibrary'])
|
||||
router.get('/browse-library', [ZimController, 'browseLibrary'])
|
||||
|
||||
router.delete('/:filename', [ZimController, 'delete'])
|
||||
})
|
||||
.prefix('/api/zim')
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user