project-nomad/admin/app/controllers/zim_controller.ts
chriscrosstalk 62e75fdb54 feat(Content): custom ZIM library sources with pre-seeded mirrors (#593)
* feat(content): add custom ZIM library sources with pre-seeded mirrors

Users reported slow download speeds from the default Kiwix CDN. This adds
the ability to browse and download ZIM files from alternative Kiwix mirrors
or self-hosted repositories, all through the GUI.

- Add "Custom Libraries" button next to "Browse the Kiwix Library"
- Source dropdown to switch between Default (Kiwix) and custom libraries
- Browsable directory structure with breadcrumb navigation
- 5 pre-seeded official Kiwix mirrors (US, DE, DK, UK, Global CDN)
- Built-in mirrors protected from deletion
- Downloads use existing pipeline (progress, cancel, Kiwix restart)
- Source selection persists across page loads via localStorage
- Scrollable directory browser (600px max) with sticky header
- SSRF protection on all custom library URLs

Closes #576

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(content): recognize Wikipedia downloads from mirror sources

When Wikipedia is downloaded via a custom mirror instead of the default
Kiwix server, the completion callback now matches by filename instead
of exact URL. This ensures the Wikipedia selector correctly shows
"Installed" status and triggers old-version cleanup regardless of
which mirror was used.

Also handles the case where no Wikipedia selection exists yet (file
downloaded before visiting the selector), creating the record
automatically.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(ZIM): use cheerio for custom mirror directory parsing

* fix(ZIM): use URL constructor for more robust joining

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Jake Turner <jturner@cosmistack.com>
2026-05-20 10:16:00 -07:00

136 lines
4.2 KiB
TypeScript

import { ZimService } from '#services/zim_service'
import {
assertNotPrivateUrl,
downloadCategoryTierValidator,
filenameParamValidator,
remoteDownloadWithMetadataValidator,
selectWikipediaValidator,
} from '#validators/common'
import { addCustomLibraryValidator, browseLibraryValidator, idParamValidator, listRemoteZimValidator } from '#validators/zim'
import { inject } from '@adonisjs/core'
import type { HttpContext } from '@adonisjs/core/http'
@inject()
export default class ZimController {
constructor(private zimService: ZimService) {}
async list({}: HttpContext) {
return await this.zimService.list()
}
async listRemote({ request }: HttpContext) {
const payload = await request.validateUsing(listRemoteZimValidator)
const { start = 0, count = 12, query } = payload
return await this.zimService.listRemote({ start, count, query })
}
async downloadRemote({ request }: HttpContext) {
const payload = await request.validateUsing(remoteDownloadWithMetadataValidator)
assertNotPrivateUrl(payload.url)
const { filename, jobId } = await this.zimService.downloadRemote(payload.url, payload.metadata)
return {
message: 'Download started successfully',
filename,
jobId,
url: payload.url,
}
}
async listCuratedCategories({}: HttpContext) {
return await this.zimService.listCuratedCategories()
}
async downloadCategoryTier({ request }: HttpContext) {
const payload = await request.validateUsing(downloadCategoryTierValidator)
const resources = await this.zimService.downloadCategoryTier(
payload.categorySlug,
payload.tierSlug
)
return {
message: 'Download started successfully',
categorySlug: payload.categorySlug,
tierSlug: payload.tierSlug,
resources,
}
}
async delete({ request, response }: HttpContext) {
const payload = await request.validateUsing(filenameParamValidator)
try {
await this.zimService.delete(payload.params.filename)
} catch (error) {
if (error.message === 'not_found') {
return response.status(404).send({
message: `ZIM file with key ${payload.params.filename} not found`,
})
}
throw error // Re-throw any other errors and let the global error handler catch
}
return {
message: 'ZIM file deleted successfully',
}
}
// Wikipedia selector endpoints
async getWikipediaState({}: HttpContext) {
return this.zimService.getWikipediaState()
}
async selectWikipedia({ request }: HttpContext) {
const payload = await request.validateUsing(selectWikipediaValidator)
return this.zimService.selectWikipedia(payload.optionId)
}
// Custom library endpoints
async listCustomLibraries({}: HttpContext) {
return this.zimService.listCustomLibraries()
}
async addCustomLibrary({ request, response }: HttpContext) {
const payload = await request.validateUsing(addCustomLibraryValidator)
assertNotPrivateUrl(payload.base_url)
try {
const source = await this.zimService.addCustomLibrary(payload.name, payload.base_url)
return { message: 'Custom library added', library: source }
} catch (error) {
if (error.message === 'Maximum of 10 custom libraries allowed') {
return response.status(400).send({ message: error.message })
}
throw error
}
}
async removeCustomLibrary({ request, response }: HttpContext) {
const payload = await request.validateUsing(idParamValidator)
try {
await this.zimService.removeCustomLibrary(payload.params.id)
return { message: 'Custom library removed' }
} catch (error) {
if (error.message === 'Custom library not found') {
return response.status(404).send({ message: error.message })
}
throw error
}
}
async browseLibrary({ request, response }: HttpContext) {
const payload = await request.validateUsing(browseLibraryValidator)
try {
return await this.zimService.browseLibraryUrl(payload.url)
} catch (error) {
if (error.message?.includes('loopback or link-local')) {
return response.status(400).send({ message: error.message })
}
return response.status(502).send({
message: 'Could not fetch directory listing from the provided URL',
})
}
}
}