github-mirror/project-nomad
1
0
Fork 0
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git synced 2026-10-17 05:14:19 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

fix(zim): extract Kiwix catalog URL into system-wide constant

Browse Source
This commit is contained in:
jakeaturner 2026-09-02 15:55:44 +00:00 committed by Jake Turner
parent 0a32344942
commit fbeb3cf7ae
3 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
admin/app/services/kiwix_catalog_service.ts
View File

@ -4,6 +4,7 @@ import { DateTime } from 'luxon'
import logger from '@adonisjs/core/services/logger'
import InstalledResource from '#models/installed_resource'
import { isRawListRemoteZimFilesResponse } from '../../util/zim.js'
import { KIWIX_CATALOG_BASE_URL } from '../../constants/kiwix.js'
/**
* Local, in-process freshness check for installed content (Kiwix ZIM files +
@ -24,14 +25,6 @@ import { isRawListRemoteZimFilesResponse } from '../../util/zim.js'
* defensive — a malformed entry is skipped, never thrown.
*/
// `browse.library.kiwix.org` is Kiwix's *human* browsing host. On 2026-08-29 it grew an
// anti-crawler interstitial that answers HTTP 200 with an HTML "Please confirm..." page
// requiring JS + a cookie, so every catalog request here returned unparseable HTML.
// `parseZimEntries` swallows that as an empty feed, which reads as "no updates available"
// -- a silent, permanent stall of ZIM auto-update. `opds.library.kiwix.org` is the
// machine-facing host for the same API and is not gated; `library.kiwix.org` now 301s to
// it. Keep catalog traffic off the browse host.
const KIWIX_CATALOG_URL = 'https://opds.library.kiwix.org/catalog/v2/entries'
const GITHUB_PMTILES_URL =
'https://api.github.com/repos/Crosstalk-Solutions/project-nomad-maps/contents/pmtiles'
@ -194,7 +187,7 @@ export class KiwixCatalogService {
count: number
start: number
}): Promise<{ entries: CatalogZimEntry[]; totalResults: number }> {
const res = await axios.get(KIWIX_CATALOG_URL, {
const res = await axios.get(KIWIX_CATALOG_BASE_URL, {
params: {
start: params.start,
count: params.count,

7
admin/app/services/zim_service.ts
View File

@ -38,6 +38,7 @@ import CustomLibrarySource from '#models/custom_library_source'
import { assertNotPrivateUrl } from '#validators/common'
import { resolveZimDownload } from '../utils/zim_download_resolution.js'
import { getHostedContentHeaders } from '../utils/hosted_content_auth.js'
import { KIWIX_CATALOG_BASE_URL } from '../../constants/kiwix.js'
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
const WIKIPEDIA_OPTIONS_URL = 'https://raw.githubusercontent.com/Crosstalk-Solutions/project-nomad/refs/heads/main/collections/wikipedia.json'
@ -81,10 +82,6 @@ export class ZimService {
count: number
query?: string
}): Promise<ListRemoteZimFilesResponse> {
// The machine-facing OPDS host. NOT `browse.library.kiwix.org`, which since 2026-08-29
// serves an anti-crawler confirmation page (HTTP 200 + HTML) that fails
// `isRawListRemoteZimFilesResponse` and 500s this endpoint. See kiwix_catalog_service.
const LIBRARY_BASE_URL = 'https://opds.library.kiwix.org/catalog/v2/entries'
// Kiwix returns pages of content unaware of what the user has installed locally. When
// the installed set is large, a single 12-item Kiwix page can come back with everything
// already installed → 0 post-filter items → frontend deadlock (#731). Accumulate across
@ -110,7 +107,7 @@ export class ZimService {
let totalResults = 0
for (let i = 0; i < MAX_KIWIX_FETCHES; i++) {
const res = await axios.get(LIBRARY_BASE_URL, {
const res = await axios.get(KIWIX_CATALOG_BASE_URL, {
params: {
start: currentStart,
count: KIWIX_PAGE_SIZE,

12
admin/constants/kiwix.ts
View File

@ -1,2 +1,12 @@
export const KIWIX_LIBRARY_CMD = '--library /data/kiwix-library.xml --monitorLibrary --address=all'
export const KIWIX_LIBRARY_CMD = '--library /data/kiwix-library.xml --monitorLibrary --address=all'
/** `browse.library.kiwix.org` is Kiwix's *human* browsing host. On 2026-08-29 it grew an
* anti-crawler interstitial that answers HTTP 200 with an HTML "Please confirm..." page
* requiring JS + a cookie, so every catalog request here returned unparseable HTML.
* `parseZimEntries` swallows that as an empty feed, which reads as "no updates available"
* -- a silent, permanent stall of ZIM auto-update. `opds.library.kiwix.org` is the
* machine-facing host for the same API and is not gated; `library.kiwix.org` now 301s to
* it. Keep catalog traffic off the browse host.
*/
export const KIWIX_CATALOG_BASE_URL = 'https://opds.library.kiwix.org/catalog/v2/entries'