From d1a4e9ade52d0144e1bb1095d0b70315dcab75cb Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 31 Aug 2026 14:03:41 -0700 Subject: [PATCH] fix(zim): use the OPDS catalog host, not Kiwix's gated browse host browse.library.kiwix.org grew an anti-crawler interstitial on 2026-08-29 that answers HTTP 200 with an HTML confirmation page. Content Explorer's remote listing 500s on it and ZIM auto-update silently reports "no updates". opds.library.kiwix.org serves the same OPDS API ungated. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01F8Mgg1vd2eKSs8StZiuyMV --- admin/app/services/kiwix_catalog_service.ts | 9 ++++++++- admin/app/services/zim_service.ts | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/admin/app/services/kiwix_catalog_service.ts b/admin/app/services/kiwix_catalog_service.ts index e7f0422d..2d172cf3 100644 --- a/admin/app/services/kiwix_catalog_service.ts +++ b/admin/app/services/kiwix_catalog_service.ts @@ -24,7 +24,14 @@ import { isRawListRemoteZimFilesResponse } from '../../util/zim.js' * defensive — a malformed entry is skipped, never thrown. */ -const KIWIX_CATALOG_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' +// `browse.library.kiwix.org` is Kiwix's *human* browsing host. On 2026-08-29 it grew an +// anti-crawler interstitial that answers HTTP 200 with an HTML "Please confirm..." page +// requiring JS + a cookie, so every catalog request here returned unparseable HTML. +// `parseZimEntries` swallows that as an empty feed, which reads as "no updates available" +// -- a silent, permanent stall of ZIM auto-update. `opds.library.kiwix.org` is the +// machine-facing host for the same API and is not gated; `library.kiwix.org` now 301s to +// it. Keep catalog traffic off the browse host. +const KIWIX_CATALOG_URL = 'https://opds.library.kiwix.org/catalog/v2/entries' const GITHUB_PMTILES_URL = 'https://api.github.com/repos/Crosstalk-Solutions/project-nomad-maps/contents/pmtiles' diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c8a735c2..a687c3be 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -81,7 +81,10 @@ export class ZimService { count: number query?: string }): Promise { - const LIBRARY_BASE_URL = 'https://browse.library.kiwix.org/catalog/v2/entries' + // The machine-facing OPDS host. NOT `browse.library.kiwix.org`, which since 2026-08-29 + // serves an anti-crawler confirmation page (HTTP 200 + HTML) that fails + // `isRawListRemoteZimFilesResponse` and 500s this endpoint. See kiwix_catalog_service. + const LIBRARY_BASE_URL = 'https://opds.library.kiwix.org/catalog/v2/entries' // Kiwix returns pages of content unaware of what the user has installed locally. When // the installed set is large, a single 12-item Kiwix page can come back with everything // already installed → 0 post-filter items → frontend deadlock (#731). Accumulate across