diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..48a73bc8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Shell scripts must keep LF endings. These are copied into Linux container +# images, where a CRLF after the shebang makes the interpreter unresolvable and +# the container exits immediately. Checkouts on Windows would otherwise convert +# them, which only shows up when building an image locally rather than in CI. +*.sh text eol=lf diff --git a/.gitignore b/.gitignore index 1c1c9712..3155ae56 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,7 @@ admin/public/assets # Admin specific development files admin/storage + +# Python bytecode (install/nomad-translate helper scripts) +__pycache__/ +*.pyc diff --git a/admin/constants/service_names.ts b/admin/constants/service_names.ts index 8bbc8174..b39fc3a8 100644 --- a/admin/constants/service_names.ts +++ b/admin/constants/service_names.ts @@ -18,4 +18,5 @@ export const SERVICE_NAMES = { HOMEBOX: 'nomad_homebox', VAULTWARDEN: 'nomad_vaultwarden', JELLYFIN: 'nomad_jellyfin', + TRANSLATE: 'nomad_translate', } diff --git a/admin/constants/supply_depot_docs.ts b/admin/constants/supply_depot_docs.ts index a4ad6cb0..4883a7f5 100644 --- a/admin/constants/supply_depot_docs.ts +++ b/admin/constants/supply_depot_docs.ts @@ -20,6 +20,7 @@ export const SUPPLY_DEPOT_DOC_ANCHORS: Record = { [SERVICE_NAMES.KOLIBRI]: 'kolibri', [SERVICE_NAMES.KOLIBRI_GEN2]: 'kolibri', [SERVICE_NAMES.MESHCORE_WEB]: 'meshcore-web', + [SERVICE_NAMES.TRANSLATE]: 'offline-translation', } // Returns the in-app docs link for a service, or null if it has no documentation section. diff --git a/admin/database/seeders/service_seeder.ts b/admin/database/seeders/service_seeder.ts index f4931033..de07770d 100644 --- a/admin/database/seeders/service_seeder.ts +++ b/admin/database/seeders/service_seeder.ts @@ -542,6 +542,51 @@ export default class ServiceSeeder extends BaseSeeder { depends_on: null, metadata: JSON.stringify({ minMemoryMB: 2048, minDiskMB: 20480 }), }, + { + service_name: SERVICE_NAMES.TRANSLATE, + friendly_name: 'Translated Library', + powered_by: 'Bergamot', + display_order: 28, + description: + 'Read the Information Library in another language. Machine translation that works offline, on CPU', + icon: 'IconWorld', + container_image: 'ghcr.io/crosstalk-solutions/project-nomad-translate:0.1.0', + source_repo: 'https://github.com/browsermt/bergamot-translator', + container_command: null, + container_config: JSON.stringify({ + HostConfig: { + RestartPolicy: { Name: 'unless-stopped' }, + PortBindings: { '8391/tcp': [{ HostPort: '8460' }] }, + Binds: [`${ServiceSeeder.NOMAD_STORAGE_ABS_PATH}/translate/models:/models`], + }, + ExposedPorts: { '8391/tcp': {} }, + // TRANSLATE_LANGS is the language set fetched on first start, about + // 74 MB per language for the pair in both directions. Editable via + // Manage > Edit; the container re-checks on restart and only fetches + // what is missing. + Env: [ + // Reached by container name on the shared NOMAD network, which + // DockerService attaches every managed container to. Using the name + // rather than a host port means this survives the library being + // remapped. + 'KIWIX=http://nomad_kiwix_server:8080', + 'TRANSLATE_LANGS=fr,es,de', + 'WORKERS=8', + ], + }), + ui_location: '8460', + installed: false, + installation_status: 'idle', + is_dependency_service: false, + is_custom: false, + category: 'education', + // Translating an article it cannot fetch is meaningless, so the library + // has to be there first. + depends_on: SERVICE_NAMES.KIWIX, + // Peak RSS measured at 729 MB with three language pairs resident; models + // are about 74 MB per language on disk. + metadata: JSON.stringify({ minMemoryMB: 1536, minDiskMB: 1024 }), + }, ] async run() { diff --git a/admin/docs/supply-depot-apps.md b/admin/docs/supply-depot-apps.md index 42247621..9cb51b40 100644 --- a/admin/docs/supply-depot-apps.md +++ b/admin/docs/supply-depot-apps.md @@ -282,3 +282,23 @@ A browser-based client for [MeshCore](https://meshcore.io) radios. MeshCore is a **Your data:** There's nothing to set up or store on your NOMAD for this app. Your radio's settings live on the radio itself, and the app's preferences live in your browser. There's no NOMAD folder to manage. **Works offline:** Fully offline, which is the whole point of MeshCore. The app is served from your NOMAD and talks to your radio directly over USB or Bluetooth, never the internet. + +## Translated Library {% #offline-translation %} + +Reads the Information Library in another language. Open an article and a **Translate this page** bar appears at the top with a button for each installed language, plus **Original** to switch back. Your choice sticks as you click through to other articles. + +**Why this instead of the AI Assistant:** the AI Assistant can translate, but this is roughly 1,600 times faster on the same machine, and it does not need a graphics card, so it works on every NOMAD. It is also more careful with names: asked to translate a page, the AI Assistant will happily translate "Project NOMAD" into another language, and this will not. + +**Choosing languages:** French, Spanish and German are set up by default. To change that, use **Manage > Edit** and set `TRANSLATE_LANGS` to a comma-separated list of language codes, for example `fr,it,pt`. Each language is about 74 MB, and new ones download the next time the app restarts. Around 40 languages are available, including Hindi, Bengali, Tamil, Telugu, Vietnamese and Indonesian. **Chinese, Japanese, Korean, Arabic and Thai are not available**, because no compact model exists for them yet. + +**First start needs internet.** The language models download when the app first runs, the same as installing any other app. After that it is entirely offline. If you install this while disconnected the app still starts and the library still works, just without translation until it can fetch the models. + +**What it does not translate:** tables and infoboxes, the page title in your browser tab, and Kiwix's own search results. Searching also still matches the original language, so look things up in English and translate the article you land on. + +**A note on two language buttons:** the library's own toolbar has a globe that changes the *menus* around the page. The bar this app adds changes the *article*. They are different things and sit close together, which is unfortunate but not something we can move. + +**Accuracy:** this is machine translation, and it is literal. It is very good for getting the sense of an article. Be careful relying on it for exact medical or safety wording, where the correct term in another language is often not the literal one. + +**Your data:** language models live in `storage/translate/models`. Nothing you read is stored or sent anywhere. + +**Works offline:** yes, once the models have downloaded. diff --git a/install/nomad-translate/Dockerfile b/install/nomad-translate/Dockerfile new file mode 100644 index 00000000..b6d2da7b --- /dev/null +++ b/install/nomad-translate/Dockerfile @@ -0,0 +1,37 @@ +# Offline translation proxy for the Information Library. +# +# amd64 only, and this is not just packaging: Bergamot's intgemm backend is +# x86 specific and there is no aarch64 wheel. ARM would need marian's `ruy` +# path. The build workflow pins linux/amd64 so this fails loudly at build time +# rather than pulling cleanly on ARM and dying at start. +FROM python:3.10-slim + +# Pinned to python 3.10 because that is what bergamot 0.4.5 publishes wheels +# for. On 3.12 pip resolves no further than 0.1.2 and the build fails outright. +# Moving off this pin means moving off the wheel, see below. +# +# bergamot 0.4.5 is the PyPI wheel, uploaded 2022-06-21. It works, and it is +# what the spike proved, but it is old: browsermt/bergamot-translator's last +# real commit was 2024-05 and the maintained successor is mozilla/translations +# (inference/). Moving to a build from that repo is tracked as follow-up work +# and would remove this pin. +RUN pip install --no-cache-dir bergamot==0.4.5 + +WORKDIR /app +COPY blocks.py proxy.py fetch_models.py entrypoint.sh /app/ +RUN chmod +x /app/entrypoint.sh + +# Models are a mounted volume, not baked in: at ~37 MB per direction the image +# would balloon, and which languages a user wants is not knowable at build time. +ENV MODELS=/models \ + KIWIX=http://nomad_kiwix_server:8080 \ + PORT=8391 \ + WORKERS=8 \ + TRANSLATE_LANGS=fr,es,de + +EXPOSE 8391 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8391/viewer', timeout=4).status==200 else 1)" + +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/install/nomad-translate/README.md b/install/nomad-translate/README.md new file mode 100644 index 00000000..9b1e7099 --- /dev/null +++ b/install/nomad-translate/README.md @@ -0,0 +1,94 @@ +# nomad-translate + +Offline machine translation for the Information Library, shipped as a Supply +Depot app. Issue [#1272](https://github.com/Crosstalk-Solutions/project-nomad/issues/1272). + +A path-preserving reverse proxy in front of `kiwix-serve`. Everything is +forwarded byte for byte on the paths Kiwix already uses; only `text/html` under +`/content/` is rewritten. + +## Why a proxy and not our own reader + +`viewer.js` keeps the content iframe in sync with `location.hash` by comparing +`contentWindow.location.pathname`. Point the iframe anywhere else and you start +the ping-pong its own comment warns about, and you are now patching Kiwix's +JavaScript and re-verifying it on every Kiwix bump. + +Because the path never moves, **no Kiwix JS is modified**, and Kiwix's header, +search, library and random keep working because they are Kiwix. Language is a +cookie, so links inside the ZIM need no rewriting either. + +Verified transparent during the spike: `/viewer`, `/skin/viewer.js` and +`/catalog/v2/entries` return byte-identical, and a malformed `/search` +reproduces Kiwix's own `400` with the same body length. + +Reimplementing or wrapping the Kiwix reading view is explicitly out of scope. + +## Layout + +| File | Purpose | +|---|---| +| `proxy.py` | The proxy, the language control, and the rewrite step | +| `blocks.py` | Finds translatable blocks; decides HTML mode versus plain text | +| `fetch_models.py` | Pulls models from Firefox Remote Settings on first start | +| `test_blocks.py` | Tests for the block planner. No Bergamot, no network | +| `entrypoint.sh` | Fetch models, then serve. Fetch failure is non-fatal | + +Run the tests with `python3 install/nomad-translate/test_blocks.py`. + +## Configuration + +| Variable | Default | Meaning | +|---|---|---| +| `KIWIX` | `http://nomad_kiwix_server:8080` | Upstream library | +| `MODELS` | `/models` | Where models are stored | +| `TRANSLATE_LANGS` | `fr,es,de` | Languages to fetch, paired with English both ways | +| `WORKERS` | `8` | Bergamot worker threads | +| `PORT` | `8391` | Listen port inside the container | +| `MAX_TRANSLATE_BYTES` | `4194304` | Pages larger than this are forwarded untranslated | + +## Models + +From **Firefox Remote Settings**, the endpoint Firefox itself uses, not the +`mozilla/firefox-translations-models` GitHub repo, which was archived on +2026-08-21. MPL-2.0. + +About 37 MB per direction, 74 MB for a language in both directions. 84 pairs +across roughly 44 languages. **No Chinese, Japanese, Korean, Arabic or Thai** in +the tiny model set. + +Fetching is skipped entirely when every requested pair is already on disk, so a +restart on a genuinely offline box does not fail. A failed fetch is non-fatal: +the proxy still starts and still forwards the library, untranslated. Translation +being unavailable must never make the Information Library unreachable. + +## amd64 only + +Not just packaging. Bergamot's `intgemm` backend is x86 specific and there is no +aarch64 wheel; ARM would need marian's `ruy` path. The build workflow pins +`linux/amd64` so this fails at build time rather than producing an image that +pulls cleanly on ARM and dies at start. + +## Known gaps + +- **Tables are not translated.** Excluded wholesale to avoid a nesting problem, + so infobox rows stay in the source language. +- **`` is not translated**, so the browser tab and Kiwix's book button + stay in the original language. +- **Kiwix's search results page is not translated** (it is not under + `/content/`), and search still matches original-language index terms. +- **No caching.** Every page view re-translates, roughly 1 s for a long article. +- **A block whose inner tag is never closed is dropped**, not translated. See + the test for why: the parser never claims a span, so the balance check never + runs. Safe, but silent. +- **Two language controls.** Kiwix's toolbar has a globe that changes the + interface language; ours changes the article. Ours is labelled "Translate this + page" rather than with a globe or a language name, which is the cheapest + available mitigation, but we do not own the other button. + +## Runtime + +The PyPI `bergamot` wheel is from 2022-06-21 and +`browsermt/bergamot-translator`'s last real commit was 2024-05. The maintained +successor is `mozilla/translations` (`inference/`). Building from there instead +is worthwhile follow-up work and would remove the pin in the Dockerfile. diff --git a/install/nomad-translate/blocks.py b/install/nomad-translate/blocks.py new file mode 100644 index 00000000..ccc93078 --- /dev/null +++ b/install/nomad-translate/blocks.py @@ -0,0 +1,211 @@ +"""Locate the translatable block elements in a ZIM article's HTML. + +Bergamot's HTML mode re-places inline tags onto the translated tokens using +alignment rather than translating fragments separately, which is what makes +`<a href="Water">water</a>` come back as `<a href="Water">l'eau</a>` with the +link on the correct word even when the target language reorders the sentence. + +That mode REQUIRES balanced markup and raises "Not all tags were closed" +otherwise, so blocks are located with a real parser. Regex cannot do this: a +non-greedy `<li>...</li>` happily stops at a nested list's closing tag. Anything +that still fails the balance check is translated as plain text instead, losing +its inline markup but not its content. +""" + +import html +import re +from html.parser import HTMLParser + +# Elements whose text is translated. Deliberately the outermost balanced block: +# translating a whole <p> at once is what gives the aligner enough context to +# place the inline tags sensibly. +BLOCK_TAGS = { + "p", "li", "h1", "h2", "h3", "h4", "h5", "h6", + "dt", "dd", "caption", "figcaption", "th", "td", +} + +# Never descend into these. `table` is here because its nesting defeated the +# balance check in the prototype, which is why infobox rows stay untranslated. +OPAQUE = {"script", "style", "code", "pre", "math", "svg", "table"} + +# `div` is not a semantic text element, but real prose lives in one often enough +# that leaving it out loses whole pages. Sotoki-generated StackExchange ZIMs put +# every question excerpt in `<div class="...excerpt">`, so the titles translated +# and the descriptions underneath them did not. Claimed only as a leaf: see +# BlockFinder.handle_starttag. +CLAIMABLE = BLOCK_TAGS | {"div"} + +VOID = { + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", +} + +# Bergamot emits <font> wrappers in HTML mode; they are not in the source and +# are stripped back out. +FONTS = re.compile(r"(?is)</?font[^>]*>") +TAGS = re.compile(r"(?is)<[^>]+>") + +# Two consecutive letters. Filters out blocks that are only punctuation, digits +# or whitespace, which are not worth a translation round trip. +HAS_TEXT = re.compile(r"[A-Za-z]{2}") + +# Bergamot's HTML mode decodes NAMED entities (`&` arrives as "&") but not +# numeric character references: it treats `'` as literal text and escapes +# the ampersand on the way out, so `I'm` comes back as `I&#x27;m` and +# renders as visible markup instead of an apostrophe. Decode them before the +# round trip. +# +# `&`, `<` and `>` are deliberately left encoded. Decoding those would inject +# real markup into the fragment and break the balance check that gates HTML +# mode, turning a translated block into a plain-text one that loses its links. +NUMERIC_REF = re.compile(r"&#(?:[xX]([0-9a-fA-F]+)|([0-9]+));") +MARKUP_CHARS = {0x26, 0x3C, 0x3E} # & < > + + +def decode_numeric_refs(frag: str) -> str: + """Turn numeric character references into characters Bergamot can carry.""" + + def one(match: "re.Match[str]") -> str: + hex_digits, dec_digits = match.group(1), match.group(2) + try: + codepoint = int(hex_digits, 16) if hex_digits else int(dec_digits) + except ValueError: + return match.group(0) + if codepoint in MARKUP_CHARS or not 0 < codepoint <= 0x10FFFF: + return match.group(0) + return chr(codepoint) + + return NUMERIC_REF.sub(one, frag) + + +class BlockFinder(HTMLParser): + """Yield (inner_start, inner_end) offsets for the outermost balanced blocks. + + Only the outermost block is claimed: once inside one, nested blocks are + ignored until it closes, so a `<li>` containing a `<p>` is translated once + rather than twice with overlapping spans. + """ + + def __init__(self, src: str): + super().__init__(convert_charrefs=False) + self.src = src + # Byte offset of the start of each line, so getpos() can be converted + # into an absolute offset into the source. + self.lines = [0] + for line in src.splitlines(keepends=True): + self.lines.append(self.lines[-1] + len(line)) + self.spans: list[tuple[int, int]] = [] + self.claim: tuple[str, int, int] | None = None + self.depth = 0 + self.opaque = 0 + + def _off(self) -> int: + line, col = self.getpos() + return self.lines[line - 1] + col + + def handle_starttag(self, tag, attrs): + if tag in VOID: + return + if tag in OPAQUE: + self.opaque += 1 + self.depth += 1 + + # A `div` is claimable only when it turns out to be a text leaf. Claim it + # speculatively, then abandon that claim the moment anything block-level + # opens inside it, so the inner block is translated rather than the + # layout wrapper that happens to contain it. Without this, `div` could + # not be in the set at all: the outermost-block rule would claim a + # top-level wrapper and swallow the whole page in one job. + if self.claim and self.claim[0] == "div" and tag in CLAIMABLE: + self.claim = None + + if self.claim is None and not self.opaque and tag in CLAIMABLE: + end = self.src.find(">", self._off()) + self.claim = (tag, self.depth, end + 1) + + def handle_startendtag(self, tag, attrs): + # Self-closing: no depth change, nothing to claim. + pass + + def handle_endtag(self, tag): + if tag in VOID: + return + if self.claim and tag == self.claim[0] and self.depth == self.claim[1]: + self.spans.append((self.claim[2], self._off())) + self.claim = None + self.depth = max(0, self.depth - 1) + if tag in OPAQUE: + self.opaque = max(0, self.opaque - 1) + + +def find_blocks(body: str) -> list[tuple[int, int]]: + """Block spans for `body`, or an empty list if it will not parse.""" + try: + finder = BlockFinder(body) + finder.feed(body) + return finder.spans + except Exception: + return [] + + +def balanced(frag: str) -> bool: + """Cheap well-formedness check for what is handed to Bergamot's HTML mode. + + Bergamot raises rather than degrading on unbalanced input, so this is the + gate that decides HTML mode versus the plain-text fallback. + """ + stack: list[str] = [] + for m in re.finditer(r"(?is)<\s*(/?)\s*([a-z0-9]+)[^>]*?(/?)\s*>", frag): + closing, tag, self_close = m.group(1), m.group(2).lower(), m.group(3) + if tag in VOID or self_close: + continue + if closing: + if not stack or stack.pop() != tag: + return False + else: + stack.append(tag) + return not stack + + +def plain(frag: str) -> str: + """Tag-stripped, entity-decoded text of a fragment.""" + return html.unescape(TAGS.sub("", frag)) + + +def plan(body: str) -> tuple[list[str], list[str], list[tuple[int, int]]]: + """Decide what to translate and how. + + Returns (jobs, kinds, spans) where kinds[i] is "html" or "text" and spans[i] + is where jobs[i] came from. Blocks with no real text are dropped entirely. + """ + jobs: list[str] = [] + kinds: list[str] = [] + spans: list[tuple[int, int]] = [] + + for start, end in find_blocks(body): + frag = body[start:end] + if not HAS_TEXT.search(plain(frag)): + continue + if balanced(frag): + jobs.append(decode_numeric_refs(frag)) + kinds.append("html") + else: + text = plain(frag).strip() + if not text: + continue + jobs.append(text) + kinds.append("text") + spans.append((start, end)) + + return jobs, kinds, spans + + +def splice(body: str, spans: list[tuple[int, int]], replacements: list[str]) -> str: + """Substitute translated blocks back into the page. + + Applied back to front so that earlier offsets stay valid as the string + length changes underneath. + """ + for (start, end), new in sorted(zip(spans, replacements), key=lambda x: -x[0][0]): + body = body[:start] + new + body[end:] + return body diff --git a/install/nomad-translate/entrypoint.sh b/install/nomad-translate/entrypoint.sh new file mode 100644 index 00000000..288ee6ec --- /dev/null +++ b/install/nomad-translate/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Fetch any missing language models, then serve. +# +# The fetch is deliberately non-fatal. A box that is offline, or whose models +# already downloaded, must still start and forward the library: translation +# being unavailable can never be allowed to make the Information Library +# unreachable. +set -u + +python /app/fetch_models.py || echo "nomad-translate: model fetch failed, continuing" >&2 + +exec python /app/proxy.py diff --git a/install/nomad-translate/fetch_models.py b/install/nomad-translate/fetch_models.py new file mode 100644 index 00000000..3ce0f012 --- /dev/null +++ b/install/nomad-translate/fetch_models.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Fetch Bergamot translation models from Firefox Remote Settings. + +Deliberately NOT `mozilla/firefox-translations-models` on GitHub: that repo was +archived on 2026-08-21 and its README now points elsewhere. Remote Settings is +the endpoint Firefox itself uses, so it stays current. + +Each language pair needs three attachments (model, lex, vocab) plus a small +marian config that we write ourselves. Models are MPL-2.0. + +Roughly 37 MB per direction. Coverage is 84 pairs across about 44 languages, +strong on European plus Hindi, Bengali, Tamil, Telugu, Vietnamese and +Indonesian. There is no Chinese, Japanese, Korean, Arabic or Thai in the tiny +model set, so those languages cannot be offered. +""" + +import json +import os +import sys +import urllib.request +from pathlib import Path + +RECORDS_URL = ( + "https://firefox.settings.services.mozilla.com/v1/buckets/main" + "/collections/translations-models/records" +) +ATTACHMENT_BASE = "https://firefox-settings-attachments.cdn.mozilla.net/" + +# Written next to the model files. Bergamot reads this, not the records. +CONFIG = """\ +relative-paths: true +models: + - model.{pair}.bin +vocabs: + - vocab.{pair}.spm + - vocab.{pair}.spm +shortlist: + - lex.{pair}.bin + - false +beam-size: 1 +normalize: 1.0 +word-penalty: 0 +max-length-break: 128 +mini-batch-words: 1024 +workspace: 128 +max-length-factor: 2.0 +skip-cost: true +cpu-threads: 0 +quiet: true +quiet-translation: true +gemm-precision: int8shiftAlphaAll +alignment: soft +""" + + +def records() -> list[dict]: + with urllib.request.urlopen(RECORDS_URL, timeout=60) as response: + return json.load(response)["data"] + + +def newest(entries: list[dict]) -> dict | None: + """Highest version wins. Remote Settings keeps older revisions around.""" + if not entries: + return None + return sorted(entries, key=lambda r: float(r.get("version", 0)))[-1] + + +def fetch_pair(data: list[dict], src: str, dst: str, out_root: Path) -> bool: + """Download one direction. Returns False if the pair is not published.""" + pair = f"{src}{dst}" + target = out_root / pair + + wanted = {} + for kind in ("model", "lex", "vocab"): + matches = [ + r + for r in data + if r.get("fileType") == kind + and r.get("fromLang") == src + and r.get("toLang") == dst + # The "tiny" variants are the ones sized for on-device use. + and "base" not in r.get("name", "") + ] + chosen = newest(matches) + if chosen is None: + print(f" {pair}: no {kind} published, skipping pair", flush=True) + return False + wanted[kind] = chosen + + target.mkdir(parents=True, exist_ok=True) + suffix = {"model": "bin", "lex": "bin", "vocab": "spm"} + + for kind, record in wanted.items(): + dest = target / f"{kind}.{pair}.{suffix[kind]}" + expected = record["attachment"]["size"] + # Resume is not worth the complexity here: a partial file is simply + # re-fetched. What matters is never leaving a truncated model in place + # that would fail cryptically at translation time. + if dest.exists() and dest.stat().st_size == expected: + continue + url = ATTACHMENT_BASE + record["attachment"]["location"] + tmp = dest.with_suffix(dest.suffix + ".part") + with urllib.request.urlopen(url, timeout=300) as response: + tmp.write_bytes(response.read()) + if tmp.stat().st_size != expected: + tmp.unlink(missing_ok=True) + raise RuntimeError( + f"{pair}/{kind}: expected {expected} bytes, got a short read" + ) + tmp.rename(dest) + + (target / "config.yml").write_text(CONFIG.format(pair=pair)) + size = sum(f.stat().st_size for f in target.iterdir()) / 1_000_000 + print(f" {pair}: ready ({size:.0f} MB)", flush=True) + return True + + +def main() -> int: + out_root = Path(os.environ.get("MODELS", "/models")) + # Comma-separated language codes to pair with English, both directions. + langs = [ + code.strip().lower() + for code in os.environ.get("TRANSLATE_LANGS", "fr,es,de").split(",") + if code.strip() + ] + + if not langs: + print("No TRANSLATE_LANGS configured, nothing to fetch.", flush=True) + return 0 + + out_root.mkdir(parents=True, exist_ok=True) + + # Skip the network entirely when every requested pair is already present, + # so a restart on a genuinely offline box does not fail. + missing = [ + code + for code in langs + if not (out_root / f"en{code}" / "config.yml").exists() + or not (out_root / f"{code}en" / "config.yml").exists() + ] + if not missing: + print(f"All {len(langs)} language pairs already present.", flush=True) + return 0 + + print(f"Fetching translation models for: {', '.join(missing)}", flush=True) + try: + data = records() + except Exception as exc: + # An offline box with some pairs already downloaded should still start + # and serve those, rather than refusing to boot. + print(f"Could not reach Firefox Remote Settings: {exc}", flush=True) + return 0 + + for code in missing: + fetch_pair(data, "en", code, out_root) + fetch_pair(data, code, "en", out_root) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/install/nomad-translate/proxy.py b/install/nomad-translate/proxy.py new file mode 100644 index 00000000..78ba1aea --- /dev/null +++ b/install/nomad-translate/proxy.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Path-preserving translating proxy in front of kiwix-serve. + +Everything is forwarded byte for byte on the SAME paths Kiwix uses, so /viewer, +/skin/*, /search, /random and /catalog/* are genuinely Kiwix's own shell running +Kiwix's own JavaScript. Only `text/html` under /content/ is rewritten, and only +its text. + +Preserving the path is the whole design. viewer.js keeps the content iframe in +sync with location.hash by comparing `contentWindow.location.pathname`, so +pointing the iframe anywhere else starts the ping-pong its own comment warns +about, and we would be patching Kiwix's JavaScript and re-verifying it on every +Kiwix bump. Because the path never moves, no Kiwix JS is modified at all, and +Kiwix's header, search, library and random keep working because they ARE Kiwix. + +Language lives in a cookie set by a control injected into the translated page, +so links inside the ZIM need no rewriting either. +""" + +import http.cookies +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import blocks + +UPSTREAM = os.environ.get("KIWIX", "http://localhost:8090").rstrip("/") +MODELS = Path(os.environ.get("MODELS", "/models")) +WORKERS = int(os.environ.get("WORKERS", "8")) +PORT = int(os.environ.get("PORT", "8391")) +# Pages larger than this are forwarded untranslated. A pathological article +# should not be able to tie up every worker or exhaust memory. +MAX_TRANSLATE_BYTES = int(os.environ.get("MAX_TRANSLATE_BYTES", str(4 * 1024 * 1024))) + +COOKIE = "nomadlang" + +# Display names for the languages Bergamot's tiny model set covers. A language +# only appears in the control if its model is actually on disk. +LANG_NAMES = { + "bg": "Bulgarian", "bn": "Bengali", "cs": "Czech", "da": "Danish", + "de": "German", "el": "Greek", "es": "Spanish", "et": "Estonian", + "fa": "Persian", "fi": "Finnish", "fr": "French", "he": "Hebrew", + "hi": "Hindi", "hu": "Hungarian", "id": "Indonesian", "is": "Icelandic", + "it": "Italian", "lt": "Lithuanian", "lv": "Latvian", "nb": "Norwegian", + "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "ro": "Romanian", + "ru": "Russian", "sk": "Slovak", "sl": "Slovenian", "sr": "Serbian", + "sv": "Swedish", "ta": "Tamil", "te": "Telugu", "tr": "Turkish", + "uk": "Ukrainian", "vi": "Vietnamese", +} + +# Hop-by-hop headers must not be forwarded. Content-Length and Content-Encoding +# are dropped too because the body length changes when a page is translated. +HOP = { + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailers", "transfer-encoding", "upgrade", "content-length", + "content-encoding", +} + + +def available_languages() -> dict[str, str]: + """Languages with an en->xx model present on disk, in display order.""" + found = {} + if MODELS.is_dir(): + for entry in sorted(MODELS.iterdir()): + if not (entry / "config.yml").is_file(): + continue + pair = entry.name + if len(pair) == 4 and pair.startswith("en"): + code = pair[2:] + found[code] = LANG_NAMES.get(code, code.upper()) + return found + + +LANGS = available_languages() + +# Bergamot is imported lazily so the proxy can still boot, and still forward +# Kiwix untouched, on a box where the models never downloaded. +_service = None +_models: dict[str, object] = {} + + +def _bergamot(): + global _service + if _service is None: + from bergamot import Service, ServiceConfig + + _service = Service(ServiceConfig(numWorkers=WORKERS, logLevel="off")) + return _service + + +def model(pair: str): + if pair not in _models: + _models[pair] = _bergamot().modelFromConfigPath(str(MODELS / pair / "config.yml")) + return _models[pair] + + +def translate_blocks(pair: str, jobs: list[str]) -> list[str]: + from bergamot import ResponseOptions, VectorString + + options = ResponseOptions(HTML=True) + responses = _bergamot().translate(model(pair), VectorString(jobs), options) + return [r.target.text for r in responses] + + +BAR_CSS = """ +#nomad-tr{position:sticky;top:0;z-index:2147483000;display:flex;align-items:center; + flex-wrap:wrap;gap:9px;padding:7px 14px;background:#36361A;color:#F1E6C8; + font:13px/1.4 ui-sans-serif,system-ui,-apple-system,Segoe UI,sans-serif} +#nomad-tr .t{opacity:.75} +#nomad-tr button{font:inherit;font-size:12.5px;color:#F1E6C8;background:transparent; + border:1px solid #66663F;border-radius:999px;padding:3px 11px;cursor:pointer} +#nomad-tr button:hover{background:#66663F} +#nomad-tr button.on{background:#F1E6C8;color:#36361A;border-color:#F1E6C8;font-weight:600} +#nomad-tr .g{flex:1} +#nomad-tr .s{opacity:.6;font-size:11.5px;font-variant-numeric:tabular-nums} +""" + +# Set the cookie client side and reload, so the control works from inside the +# Kiwix content iframe without navigating the outer frame. +BAR_JS = """ +function nomadSetLang(c){ + document.cookie='%s='+c+';path=/;max-age=31536000'; + location.reload(); +} +""" % COOKIE + + +def build_bar(lang: str, status: str) -> str: + """The in-page language control. + + Labelled "Translate this page" rather than with a globe or a bare language + name, deliberately: Kiwix's own toolbar already has a globe that switches + the INTERFACE language, and the two sit close together with no visual + relationship. Naming ours after what it acts on is the cheapest way to stop + a user reaching for the wrong one and concluding the feature is broken. + """ + buttons = [] + for code, name in [("", "Original")] + sorted(LANGS.items(), key=lambda kv: kv[1]): + classes = "on" if code == lang else "" + label = name + buttons.append( + f'<button class="{classes}" onclick="nomadSetLang(\'{code}\')">{label}</button>' + ) + return ( + f'<div id="nomad-tr"><span class="t">Translate this page</span>{"".join(buttons)}' + f'<span class="g"></span><span class="s">{status}</span></div>' + f"<style>{BAR_CSS}</style><script>{BAR_JS}</script>" + ) + + +def rewrite(body: str, lang: str) -> str: + """Translate block text in place and inject the control.""" + status = "not translated" + + if lang: + started = time.time() + jobs, kinds, spans = blocks.plan(body) + if jobs: + import html as html_mod + + words = sum(len(blocks.plain(job).split()) for job in jobs) + done = translate_blocks("en" + lang, jobs) + out = [ + blocks.FONTS.sub("", text) if kind == "html" else html_mod.escape(text) + for kind, text in zip(kinds, done) + ] + body = blocks.splice(body, spans, out) + elapsed = round((time.time() - started) * 1000) + status = f"{words:,} words · {len(jobs)} blocks · {elapsed:,} ms" + else: + status = "nothing to translate" + + bar = build_bar(lang, status) + if re.search(r"(?i)<body[^>]*>", body): + return re.sub(r"(?i)(<body[^>]*>)", lambda m: m.group(1) + bar, body, count=1) + return bar + body + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "nomad-translate-proxy" + + def log_message(self, *args): + # The upstream Kiwix container already logs requests. + pass + + def _lang(self) -> str: + raw = self.headers.get("Cookie", "") + try: + jar = http.cookies.SimpleCookie(raw) + except Exception: + return "" + value = jar[COOKIE].value if COOKIE in jar else "" + return value if value in LANGS else "" + + def do_GET(self, body_only: bool = False): + # Setting the language via a URL makes a given language linkable and + # bookmarkable rather than only reachable by clicking the control. + if self.path.startswith("/nomad-lang"): + query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + code = query.get("set", [""])[0] + code = code if code in LANGS else "" + target = query.get("to", ["/viewer"])[0] + # Only same-origin paths. Without this the proxy is an open + # redirector that any page could bounce a user through. + if not target.startswith("/") or target.startswith("//"): + target = "/viewer" + self.send_response(302) + self.send_header("Set-Cookie", f"{COOKIE}={code}; Path=/; Max-Age=31536000; SameSite=Lax") + self.send_header("Location", target) + self.send_header("Content-Length", "0") + self.end_headers() + return + + request = urllib.request.Request(UPSTREAM + self.path, method="GET") + for header in ("Range", "If-None-Match", "If-Modified-Since", "User-Agent", "Accept"): + if self.headers.get(header): + request.add_header(header, self.headers[header]) + # Accept-Encoding is deliberately not forwarded: we want plain bytes so + # HTML can be rewritten without decompressing first. + + try: + response = urllib.request.urlopen(request, timeout=120) + status, headers, data = response.status, response.headers, response.read() + except urllib.error.HTTPError as exc: + # Forwarded as-is, which is how Kiwix's own 404 and 400 pages keep + # coming back byte for byte. + status, headers, data = exc.code, exc.headers, exc.read() + except Exception as exc: + message = f"Translation proxy could not reach the library: {exc}".encode() + self.send_response(502) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(message))) + self.end_headers() + self.wfile.write(message) + return + + content_type = headers.get("Content-Type", "") + translatable = ( + self.path.startswith("/content/") + and "text/html" in content_type.lower() + and status == 200 + and len(data) <= MAX_TRANSLATE_BYTES + ) + + lang = self._lang() if translatable else "" + if translatable: + try: + data = rewrite(data.decode("utf-8", "ignore"), lang).encode("utf-8") + except Exception as exc: + # Never take the page down over a translation failure. The + # reader gets the original article and a comment explaining why. + data = data + f"<!-- nomad-translate failed: {exc} -->".encode() + + self.send_response(status) + for key, value in headers.items(): + if key.lower() in HOP: + continue + # A translated page varies by the language cookie, so Kiwix's + # validators cannot be forwarded unchanged. Without this the browser + # caches the English article for an hour and serves it straight back + # when the reader picks a language, which looks exactly like the + # feature not working. The ETag is namespaced by language so + # revalidation still works per language rather than being disabled. + if translatable and key.lower() == "etag": + value = f'{value.rstrip()}-nomadlang-{lang or "orig"}' + self.send_header(key, value) + if translatable: + self.send_header("Vary", "Cookie") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if not body_only: + try: + self.wfile.write(data) + except (BrokenPipeError, ConnectionResetError): + pass + + def do_HEAD(self): + self.do_GET(body_only=True) + + +if __name__ == "__main__": + if LANGS: + print( + f"nomad-translate: {len(LANGS)} languages available " + f"({', '.join(sorted(LANGS))})", + flush=True, + ) + else: + print( + "nomad-translate: no models found, forwarding the library untranslated", + file=sys.stderr, + flush=True, + ) + + print(f"nomad-translate: listening on 0.0.0.0:{PORT} -> {UPSTREAM}", flush=True) + ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() diff --git a/install/nomad-translate/test_blocks.py b/install/nomad-translate/test_blocks.py new file mode 100644 index 00000000..0d98e96b --- /dev/null +++ b/install/nomad-translate/test_blocks.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Tests for the block planner. + +Pure functions only, no Bergamot and no network, so this runs anywhere: + + python3 install/nomad-translate/test_blocks.py + +Each case here is a failure the prototype actually hit. HTML mode raises +"Not all tags were closed" rather than degrading, so anything that gets the +balance check wrong takes the whole article down. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +import blocks # noqa: E402 + +failures: list[str] = [] + + +def check(name: str, got, want): + if got != want: + failures.append(f"{name}\n expected: {want!r}\n got: {got!r}") + + +def check_true(name: str, got): + check(name, bool(got), True) + + +def check_false(name: str, got): + check(name, bool(got), False) + + +# --- balance check ----------------------------------------------------------- + +check_true("balanced: plain text", blocks.balanced("just words")) +check_true("balanced: matched inline tags", blocks.balanced('<a href="X">water</a>')) +check_true("balanced: nested inline tags", blocks.balanced("<b><i>x</i></b>")) +check_true("balanced: void tag needs no close", blocks.balanced("a<br>b")) +check_true("balanced: self-closing", blocks.balanced('<img src="x"/>')) +check_false("balanced: unclosed tag", blocks.balanced("<b>x")) +check_false("balanced: stray close", blocks.balanced("x</b>")) +check_false("balanced: crossed tags", blocks.balanced("<b><i>x</b></i>")) + +# --- block finding ----------------------------------------------------------- + +# The nesting case regex cannot handle: a non-greedy <li>...</li> stops at the +# INNER list's closing tag, silently truncating the block. +NESTED = "<ul><li>outer <ul><li>inner</li></ul> tail</li></ul>" +jobs, kinds, spans = blocks.plan(NESTED) +check("nested li: one outermost block claimed", len(jobs), 1) +check_true("nested li: keeps the whole outer li", "tail" in jobs[0]) +check_true("nested li: contains the inner list", "inner" in jobs[0]) + +# Opaque elements must never be descended into. +check("script is opaque", blocks.plan("<script><p>var x</p></script>")[0], []) +check("style is opaque", blocks.plan("<style><p>a{}</p></style>")[0], []) +check("pre is opaque", blocks.plan("<pre><p>code</p></pre>")[0], []) +# Tables are excluded wholesale to dodge the nesting problem, which is why +# infobox rows stay in the source language. +check("table is opaque", blocks.plan("<table><tr><td>cell text</td></tr></table>")[0], []) + +# Blocks with no real text are not worth a round trip. +check("digits only are skipped", blocks.plan("<p>12345</p>")[0], []) +check("punctuation only is skipped", blocks.plan("<p>--- .</p>")[0], []) +check("single letter is skipped", blocks.plan("<p>a</p>")[0], []) +check("two letters count as text", blocks.plan("<p>ok</p>")[0], ["ok"]) + +# Crossed inline markup inside a block that DOES close: the span is found, the +# balance check rejects it, and it falls back to plain text. Losing the inline +# tags is the price of not having Bergamot raise on the whole article. +jobs, kinds, _ = blocks.plan("<p><b><i>hello</b></i></p>") +check("crossed inline markup falls back to text", kinds, ["text"]) +check("fallback strips the tags", jobs, ["hello"]) + +# A block whose own closing tag is unreachable because an inner tag was never +# closed is DROPPED, not translated. handle_endtag sees the wrong depth, so no +# span is ever claimed and the balance check is never consulted. +# +# This is safe (the reader gets the original) but silent: that paragraph simply +# stays in the source language with nothing to indicate why. Worth knowing when +# a page comes back partly translated. +check("block with an unclosed inner tag is dropped", blocks.plan("<p>see <b>this</p>")[0], []) + +# Balanced markup goes through HTML mode so the aligner can place inline tags. +jobs, kinds, _ = blocks.plan('<p>drink <a href="Water">water</a></p>') +check("balanced block uses html mode", kinds, ["html"]) +check_true("html mode keeps the anchor", "<a href=" in jobs[0]) + +# --- splice ------------------------------------------------------------------ + +body = "<p>one</p><p>two</p>" +jobs, kinds, spans = blocks.plan(body) +check("two paragraphs found", jobs, ["one", "two"]) +# Back-to-front application is what keeps the earlier offsets valid. +check( + "splice replaces both, longer than the source", + blocks.splice(body, spans, ["un ONE", "deux TWO"]), + "<p>un ONE</p><p>deux TWO</p>", +) + +# --- plain ------------------------------------------------------------------- + +check("plain strips tags", blocks.plain("<b>hi</b> there"), "hi there") +check("plain decodes entities", blocks.plain("café"), "café") + +# --- malformed input --------------------------------------------------------- + +# A page that will not parse must yield no work rather than raising: the reader +# gets the untranslated article instead of an error. +check("garbage yields no blocks", blocks.plan("<<<>>><p")[0], []) +check("empty document yields no blocks", blocks.plan("")[0], []) + + + + +# --- leaf-div claiming ------------------------------------------------------- +# +# Real prose lives in a bare <div> often enough that excluding them lost whole +# pages: sotoki StackExchange ZIMs put every question excerpt in one, so the +# titles translated while the descriptions under them did not. + +check( + "leaf div is claimed", + blocks.plan('<div class="excerpt">Why do people still use Morse code?</div>')[0], + ["Why do people still use Morse code?"], +) + +# A wrapper must yield to the blocks inside it rather than swallowing the page. +check( + "layout div yields to its children", + blocks.plan('<div class="wrap"><p>First para.</p><p>Second para.</p></div>')[0], + ["First para.", "Second para."], +) + +check( + "nested divs claim the innermost text", + blocks.plan('<div><div><div class="excerpt">Inner text.</div></div></div>')[0], + ["Inner text."], +) + +# Inline tags stay inside the job so the aligner can re-place them. +check( + "leaf div keeps inline markup", + blocks.plan('<div>See <a href="X">the antenna</a> guide.</div>')[0], + ['See <a href="X">the antenna</a> guide.'], +) + +check( + "div inside an opaque table is still skipped", + blocks.plan("<table><tr><td><div>Cell text.</div></td></tr></table>")[0], + [], +) + +check( + "textless div is dropped", + blocks.plan('<div class="spacer"> </div><div>42</div>')[0], + [], +) + + + + +# --- numeric character references -------------------------------------------- +# +# Bergamot decodes named entities but not numeric ones: it treats `'` as +# literal text and escapes the ampersand on output, so an apostrophe came back +# as the visible string `'`. Reported against the ham radio ZIM. + +check( + "hex charref is decoded before translation", + blocks.plan("<p>If I'm here</p>")[0], + ["If I'm here"], +) + +check( + "decimal charref is decoded before translation", + blocks.plan("<p>If I'm here</p>")[0], + ["If I'm here"], +) + +check( + "non-ascii charref is decoded", + blocks.plan("<p>café here</p>")[0], + ["café here"], +) + +# Decoding these would inject real markup and break the balance check that gates +# HTML mode, dropping the block to plain text and losing its links. +check( + "markup-significant charrefs stay encoded", + blocks.plan("<p>Compare <dipole> designs & feeds</p>")[0], + ["Compare <dipole> designs & feeds"], +) + +check( + "named entities are left alone", + blocks.plan("<p>Tom & Jerry <tag></p>")[0], + ["Tom & Jerry <tag>"], +) + +check( + "malformed charrefs are left untouched", + blocks.decode_numeric_refs("bare &# and &#xZZ; and � here"), + "bare &# and &#xZZ; and � here", +) + +check( + "helper decodes hex and decimal, leaves markup chars", + blocks.decode_numeric_refs("' ' é < &"), + "' ' é < &", +) + +if failures: + print(f"FAIL: {len(failures)} of the checks above did not hold\n") + for failure in failures: + print(f" {failure}\n") + sys.exit(1) + +print("ok: all block planner checks passed")