mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 13:19:18 +02:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
54ee30a449 | ||
|
|
0dbbd21c2f | ||
|
|
f257d54a29 | ||
|
|
26f40baf58 | ||
|
|
ee203df485 | ||
|
|
eef5b6cd37 | ||
|
|
a483e15f7d | ||
|
|
5e1702efeb | ||
|
|
b54cb55c1a |
|
|
@ -22,7 +22,6 @@ import { NOMAD_API_DEFAULT_BASE_URL } from '../../constants/misc.js'
|
|||
import KVStore from '#models/kv_store'
|
||||
import type { ModelCapabilities } from '../utils/model_capabilities.js'
|
||||
import {
|
||||
capabilitiesFromOllamaShow,
|
||||
installedModelsFromOpenAIResponse,
|
||||
resolveModelCapabilities,
|
||||
} from '../utils/model_capabilities.js'
|
||||
|
|
@ -928,23 +927,14 @@ export class OllamaService {
|
|||
await this._ensureDependencies()
|
||||
if (!this.baseUrl) return { thinking: false, vision: 'unknown' }
|
||||
|
||||
// Composite OpenAI-compatible routers can advertise different capabilities for each model.
|
||||
// Prefer that per-model metadata before probing backend-specific endpoints.
|
||||
const advertised = capabilitiesFromOllamaShow(advertisedMetadata)
|
||||
if (advertised) {
|
||||
this.modelCapabilityCache.set(modelName, {
|
||||
value: advertised,
|
||||
expiresAt: Number.POSITIVE_INFINITY,
|
||||
})
|
||||
return advertised
|
||||
}
|
||||
|
||||
const cached = this.modelCapabilityCache.get(modelName)
|
||||
if (cached && cached.expiresAt > Date.now()) return cached.value
|
||||
|
||||
// Probe per model instead of relying on the backend-wide classification from getModels().
|
||||
// Composite OpenAI-compatible routers can advertise different capabilities for each
|
||||
// model, and that metadata is passed through here. A list that omits vision is not
|
||||
// trusted on its own (see resolveModelCapabilities), so the model is still probed.
|
||||
// Hybrid routers may expose /v1/models plus /api/show without exposing /api/tags.
|
||||
const detected = await resolveModelCapabilities(null, {
|
||||
const detected = await resolveModelCapabilities(advertisedMetadata, {
|
||||
ollamaShow: async () => {
|
||||
const response = await axios.post(
|
||||
`${this.baseUrl}/api/show`,
|
||||
|
|
|
|||
|
|
@ -49,7 +49,11 @@ export async function resolveModelCapabilities(
|
|||
probes: ModelCapabilityProbes
|
||||
): Promise<ModelCapabilities | null> {
|
||||
const advertised = capabilitiesFromOllamaShow(advertisedMetadata)
|
||||
if (advertised) return advertised
|
||||
// A model list that reports vision is trusted outright. One that does not is
|
||||
// not conclusive: Ollama's /api/tags lists gemma3 as ['completion'] while its
|
||||
// /api/show for the same model reports ['completion', 'vision'], which left
|
||||
// image upload disabled for one of the most common small vision models.
|
||||
if (advertised?.vision === 'supported') return advertised
|
||||
|
||||
try {
|
||||
const ollama = capabilitiesFromOllamaShow(await probes.ollamaShow())
|
||||
|
|
@ -58,10 +62,10 @@ export async function resolveModelCapabilities(
|
|||
|
||||
try {
|
||||
const vision = visionCapabilityFromLlamaProps(await probes.llamaProps())
|
||||
if (vision) return { thinking: false, vision }
|
||||
if (vision) return { thinking: advertised?.thinking ?? false, vision }
|
||||
} catch {}
|
||||
|
||||
return null
|
||||
return advertised
|
||||
}
|
||||
|
||||
export function installedModelsFromOpenAIResponse(value: unknown): InstalledModelMetadata[] {
|
||||
|
|
|
|||
|
|
@ -550,7 +550,7 @@ export default class ServiceSeeder extends BaseSeeder {
|
|||
description:
|
||||
'Read the Information Library in another language. Machine translation that works offline, on CPU',
|
||||
icon: 'IconWorld',
|
||||
container_image: 'ghcr.io/crosstalk-solutions/project-nomad-translate:0.1.0',
|
||||
container_image: 'ghcr.io/crosstalk-solutions/project-nomad-translate:0.1.1',
|
||||
source_repo: 'https://github.com/browsermt/bergamot-translator',
|
||||
container_command: null,
|
||||
container_config: JSON.stringify({
|
||||
|
|
@ -561,7 +561,7 @@ export default class ServiceSeeder extends BaseSeeder {
|
|||
},
|
||||
ExposedPorts: { '8391/tcp': {} },
|
||||
// TRANSLATE_LANGS is the language set fetched on first start, about
|
||||
// 74 MB per language for the pair in both directions. Editable via
|
||||
// 45-140 MB per language for the pair in both directions. Editable via
|
||||
// Manage > Edit; the container re-checks on restart and only fetches
|
||||
// what is missing.
|
||||
Env: [
|
||||
|
|
@ -584,7 +584,7 @@ export default class ServiceSeeder extends BaseSeeder {
|
|||
// has to be there first.
|
||||
depends_on: SERVICE_NAMES.KIWIX,
|
||||
// Peak RSS measured at 729 MB with three language pairs resident; models
|
||||
// are about 74 MB per language on disk.
|
||||
// are about 45-140 MB per language on disk.
|
||||
metadata: JSON.stringify({ minMemoryMB: 1536, minDiskMB: 1024 }),
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,17 @@
|
|||
# Release Notes
|
||||
|
||||
## Unreleased
|
||||
## Version 1.35.1 - October 5, 2026
|
||||
|
||||
### Features
|
||||
|
||||
### Bug Fixes
|
||||
- **AI**: probe a model the list reports without vision before disabling image attachments, so `gemma3` accepts images again (#1409). Thanks @chriscrosstalk for the contribution!
|
||||
- **Translate**: forward upstream redirects instead of following them, so books open styled and their article links resolve (#1379). Thanks @chriscrosstalk for the contribution!
|
||||
- **Translate**: download every released language instead of crashing on pre-release models, and list all supported languages in the docs (#1405). Thanks @chriscrosstalk for the contribution!
|
||||
|
||||
### Improvements
|
||||
|
||||
## Version 1.35.0 - September 29, 2026
|
||||
|
||||
### Features
|
||||
- **AI**: support local vision attachments in chat (#1288). Thanks @gabegraves and @chriscrosstalk for the contribution!
|
||||
|
|
|
|||
|
|
@ -289,7 +289,69 @@ Reads the Information Library in another language. Open an article and a **Trans
|
|||
|
||||
**Why this instead of the AI Assistant:** the AI Assistant can translate, but this is roughly 1,600 times faster on the same machine, and it does not need a graphics card, so it works on every NOMAD. It is also more careful with names: asked to translate a page, the AI Assistant will happily translate "Project NOMAD" into another language, and this will not.
|
||||
|
||||
**Choosing languages:** French, Spanish and German are set up by default. To change that, use **Manage > Edit** and set `TRANSLATE_LANGS` to a comma-separated list of language codes, for example `fr,it,pt`. Each language is about 74 MB, and new ones download the next time the app restarts. Around 40 languages are available, including Hindi, Bengali, Tamil, Telugu, Vietnamese and Indonesian. **Chinese, Japanese, Korean, Arabic and Thai are not available**, because no compact model exists for them yet.
|
||||
**Choosing languages:** French, Spanish and German are set up by default. To add or remove languages:
|
||||
|
||||
1. In the **Supply Depot**, open the Translated Library's **Manage** menu and choose **Edit**.
|
||||
2. Under **Environment Variables**, find `TRANSLATE_LANGS=fr,es,de`.
|
||||
3. Change the list of language codes, separated by commas. For example, `TRANSLATE_LANGS=fr,es,de,sv` adds Swedish.
|
||||
4. Click **Save**. The app restarts, downloads any new languages, and adds a button for each one to the **Translate this page** bar.
|
||||
|
||||
Adding a language needs an internet connection, and each one takes between about 45 MB and 140 MB of disk. Languages you already have keep working offline. Removing a code from the list does not remove a language you have already downloaded; it stays in `storage/translate/models` and keeps its button.
|
||||
|
||||
These 50 languages are available. Chinese is not available yet.
|
||||
|
||||
| Language | Code |
|
||||
|---|---|
|
||||
| Afrikaans | `af` |
|
||||
| Arabic | `ar` |
|
||||
| Basque | `eu` |
|
||||
| Bengali | `bn` |
|
||||
| Bosnian | `bs` |
|
||||
| Bulgarian | `bg` |
|
||||
| Catalan | `ca` |
|
||||
| Croatian | `hr` |
|
||||
| Czech | `cs` |
|
||||
| Danish | `da` |
|
||||
| Dutch | `nl` |
|
||||
| Estonian | `et` |
|
||||
| Finnish | `fi` |
|
||||
| French | `fr` |
|
||||
| Galician | `gl` |
|
||||
| German | `de` |
|
||||
| Greek | `el` |
|
||||
| Gujarati | `gu` |
|
||||
| Hebrew | `he` |
|
||||
| Hindi | `hi` |
|
||||
| Hungarian | `hu` |
|
||||
| Icelandic | `is` |
|
||||
| Indonesian | `id` |
|
||||
| Italian | `it` |
|
||||
| Japanese | `ja` |
|
||||
| Kannada | `kn` |
|
||||
| Korean | `ko` |
|
||||
| Latvian | `lv` |
|
||||
| Lithuanian | `lt` |
|
||||
| Malay | `ms` |
|
||||
| Malayalam | `ml` |
|
||||
| Marathi | `mr` |
|
||||
| Norwegian | `nb` |
|
||||
| Persian | `fa` |
|
||||
| Polish | `pl` |
|
||||
| Portuguese | `pt` |
|
||||
| Romanian | `ro` |
|
||||
| Russian | `ru` |
|
||||
| Serbian | `sr` |
|
||||
| Slovak | `sk` |
|
||||
| Slovenian | `sl` |
|
||||
| Spanish | `es` |
|
||||
| Swedish | `sv` |
|
||||
| Tamil | `ta` |
|
||||
| Telugu | `te` |
|
||||
| Thai | `th` |
|
||||
| Turkish | `tr` |
|
||||
| Ukrainian | `uk` |
|
||||
| Urdu | `ur` |
|
||||
| Vietnamese | `vi` |
|
||||
|
||||
**First start needs internet.** The language models download when the app first runs, the same as installing any other app. After that it is entirely offline. If you install this while disconnected the app still starts and the library still works, just without translation until it can fetch the models.
|
||||
|
||||
|
|
|
|||
|
|
@ -164,8 +164,26 @@ test('resolves per-model capabilities advertised by a hybrid router', async () =
|
|||
|
||||
assert.deepEqual(visionModel, { thinking: true, vision: 'supported' })
|
||||
assert.deepEqual(textModel, { thinking: true, vision: 'unsupported' })
|
||||
assert.equal(showProbeCalls, 0)
|
||||
assert.equal(propsProbeCalls, 0)
|
||||
// A listed vision model is trusted without probing. A listed text-only model is
|
||||
// probed once, and the router's answer stands when the probes have nothing better.
|
||||
assert.equal(showProbeCalls, 1)
|
||||
assert.equal(propsProbeCalls, 1)
|
||||
})
|
||||
|
||||
test('probes a model the list reports without vision, as Ollama does for gemma3', async () => {
|
||||
// Measured on Ollama 0.33.3: /api/tags lists gemma3:4b as ['completion'] while
|
||||
// /api/show for the same model reports ['completion', 'vision'].
|
||||
const gemma = await resolveModelCapabilities(
|
||||
{ capabilities: ['completion'] },
|
||||
{
|
||||
ollamaShow: async () => ({ capabilities: ['completion', 'vision'] }),
|
||||
llamaProps: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert.deepEqual(gemma, { thinking: false, vision: 'supported' })
|
||||
})
|
||||
|
||||
test('preserves per-model capabilities from an OpenAI-compatible model list', () => {
|
||||
|
|
@ -243,9 +261,7 @@ test('classifies the four supported backend capability cases', async () => {
|
|||
{
|
||||
name: 'known text-only',
|
||||
advertised: { capabilities: ['completion', 'thinking'] },
|
||||
ollamaShow: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
ollamaShow: async () => ({ capabilities: ['completion', 'thinking'] }),
|
||||
llamaProps: async () => {
|
||||
throw new Error('not used')
|
||||
},
|
||||
|
|
|
|||
|
|
@ -97,11 +97,11 @@
|
|||
},
|
||||
{
|
||||
"id": "librepathology_en_all_maxi",
|
||||
"version": "2025-09",
|
||||
"version": "2026-09",
|
||||
"title": "LibrePathology",
|
||||
"description": "Pathology reference for disease identification",
|
||||
"url": "https://download.kiwix.org/zim/other/librepathology_en_all_maxi_2025-09.zim",
|
||||
"size_mb": 76
|
||||
"url": "https://download.kiwix.org/zim/other/librepathology_en_all_maxi_2026-09.zim",
|
||||
"size_mb": 95
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -145,20 +145,20 @@
|
|||
"size_mb": 27
|
||||
},
|
||||
{
|
||||
"id": "canadian_prepper_winterprepping_en",
|
||||
"version": "2026-02",
|
||||
"id": "canadian-prepper_en_winterprepping",
|
||||
"version": "2026-08",
|
||||
"title": "Canadian Prepper: Winter Prepping",
|
||||
"description": "Video guides for winter survival and cold weather emergencies",
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian_prepper_winterprepping_en_2026-02.zim",
|
||||
"size_mb": 1340
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian-prepper_en_winterprepping_2026-08.zim",
|
||||
"size_mb": 1342
|
||||
},
|
||||
{
|
||||
"id": "canadian_prepper_bugoutroll_en",
|
||||
"version": "2025-08",
|
||||
"id": "canadian-prepper_en_bugoutroll",
|
||||
"version": "2026-07",
|
||||
"title": "Canadian Prepper: Bug Out Roll",
|
||||
"description": "Essential gear selection for your bug-out bag",
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutroll_en_2025-08.zim",
|
||||
"size_mb": 975
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian-prepper_en_bugoutroll_2026-07.zim",
|
||||
"size_mb": 879
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -193,20 +193,20 @@
|
|||
"size_mb": 75
|
||||
},
|
||||
{
|
||||
"id": "canadian_prepper_bugoutconcepts_en",
|
||||
"version": "2026-02",
|
||||
"id": "canadian-prepper_en_bugoutconcepts",
|
||||
"version": "2026-08",
|
||||
"title": "Canadian Prepper: Bug Out Concepts",
|
||||
"description": "Strategies and planning for emergency evacuation",
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian_prepper_bugoutconcepts_en_2026-02.zim",
|
||||
"size_mb": 2890
|
||||
"url": "https://download.kiwix.org/zim/videos/canadian-prepper_en_bugoutconcepts_2026-08.zim",
|
||||
"size_mb": 2886
|
||||
},
|
||||
{
|
||||
"id": "urban-prepper_en_all",
|
||||
"version": "2026-02",
|
||||
"version": "2026-08",
|
||||
"title": "Urban Prepper",
|
||||
"description": "Comprehensive urban emergency preparedness video series",
|
||||
"url": "https://download.kiwix.org/zim/videos/urban-prepper_en_all_2026-02.zim",
|
||||
"size_mb": 2240
|
||||
"url": "https://download.kiwix.org/zim/videos/urban-prepper_en_all_2026-08.zim",
|
||||
"size_mb": 2245
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -291,11 +291,11 @@
|
|||
"resources": [
|
||||
{
|
||||
"id": "ted_mul_ted-ed",
|
||||
"version": "2026-01",
|
||||
"version": "2026-09",
|
||||
"title": "TED-Ed",
|
||||
"description": "Educational video lessons on science, history, literature, and more",
|
||||
"url": "https://download.kiwix.org/zim/ted/ted_mul_ted-ed_2026-01.zim",
|
||||
"size_mb": 5610
|
||||
"url": "https://download.kiwix.org/zim/ted/ted_mul_ted-ed_2026-09.zim",
|
||||
"size_mb": 6381
|
||||
},
|
||||
{
|
||||
"id": "wikiversity_en_all_maxi",
|
||||
|
|
@ -355,11 +355,11 @@
|
|||
},
|
||||
{
|
||||
"id": "ted_mul_ted-conference",
|
||||
"version": "2026-02",
|
||||
"version": "2026-09",
|
||||
"title": "TED Conference",
|
||||
"description": "Main TED conference talks on ideas worth spreading",
|
||||
"url": "https://download.kiwix.org/zim/ted/ted_mul_ted-conference_2026-02.zim",
|
||||
"size_mb": 16500
|
||||
"url": "https://download.kiwix.org/zim/ted/ted_mul_ted-conference_2026-09.zim",
|
||||
"size_mb": 18136
|
||||
},
|
||||
{
|
||||
"id": "libretexts.org_en_human",
|
||||
|
|
@ -485,10 +485,10 @@
|
|||
},
|
||||
{
|
||||
"id": "based.cooking_en_all",
|
||||
"version": "2026-02",
|
||||
"version": "2026-08",
|
||||
"title": "Based.Cooking",
|
||||
"description": "Simple, practical recipes from the community",
|
||||
"url": "https://download.kiwix.org/zim/zimit/based.cooking_en_all_2026-02.zim",
|
||||
"url": "https://download.kiwix.org/zim/zimit/based.cooking_en_all_2026-08.zim",
|
||||
"size_mb": 16
|
||||
}
|
||||
]
|
||||
|
|
@ -566,18 +566,18 @@
|
|||
"resources": [
|
||||
{
|
||||
"id": "freecodecamp_en_all",
|
||||
"version": "2026-02",
|
||||
"version": "2026-08",
|
||||
"title": "freeCodeCamp",
|
||||
"description": "Interactive programming tutorials - JavaScript, algorithms, and data structures",
|
||||
"url": "https://download.kiwix.org/zim/freecodecamp/freecodecamp_en_all_2026-02.zim",
|
||||
"url": "https://download.kiwix.org/zim/freecodecamp/freecodecamp_en_all_2026-08.zim",
|
||||
"size_mb": 8
|
||||
},
|
||||
{
|
||||
"id": "devdocs_en_python",
|
||||
"version": "2026-02",
|
||||
"version": "2026-08",
|
||||
"title": "Python Documentation",
|
||||
"description": "Complete Python language reference and tutorials",
|
||||
"url": "https://download.kiwix.org/zim/devdocs/devdocs_en_python_2026-02.zim",
|
||||
"url": "https://download.kiwix.org/zim/devdocs/devdocs_en_python_2026-08.zim",
|
||||
"size_mb": 4
|
||||
},
|
||||
{
|
||||
|
|
@ -630,10 +630,10 @@
|
|||
},
|
||||
{
|
||||
"id": "devdocs_en_node",
|
||||
"version": "2026-02",
|
||||
"version": "2026-08",
|
||||
"title": "Node.js Documentation",
|
||||
"description": "Node.js API reference and guides",
|
||||
"url": "https://download.kiwix.org/zim/devdocs/devdocs_en_node_2026-02.zim",
|
||||
"url": "https://download.kiwix.org/zim/devdocs/devdocs_en_node_2026-08.zim",
|
||||
"size_mb": 1
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -32,9 +32,11 @@ Reimplementing or wrapping the Kiwix reading view is explicitly out of scope.
|
|||
| `blocks.py` | Finds translatable blocks; decides HTML mode versus plain text |
|
||||
| `fetch_models.py` | Pulls models from Firefox Remote Settings on first start |
|
||||
| `test_blocks.py` | Tests for the block planner. No Bergamot, no network |
|
||||
| `test_fetch_models.py` | Tests for which model files get downloaded. No network |
|
||||
| `entrypoint.sh` | Fetch models, then serve. Fetch failure is non-fatal |
|
||||
|
||||
Run the tests with `python3 install/nomad-translate/test_blocks.py`.
|
||||
Run the tests with `python3 install/nomad-translate/test_blocks.py` and
|
||||
`python3 install/nomad-translate/test_fetch_models.py`.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
|
@ -53,9 +55,11 @@ From **Firefox Remote Settings**, the endpoint Firefox itself uses, not the
|
|||
`mozilla/firefox-translations-models` GitHub repo, which was archived on
|
||||
2026-08-21. MPL-2.0.
|
||||
|
||||
About 37 MB per direction, 74 MB for a language in both directions. 84 pairs
|
||||
across roughly 44 languages. **No Chinese, Japanese, Korean, Arabic or Thai** in
|
||||
the tiny model set.
|
||||
50 languages have a release in both directions (checked 2026-10), each from
|
||||
about 45 MB to about 140 MB for both directions. Only releases desktop Firefox
|
||||
would use are fetched; pre-releases (`1.0a1`) and Android-only builds are
|
||||
skipped. **Chinese is not offered**: it is published as `zh-Hans`/`zh-Hant`,
|
||||
which the proxy's two-letter language codes do not handle yet.
|
||||
|
||||
Fetching is skipped entirely when every requested pair is already on disk, so a
|
||||
restart on a genuinely offline box does not fail. A failed fetch is non-fatal:
|
||||
|
|
|
|||
|
|
@ -5,13 +5,15 @@ Deliberately NOT `mozilla/firefox-translations-models` on GitHub: that repo was
|
|||
archived on 2026-08-21 and its README now points elsewhere. Remote Settings is
|
||||
the endpoint Firefox itself uses, so it stays current.
|
||||
|
||||
Each language pair needs three attachments (model, lex, vocab) plus a small
|
||||
marian config that we write ourselves. Models are MPL-2.0.
|
||||
Each language pair needs its attachments (model, lex, and one shared vocab or a
|
||||
separate source and target vocab) plus a small marian config that we write
|
||||
ourselves. Models are MPL-2.0.
|
||||
|
||||
Roughly 37 MB per direction. Coverage is 84 pairs across about 44 languages,
|
||||
strong on European plus Hindi, Bengali, Tamil, Telugu, Vietnamese and
|
||||
Indonesian. There is no Chinese, Japanese, Korean, Arabic or Thai in the tiny
|
||||
model set, so those languages cannot be offered.
|
||||
Only releases desktop Firefox would use are taken: pre-releases ("1.0a1") are
|
||||
gated to Nightly, and some models are Android-only. As of 2026-10 that leaves
|
||||
50 languages with a release in both directions, from about 45 MB (Swedish) to
|
||||
about 140 MB (Korean) per language. Chinese is published under region codes
|
||||
(zh-Hans, zh-Hant) that the proxy's two-letter language codes cannot carry yet.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -32,8 +34,8 @@ relative-paths: true
|
|||
models:
|
||||
- model.{pair}.bin
|
||||
vocabs:
|
||||
- vocab.{pair}.spm
|
||||
- vocab.{pair}.spm
|
||||
- {src_vocab}
|
||||
- {trg_vocab}
|
||||
shortlist:
|
||||
- lex.{pair}.bin
|
||||
- false
|
||||
|
|
@ -58,11 +60,47 @@ def records() -> list[dict]:
|
|||
return json.load(response)["data"]
|
||||
|
||||
|
||||
def newest(entries: list[dict]) -> dict | None:
|
||||
"""Highest version wins. Remote Settings keeps older revisions around."""
|
||||
if not entries:
|
||||
# A record is for us only if desktop Firefox on the release channel would use
|
||||
# it. The rest are pre-releases ("1.0a1") gated to Nightly, or Android builds.
|
||||
# Picking them up is how a pre-release version string once crashed the fetch.
|
||||
RELEASE_FILTERS = {"", "env.appinfo.OS != 'Android' || env.channel != 'release'"}
|
||||
|
||||
|
||||
def release_version(record: dict) -> tuple[int, ...] | None:
|
||||
"""The record's version as a sortable tuple, or None if it is not for us."""
|
||||
if (record.get("filter_expression") or "").strip() not in RELEASE_FILTERS:
|
||||
return None
|
||||
return sorted(entries, key=lambda r: float(r.get("version", 0)))[-1]
|
||||
parts = str(record.get("version", "")).split(".")
|
||||
if not all(part.isdigit() for part in parts):
|
||||
return None
|
||||
return tuple(int(part) for part in parts)
|
||||
|
||||
|
||||
def choose_files(data: list[dict], src: str, dst: str) -> dict[str, dict] | None:
|
||||
"""The newest release whose files are all published, as {fileType: record}.
|
||||
|
||||
The files of one version belong together. Taking the newest model and the
|
||||
newest vocab independently could mix two versions that were never trained
|
||||
together. Most pairs share one vocab between both sides; some (Japanese,
|
||||
Chinese) publish a separate source and target vocab instead.
|
||||
"""
|
||||
by_version: dict[tuple[int, ...], dict[str, dict]] = {}
|
||||
for record in data:
|
||||
if record.get("fromLang") != src or record.get("toLang") != dst:
|
||||
continue
|
||||
version = release_version(record)
|
||||
if version is not None:
|
||||
by_version.setdefault(version, {})[record.get("fileType")] = record
|
||||
|
||||
for version in sorted(by_version, reverse=True):
|
||||
files = by_version[version]
|
||||
if "model" not in files or "lex" not in files:
|
||||
continue
|
||||
if "vocab" in files:
|
||||
return {kind: files[kind] for kind in ("model", "lex", "vocab")}
|
||||
if "srcvocab" in files and "trgvocab" in files:
|
||||
return {kind: files[kind] for kind in ("model", "lex", "srcvocab", "trgvocab")}
|
||||
return None
|
||||
|
||||
|
||||
def fetch_pair(data: list[dict], src: str, dst: str, out_root: Path) -> bool:
|
||||
|
|
@ -70,25 +108,13 @@ def fetch_pair(data: list[dict], src: str, dst: str, out_root: Path) -> bool:
|
|||
pair = f"{src}{dst}"
|
||||
target = out_root / pair
|
||||
|
||||
wanted = {}
|
||||
for kind in ("model", "lex", "vocab"):
|
||||
matches = [
|
||||
r
|
||||
for r in data
|
||||
if r.get("fileType") == kind
|
||||
and r.get("fromLang") == src
|
||||
and r.get("toLang") == dst
|
||||
# The "tiny" variants are the ones sized for on-device use.
|
||||
and "base" not in r.get("name", "")
|
||||
]
|
||||
chosen = newest(matches)
|
||||
if chosen is None:
|
||||
print(f" {pair}: no {kind} published, skipping pair", flush=True)
|
||||
return False
|
||||
wanted[kind] = chosen
|
||||
wanted = choose_files(data, src, dst)
|
||||
if wanted is None:
|
||||
print(f" {pair}: no released model published, skipping pair", flush=True)
|
||||
return False
|
||||
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
suffix = {"model": "bin", "lex": "bin", "vocab": "spm"}
|
||||
suffix = {"model": "bin", "lex": "bin", "vocab": "spm", "srcvocab": "spm", "trgvocab": "spm"}
|
||||
|
||||
for kind, record in wanted.items():
|
||||
dest = target / f"{kind}.{pair}.{suffix[kind]}"
|
||||
|
|
@ -109,7 +135,13 @@ def fetch_pair(data: list[dict], src: str, dst: str, out_root: Path) -> bool:
|
|||
)
|
||||
tmp.rename(dest)
|
||||
|
||||
(target / "config.yml").write_text(CONFIG.format(pair=pair))
|
||||
if "vocab" in wanted:
|
||||
src_vocab = trg_vocab = f"vocab.{pair}.spm"
|
||||
else:
|
||||
src_vocab, trg_vocab = f"srcvocab.{pair}.spm", f"trgvocab.{pair}.spm"
|
||||
(target / "config.yml").write_text(
|
||||
CONFIG.format(pair=pair, src_vocab=src_vocab, trg_vocab=trg_vocab)
|
||||
)
|
||||
size = sum(f.stat().st_size for f in target.iterdir()) / 1_000_000
|
||||
print(f" {pair}: ready ({size:.0f} MB)", flush=True)
|
||||
return True
|
||||
|
|
@ -152,8 +184,12 @@ def main() -> int:
|
|||
return 0
|
||||
|
||||
for code in missing:
|
||||
fetch_pair(data, "en", code, out_root)
|
||||
fetch_pair(data, code, "en", out_root)
|
||||
# One language failing must not stop the ones after it from loading.
|
||||
try:
|
||||
fetch_pair(data, "en", code, out_root)
|
||||
fetch_pair(data, code, "en", out_root)
|
||||
except Exception as exc:
|
||||
print(f" {code}: download failed, skipping: {exc}", flush=True)
|
||||
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -40,18 +40,40 @@ MAX_TRANSLATE_BYTES = int(os.environ.get("MAX_TRANSLATE_BYTES", str(4 * 1024 * 1
|
|||
|
||||
COOKIE = "nomadlang"
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Hand upstream redirects back to the browser instead of following them.
|
||||
|
||||
Kiwix answers a book root such as /content/<book> with a 302 to the ZIM's
|
||||
main page. Following it here would serve that page's HTML at the ORIGINAL
|
||||
URL, so every relative link and stylesheet in it resolves one directory
|
||||
too high: styling is lost and every article link 404s.
|
||||
"""
|
||||
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
# urlopen() follows redirects by default; this opener raises them as HTTPError
|
||||
# so they are forwarded below like any other non-200 response.
|
||||
_opener = urllib.request.build_opener(_NoRedirect)
|
||||
|
||||
# Display names for the languages Bergamot's tiny model set covers. A language
|
||||
# only appears in the control if its model is actually on disk.
|
||||
LANG_NAMES = {
|
||||
"bg": "Bulgarian", "bn": "Bengali", "cs": "Czech", "da": "Danish",
|
||||
"af": "Afrikaans", "ar": "Arabic", "bg": "Bulgarian", "bn": "Bengali",
|
||||
"bs": "Bosnian", "ca": "Catalan", "cs": "Czech", "da": "Danish",
|
||||
"de": "German", "el": "Greek", "es": "Spanish", "et": "Estonian",
|
||||
"fa": "Persian", "fi": "Finnish", "fr": "French", "he": "Hebrew",
|
||||
"hi": "Hindi", "hu": "Hungarian", "id": "Indonesian", "is": "Icelandic",
|
||||
"it": "Italian", "lt": "Lithuanian", "lv": "Latvian", "nb": "Norwegian",
|
||||
"nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "ro": "Romanian",
|
||||
"ru": "Russian", "sk": "Slovak", "sl": "Slovenian", "sr": "Serbian",
|
||||
"sv": "Swedish", "ta": "Tamil", "te": "Telugu", "tr": "Turkish",
|
||||
"uk": "Ukrainian", "vi": "Vietnamese",
|
||||
"eu": "Basque", "fa": "Persian", "fi": "Finnish", "fr": "French",
|
||||
"gl": "Galician", "gu": "Gujarati", "he": "Hebrew", "hi": "Hindi",
|
||||
"hr": "Croatian", "hu": "Hungarian", "id": "Indonesian", "is": "Icelandic",
|
||||
"it": "Italian", "ja": "Japanese", "kn": "Kannada", "ko": "Korean",
|
||||
"lt": "Lithuanian", "lv": "Latvian", "ml": "Malayalam", "mr": "Marathi",
|
||||
"ms": "Malay", "nb": "Norwegian", "nl": "Dutch", "pl": "Polish",
|
||||
"pt": "Portuguese", "ro": "Romanian", "ru": "Russian", "sk": "Slovak",
|
||||
"sl": "Slovenian", "sr": "Serbian", "sv": "Swedish", "ta": "Tamil",
|
||||
"te": "Telugu", "th": "Thai", "tr": "Turkish", "uk": "Ukrainian",
|
||||
"ur": "Urdu", "vi": "Vietnamese",
|
||||
}
|
||||
|
||||
# Hop-by-hop headers must not be forwarded. Content-Length and Content-Encoding
|
||||
|
|
@ -78,6 +100,9 @@ def available_languages() -> dict[str, str]:
|
|||
|
||||
|
||||
LANGS = available_languages()
|
||||
# Part of every translated page's ETag, so a cached page from before a language
|
||||
# was added can never revalidate as unchanged.
|
||||
LANGS_TAG = "".join(sorted(LANGS)) or "none"
|
||||
|
||||
# Bergamot is imported lazily so the proxy can still boot, and still forward
|
||||
# Kiwix untouched, on a box where the models never downloaded.
|
||||
|
|
@ -226,12 +251,18 @@ class Handler(BaseHTTPRequestHandler):
|
|||
# HTML can be rewritten without decompressing first.
|
||||
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=120)
|
||||
response = _opener.open(request, timeout=120)
|
||||
status, headers, data = response.status, response.headers, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
# Forwarded as-is, which is how Kiwix's own 404 and 400 pages keep
|
||||
# coming back byte for byte.
|
||||
status, headers, data = exc.code, exc.headers, exc.read()
|
||||
# An absolute Location would point the browser at the upstream
|
||||
# container name, which it cannot reach. Keep it on this origin.
|
||||
location = headers.get("Location", "")
|
||||
if location.startswith(UPSTREAM):
|
||||
del headers["Location"]
|
||||
headers["Location"] = location[len(UPSTREAM):] or "/"
|
||||
except Exception as exc:
|
||||
message = f"Translation proxy could not reach the library: {exc}".encode()
|
||||
self.send_response(502)
|
||||
|
|
@ -269,10 +300,17 @@ class Handler(BaseHTTPRequestHandler):
|
|||
# feature not working. The ETag is namespaced by language so
|
||||
# revalidation still works per language rather than being disabled.
|
||||
if translatable and key.lower() == "etag":
|
||||
value = f'{value.rstrip()}-nomadlang-{lang or "orig"}'
|
||||
value = f'{value.rstrip()}-nomadlang-{lang or "orig"}-{LANGS_TAG}'
|
||||
# The page also carries the language bar, which changes whenever a
|
||||
# language is added. Kiwix's max-age=3600 kept the old bar on screen
|
||||
# for up to an hour, and a normal refresh does not reach the
|
||||
# article frame. Revalidating every time is cheap on a local box.
|
||||
if translatable and key.lower() == "cache-control":
|
||||
continue
|
||||
self.send_header(key, value)
|
||||
if translatable:
|
||||
self.send_header("Vary", "Cookie")
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
if not body_only:
|
||||
|
|
|
|||
100
install/nomad-translate/test_fetch_models.py
Normal file
100
install/nomad-translate/test_fetch_models.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Tests for picking which model files to download.
|
||||
|
||||
Pure functions only, no Bergamot and no network, so this runs anywhere:
|
||||
|
||||
python3 install/nomad-translate/test_fetch_models.py
|
||||
|
||||
The records below copy the shapes Firefox Remote Settings actually returned.
|
||||
The first case is the one that broke: a pre-release version string ("1.0a1")
|
||||
crashed the fetch, so Swedish and 21 other languages never downloaded.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
import fetch_models # noqa: E402
|
||||
|
||||
failures: list[str] = []
|
||||
|
||||
NIGHTLY = "env.channel == 'default' || env.channel == 'nightly'"
|
||||
DESKTOP = "env.appinfo.OS != 'Android' || env.channel != 'release'"
|
||||
ANDROID = "env.appinfo.OS == 'Android' "
|
||||
|
||||
|
||||
def check(name: str, got, want):
|
||||
if got != want:
|
||||
failures.append(f"{name}\n expected: {want!r}\n got: {got!r}")
|
||||
|
||||
|
||||
def record(src, dst, kind, version, filter_expression=""):
|
||||
return {
|
||||
"fromLang": src,
|
||||
"toLang": dst,
|
||||
"fileType": kind,
|
||||
"version": version,
|
||||
"filter_expression": filter_expression,
|
||||
"name": f"{kind}.{src}{dst}",
|
||||
}
|
||||
|
||||
|
||||
def pair(src, dst, version, filter_expression="", kinds=("model", "lex", "vocab")):
|
||||
return [record(src, dst, kind, version, filter_expression) for kind in kinds]
|
||||
|
||||
|
||||
def chosen_versions(data, src, dst):
|
||||
files = fetch_models.choose_files(data, src, dst)
|
||||
if files is None:
|
||||
return None
|
||||
return {kind: r["version"] for kind, r in files.items()}
|
||||
|
||||
|
||||
# --- versions ------------------------------------------------------------------
|
||||
|
||||
check("release: plain version", fetch_models.release_version(record("en", "sv", "model", "1.0")), (1, 0))
|
||||
check("release: desktop filter", fetch_models.release_version(record("en", "ja", "model", "2.1", DESKTOP)), (2, 1))
|
||||
check("release: missing filter", fetch_models.release_version({"version": "2.0", "filter_expression": None}), (2, 0))
|
||||
check("skip: pre-release", fetch_models.release_version(record("en", "sv", "model", "1.0a1")), None)
|
||||
check("skip: nightly only", fetch_models.release_version(record("en", "sv", "model", "1.0", NIGHTLY)), None)
|
||||
check("skip: android only", fetch_models.release_version(record("en", "ja", "model", "2.2", ANDROID)), None)
|
||||
check("compare: 2.10 beats 2.9", (2, 10) > fetch_models.release_version(record("en", "x", "model", "2.9")), True)
|
||||
|
||||
# --- choosing a pair -----------------------------------------------------------
|
||||
|
||||
# Swedish as published: a release and a nightly-only pre-release side by side.
|
||||
SWEDISH = pair("en", "sv", "1.0") + pair("en", "sv", "1.0a1", NIGHTLY)
|
||||
check("sv: pre-release ignored, release kept", chosen_versions(SWEDISH, "en", "sv"),
|
||||
{"model": "1.0", "lex": "1.0", "vocab": "1.0"})
|
||||
|
||||
# Newest complete release wins.
|
||||
FRENCH = pair("en", "fr", "1.0") + pair("en", "fr", "2.0")
|
||||
check("fr: newest release", chosen_versions(FRENCH, "en", "fr"),
|
||||
{"model": "2.0", "lex": "2.0", "vocab": "2.0"})
|
||||
|
||||
# A newer version missing a file must not be mixed with an older one.
|
||||
PARTIAL = pair("en", "xx", "1.0") + pair("en", "xx", "2.0", kinds=("model", "lex"))
|
||||
check("incomplete newer version: falls back whole", chosen_versions(PARTIAL, "en", "xx"),
|
||||
{"model": "1.0", "lex": "1.0", "vocab": "1.0"})
|
||||
|
||||
# Japanese publishes separate source and target vocabs, and an Android build.
|
||||
JAPANESE = (
|
||||
pair("en", "ja", "2.1", DESKTOP, kinds=("model", "lex", "srcvocab", "trgvocab"))
|
||||
+ pair("en", "ja", "2.2", ANDROID, kinds=("model", "lex", "srcvocab", "trgvocab"))
|
||||
+ pair("en", "ja", "2.3", kinds=("model", "lex", "srcvocab", "trgvocab"))
|
||||
)
|
||||
check("ja: split vocab, android skipped", chosen_versions(JAPANESE, "en", "ja"),
|
||||
{"model": "2.3", "lex": "2.3", "srcvocab": "2.3", "trgvocab": "2.3"})
|
||||
|
||||
# Only pre-releases published: nothing to fetch, and no crash.
|
||||
check("only pre-releases: none", chosen_versions(pair("en", "nn", "1.0a1", NIGHTLY), "en", "nn"), None)
|
||||
check("other direction ignored", chosen_versions(SWEDISH, "sv", "en"), None)
|
||||
|
||||
if failures:
|
||||
print(f"FAIL: {len(failures)} of the checks above did not hold\n")
|
||||
for failure in failures:
|
||||
print(f" {failure}\n")
|
||||
sys.exit(1)
|
||||
|
||||
print("ok: all model selection checks passed")
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "project-nomad",
|
||||
"version": "1.35.0",
|
||||
"version": "1.35.1",
|
||||
"description": "Project NOMAD - an offline-first knowledge and education server.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user