mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-10-17 05:14:19 +02:00
a3811ad49f
The batch continuation gated on articles that produced chunks rather than articles the extractor consumed. Articles whose text is empty after HTML cleaning — redirect stubs, category listings, media and PDF wrappers — return no chunks and so no documentIds, which made a normal sparse batch look like the end of the archive. Ingestion stopped after the first batch and the rest of the file was silently skipped. Medicine LibreTexts embedded 16 chunks out of 23,171 articles: its first 50 entries are navigation pages, only 10 of which had text, so 10 >= 50 was false and no continuation was ever dispatched. Measured against the same install, WikiMed reached 28% of its articles, Libre Pathology 31%, MedlinePlus 84%, and CDC Travelers' Health 2.7%. The same value was also returned as `articlesProcessed`, which EmbedFileJob uses to advance the next batch offset. Advancing by the content-bearing count re-read the overlap on every sparse batch, and a batch yielding no text at all would have advanced by zero and re-run the same window indefinitely. Both now use the extractor's consumed-article count, which it already computed and logged but discarded. The end-of-archive signal is the iterator running dry (articlesProcessed < batchSize); archive.articleCount cannot serve as a bound because iterByPath() legitimately yields more article entries than that figure for some archives. The log line now reports both counts (content-bearing/consumed) so a sparse batch is visible without re-deriving it.
56 lines
2.2 KiB
TypeScript
56 lines
2.2 KiB
TypeScript
import * as assert from 'node:assert/strict'
|
|
import { test } from 'node:test'
|
|
|
|
import { hasMoreArticleBatches } from '../../app/utils/zim_batch_decision.js'
|
|
|
|
const BATCH = 50
|
|
|
|
test('continues while the extractor keeps filling the batch', () => {
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: BATCH, batchSize: BATCH }), true)
|
|
})
|
|
|
|
test('stops once the extractor runs out of article entries', () => {
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: 12, batchSize: BATCH }), false)
|
|
})
|
|
|
|
test('stops on an archive smaller than one batch', () => {
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: 1, batchSize: BATCH }), false)
|
|
})
|
|
|
|
test('stops on an exhausted iterator that yielded nothing', () => {
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: 0, batchSize: BATCH }), false)
|
|
})
|
|
|
|
/**
|
|
* Regression: Medicine LibreTexts (23,171 articles) embedded only 16 chunks.
|
|
*
|
|
* Its first 50 article entries are navigation and category pages; only 10 of
|
|
* them produced any text. The previous gate counted articles that produced
|
|
* chunks, so it evaluated `10 >= 50` -> false, never dispatched a continuation,
|
|
* and abandoned the remaining 23,121 articles. A full batch was consumed, so the
|
|
* ingestion must continue regardless of how little text came out of it.
|
|
*/
|
|
test('continues through a full batch that produced almost no text', () => {
|
|
const articlesWithContent = 10
|
|
assert.equal(
|
|
hasMoreArticleBatches({ articlesProcessed: BATCH, batchSize: BATCH }),
|
|
true,
|
|
'a full batch must continue even when most of its articles are empty'
|
|
)
|
|
assert.equal(
|
|
articlesWithContent >= BATCH,
|
|
false,
|
|
'the old chunk-derived gate stopped here — this is the bug being fixed'
|
|
)
|
|
})
|
|
|
|
/**
|
|
* Regression: a media archive (Canadian Prepper, 92 articles / 187 media files)
|
|
* yields zero text chunks, but its articles still have to be walked to the end
|
|
* rather than abandoned after the first batch.
|
|
*/
|
|
test('walks a zero-text archive to the end instead of stopping at batch one', () => {
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: BATCH, batchSize: BATCH }), true)
|
|
assert.equal(hasMoreArticleBatches({ articlesProcessed: 42, batchSize: BATCH }), false)
|
|
})
|