fix: download util improvements

This commit is contained in:
Jake Turner 2025-12-05 18:14:33 -08:00
parent b25b652850
commit f733ecf568
No known key found for this signature in database
GPG Key ID: 694BC38EF2ED4844
3 changed files with 69 additions and 27 deletions

View File

@ -333,6 +333,7 @@ export class DockerService {
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim' 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim' const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath) const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
this._broadcast( this._broadcast(
DockerService.KIWIX_SERVICE_NAME, DockerService.KIWIX_SERVICE_NAME,
@ -345,17 +346,26 @@ export class DockerService {
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...` `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
) )
await doSimpleDownload({ try {
url: WIKIPEDIA_ZIM_URL, await doSimpleDownload({
filepath, url: WIKIPEDIA_ZIM_URL,
timeout: 60000, filepath,
}) timeout: 60000,
})
this._broadcast( this._broadcast(
DockerService.KIWIX_SERVICE_NAME, DockerService.KIWIX_SERVICE_NAME,
'preinstall', 'preinstall',
`Downloaded Wikipedia ZIM file to ${filepath}` `Downloaded Wikipedia ZIM file to ${filepath}`
) )
} catch (error) {
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall-error',
`Failed to download Wikipedia ZIM file: ${error.message}`
)
throw new Error(`Pre-install action failed: ${error.message}`)
}
} }
private _broadcast(service: string, status: string, message: string) { private _broadcast(service: string, status: string, message: string) {

View File

@ -5,7 +5,7 @@ import {
DoResumableDownloadWithRetryParams, DoResumableDownloadWithRetryParams,
DoSimpleDownloadParams, DoSimpleDownloadParams,
} from '../../types/downloads.js' } from '../../types/downloads.js'
import axios from 'axios' import axios, { AxiosResponse } from 'axios'
import { Transform } from 'stream' import { Transform } from 'stream'
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js' import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
import { createWriteStream } from 'fs' import { createWriteStream } from 'fs'
@ -21,24 +21,56 @@ export async function doSimpleDownload({
timeout = 30000, timeout = 30000,
signal, signal,
}: DoSimpleDownloadParams): Promise<string> { }: DoSimpleDownloadParams): Promise<string> {
const dirname = path.dirname(filepath) return new Promise(async (resolve, reject) => {
await ensureDirectoryExists(dirname) let response: AxiosResponse<any> | undefined
let writer: ReturnType<typeof createWriteStream> | undefined
const response = await axios.get(url, { const cleanup = (err?: Error) => {
responseType: 'stream', try {
signal, response?.data?.destroy?.()
timeout, } catch {}
}) try {
const writer = createWriteStream(filepath) writer?.destroy?.()
response.data.pipe(writer) } catch {}
if (err) {
try {
logger.error(`Download failed for ${url}: ${err.message}`)
} catch {}
reject(err)
}
}
return new Promise((resolve, reject) => { try {
writer.on('finish', () => { const dirname = path.dirname(filepath)
resolve(filepath) await ensureDirectoryExists(dirname)
})
writer.on('error', (error) => { response = await axios.get(url, {
reject(error) responseType: 'stream',
}) signal,
timeout,
})
writer = createWriteStream(filepath)
response?.data.pipe(writer)
response?.data.on('error', cleanup)
writer?.on('error', cleanup)
writer?.on('finish', () => {
cleanup()
resolve(filepath)
})
signal?.addEventListener(
'abort',
() => {
cleanup(new Error('Download aborted'))
},
{ once: true }
)
} catch (error) {
cleanup(error as Error)
}
}) })
} }

View File