fix: download util improvements

This commit is contained in:
Jake Turner 2025-12-05 18:14:33 -08:00
parent b25b652850
commit f733ecf568
No known key found for this signature in database
GPG Key ID: 694BC38EF2ED4844
3 changed files with 69 additions and 27 deletions

View File

@ -333,6 +333,7 @@ export class DockerService {
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim' 'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim' const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath) const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
this._broadcast( this._broadcast(
DockerService.KIWIX_SERVICE_NAME, DockerService.KIWIX_SERVICE_NAME,
@ -345,6 +346,7 @@ export class DockerService {
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...` `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
) )
try {
await doSimpleDownload({ await doSimpleDownload({
url: WIKIPEDIA_ZIM_URL, url: WIKIPEDIA_ZIM_URL,
filepath, filepath,
@ -356,6 +358,14 @@ export class DockerService {
'preinstall', 'preinstall',
`Downloaded Wikipedia ZIM file to ${filepath}` `Downloaded Wikipedia ZIM file to ${filepath}`
) )
} catch (error) {
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall-error',
`Failed to download Wikipedia ZIM file: ${error.message}`
)
throw new Error(`Pre-install action failed: ${error.message}`)
}
} }
private _broadcast(service: string, status: string, message: string) { private _broadcast(service: string, status: string, message: string) {

View File

@ -5,7 +5,7 @@ import {
DoResumableDownloadWithRetryParams, DoResumableDownloadWithRetryParams,
DoSimpleDownloadParams, DoSimpleDownloadParams,
} from '../../types/downloads.js' } from '../../types/downloads.js'
import axios from 'axios' import axios, { AxiosResponse } from 'axios'
import { Transform } from 'stream' import { Transform } from 'stream'
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js' import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
import { createWriteStream } from 'fs' import { createWriteStream } from 'fs'
@ -21,24 +21,56 @@ export async function doSimpleDownload({
timeout = 30000, timeout = 30000,
signal, signal,
}: DoSimpleDownloadParams): Promise<string> { }: DoSimpleDownloadParams): Promise<string> {
return new Promise(async (resolve, reject) => {
let response: AxiosResponse<any> | undefined
let writer: ReturnType<typeof createWriteStream> | undefined
const cleanup = (err?: Error) => {
try {
response?.data?.destroy?.()
} catch {}
try {
writer?.destroy?.()
} catch {}
if (err) {
try {
logger.error(`Download failed for ${url}: ${err.message}`)
} catch {}
reject(err)
}
}
try {
const dirname = path.dirname(filepath) const dirname = path.dirname(filepath)
await ensureDirectoryExists(dirname) await ensureDirectoryExists(dirname)
const response = await axios.get(url, { response = await axios.get(url, {
responseType: 'stream', responseType: 'stream',
signal, signal,
timeout, timeout,
}) })
const writer = createWriteStream(filepath)
response.data.pipe(writer)
return new Promise((resolve, reject) => { writer = createWriteStream(filepath)
writer.on('finish', () => { response?.data.pipe(writer)
response?.data.on('error', cleanup)
writer?.on('error', cleanup)
writer?.on('finish', () => {
cleanup()
resolve(filepath) resolve(filepath)
}) })
writer.on('error', (error) => {
reject(error) signal?.addEventListener(
}) 'abort',
() => {
cleanup(new Error('Download aborted'))
},
{ once: true }
)
} catch (error) {
cleanup(error as Error)
}
}) })
} }

View File