fix: download util improvements

This commit is contained in:
Jake Turner 2025-12-05 18:14:33 -08:00 committed by Jake Turner
parent a8bfc083d4
commit 95ba0a95c9
3 changed files with 69 additions and 27 deletions

View File

@ -333,6 +333,7 @@ export class DockerService {
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
@ -345,17 +346,26 @@ export class DockerService {
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
)
await doSimpleDownload({
url: WIKIPEDIA_ZIM_URL,
filepath,
timeout: 60000,
})
try {
await doSimpleDownload({
url: WIKIPEDIA_ZIM_URL,
filepath,
timeout: 60000,
})
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall',
`Downloaded Wikipedia ZIM file to ${filepath}`
)
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall',
`Downloaded Wikipedia ZIM file to ${filepath}`
)
} catch (error) {
this._broadcast(
DockerService.KIWIX_SERVICE_NAME,
'preinstall-error',
`Failed to download Wikipedia ZIM file: ${error.message}`
)
throw new Error(`Pre-install action failed: ${error.message}`)
}
}
private _broadcast(service: string, status: string, message: string) {

View File

@ -5,7 +5,7 @@ import {
DoResumableDownloadWithRetryParams,
DoSimpleDownloadParams,
} from '../../types/downloads.js'
import axios from 'axios'
import axios, { AxiosResponse } from 'axios'
import { Transform } from 'stream'
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
import { createWriteStream } from 'fs'
@ -21,24 +21,56 @@ export async function doSimpleDownload({
timeout = 30000,
signal,
}: DoSimpleDownloadParams): Promise<string> {
const dirname = path.dirname(filepath)
await ensureDirectoryExists(dirname)
return new Promise(async (resolve, reject) => {
let response: AxiosResponse<any> | undefined
let writer: ReturnType<typeof createWriteStream> | undefined
const response = await axios.get(url, {
responseType: 'stream',
signal,
timeout,
})
const writer = createWriteStream(filepath)
response.data.pipe(writer)
const cleanup = (err?: Error) => {
try {
response?.data?.destroy?.()
} catch {}
try {
writer?.destroy?.()
} catch {}
if (err) {
try {
logger.error(`Download failed for ${url}: ${err.message}`)
} catch {}
reject(err)
}
}
return new Promise((resolve, reject) => {
writer.on('finish', () => {
resolve(filepath)
})
writer.on('error', (error) => {
reject(error)
})
try {
const dirname = path.dirname(filepath)
await ensureDirectoryExists(dirname)
response = await axios.get(url, {
responseType: 'stream',
signal,
timeout,
})
writer = createWriteStream(filepath)
response?.data.pipe(writer)
response?.data.on('error', cleanup)
writer?.on('error', cleanup)
writer?.on('finish', () => {
cleanup()
resolve(filepath)
})
signal?.addEventListener(
'abort',
() => {
cleanup(new Error('Download aborted'))
},
{ once: true }
)
} catch (error) {
cleanup(error as Error)
}
})
}

View File