mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-03-31 05:59:26 +02:00
fix: download util improvements
This commit is contained in:
parent
a8bfc083d4
commit
95ba0a95c9
|
|
@ -333,6 +333,7 @@ export class DockerService {
|
|||
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
|
||||
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
|
||||
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
|
||||
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
|
||||
|
||||
this._broadcast(
|
||||
DockerService.KIWIX_SERVICE_NAME,
|
||||
|
|
@ -345,17 +346,26 @@ export class DockerService {
|
|||
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
|
||||
)
|
||||
|
||||
await doSimpleDownload({
|
||||
url: WIKIPEDIA_ZIM_URL,
|
||||
filepath,
|
||||
timeout: 60000,
|
||||
})
|
||||
try {
|
||||
await doSimpleDownload({
|
||||
url: WIKIPEDIA_ZIM_URL,
|
||||
filepath,
|
||||
timeout: 60000,
|
||||
})
|
||||
|
||||
this._broadcast(
|
||||
DockerService.KIWIX_SERVICE_NAME,
|
||||
'preinstall',
|
||||
`Downloaded Wikipedia ZIM file to ${filepath}`
|
||||
)
|
||||
this._broadcast(
|
||||
DockerService.KIWIX_SERVICE_NAME,
|
||||
'preinstall',
|
||||
`Downloaded Wikipedia ZIM file to ${filepath}`
|
||||
)
|
||||
} catch (error) {
|
||||
this._broadcast(
|
||||
DockerService.KIWIX_SERVICE_NAME,
|
||||
'preinstall-error',
|
||||
`Failed to download Wikipedia ZIM file: ${error.message}`
|
||||
)
|
||||
throw new Error(`Pre-install action failed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
private _broadcast(service: string, status: string, message: string) {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import {
|
|||
DoResumableDownloadWithRetryParams,
|
||||
DoSimpleDownloadParams,
|
||||
} from '../../types/downloads.js'
|
||||
import axios from 'axios'
|
||||
import axios, { AxiosResponse } from 'axios'
|
||||
import { Transform } from 'stream'
|
||||
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
|
||||
import { createWriteStream } from 'fs'
|
||||
|
|
@ -21,24 +21,56 @@ export async function doSimpleDownload({
|
|||
timeout = 30000,
|
||||
signal,
|
||||
}: DoSimpleDownloadParams): Promise<string> {
|
||||
const dirname = path.dirname(filepath)
|
||||
await ensureDirectoryExists(dirname)
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let response: AxiosResponse<any> | undefined
|
||||
let writer: ReturnType<typeof createWriteStream> | undefined
|
||||
|
||||
const response = await axios.get(url, {
|
||||
responseType: 'stream',
|
||||
signal,
|
||||
timeout,
|
||||
})
|
||||
const writer = createWriteStream(filepath)
|
||||
response.data.pipe(writer)
|
||||
const cleanup = (err?: Error) => {
|
||||
try {
|
||||
response?.data?.destroy?.()
|
||||
} catch {}
|
||||
try {
|
||||
writer?.destroy?.()
|
||||
} catch {}
|
||||
if (err) {
|
||||
try {
|
||||
logger.error(`Download failed for ${url}: ${err.message}`)
|
||||
} catch {}
|
||||
reject(err)
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writer.on('finish', () => {
|
||||
resolve(filepath)
|
||||
})
|
||||
writer.on('error', (error) => {
|
||||
reject(error)
|
||||
})
|
||||
try {
|
||||
const dirname = path.dirname(filepath)
|
||||
await ensureDirectoryExists(dirname)
|
||||
|
||||
response = await axios.get(url, {
|
||||
responseType: 'stream',
|
||||
signal,
|
||||
timeout,
|
||||
})
|
||||
|
||||
writer = createWriteStream(filepath)
|
||||
response?.data.pipe(writer)
|
||||
|
||||
response?.data.on('error', cleanup)
|
||||
writer?.on('error', cleanup)
|
||||
|
||||
writer?.on('finish', () => {
|
||||
cleanup()
|
||||
resolve(filepath)
|
||||
})
|
||||
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
cleanup(new Error('Download aborted'))
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
} catch (error) {
|
||||
cleanup(error as Error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user