mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-04 15:56:16 +02:00
fix: download util improvements
This commit is contained in:
parent
b25b652850
commit
f733ecf568
|
|
@ -333,6 +333,7 @@ export class DockerService {
|
||||||
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
|
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/install/wikipedia_en_100_mini_2025-06.zim'
|
||||||
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
|
const zimPath = '/zim/wikipedia_en_100_mini_2025-06.zim'
|
||||||
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
|
const filepath = path.join(DockerService.NOMAD_STORAGE_ABS_PATH, zimPath)
|
||||||
|
logger.info(`[DockerService] Kiwix Serve pre-install: Downloading ZIM file to ${filepath}`)
|
||||||
|
|
||||||
this._broadcast(
|
this._broadcast(
|
||||||
DockerService.KIWIX_SERVICE_NAME,
|
DockerService.KIWIX_SERVICE_NAME,
|
||||||
|
|
@ -345,17 +346,26 @@ export class DockerService {
|
||||||
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
|
`Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`
|
||||||
)
|
)
|
||||||
|
|
||||||
await doSimpleDownload({
|
try {
|
||||||
url: WIKIPEDIA_ZIM_URL,
|
await doSimpleDownload({
|
||||||
filepath,
|
url: WIKIPEDIA_ZIM_URL,
|
||||||
timeout: 60000,
|
filepath,
|
||||||
})
|
timeout: 60000,
|
||||||
|
})
|
||||||
|
|
||||||
this._broadcast(
|
this._broadcast(
|
||||||
DockerService.KIWIX_SERVICE_NAME,
|
DockerService.KIWIX_SERVICE_NAME,
|
||||||
'preinstall',
|
'preinstall',
|
||||||
`Downloaded Wikipedia ZIM file to ${filepath}`
|
`Downloaded Wikipedia ZIM file to ${filepath}`
|
||||||
)
|
)
|
||||||
|
} catch (error) {
|
||||||
|
this._broadcast(
|
||||||
|
DockerService.KIWIX_SERVICE_NAME,
|
||||||
|
'preinstall-error',
|
||||||
|
`Failed to download Wikipedia ZIM file: ${error.message}`
|
||||||
|
)
|
||||||
|
throw new Error(`Pre-install action failed: ${error.message}`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private _broadcast(service: string, status: string, message: string) {
|
private _broadcast(service: string, status: string, message: string) {
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import {
|
||||||
DoResumableDownloadWithRetryParams,
|
DoResumableDownloadWithRetryParams,
|
||||||
DoSimpleDownloadParams,
|
DoSimpleDownloadParams,
|
||||||
} from '../../types/downloads.js'
|
} from '../../types/downloads.js'
|
||||||
import axios from 'axios'
|
import axios, { AxiosResponse } from 'axios'
|
||||||
import { Transform } from 'stream'
|
import { Transform } from 'stream'
|
||||||
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
|
import { deleteFileIfExists, ensureDirectoryExists, getFileStatsIfExists } from './fs.js'
|
||||||
import { createWriteStream } from 'fs'
|
import { createWriteStream } from 'fs'
|
||||||
|
|
@ -21,24 +21,56 @@ export async function doSimpleDownload({
|
||||||
timeout = 30000,
|
timeout = 30000,
|
||||||
signal,
|
signal,
|
||||||
}: DoSimpleDownloadParams): Promise<string> {
|
}: DoSimpleDownloadParams): Promise<string> {
|
||||||
const dirname = path.dirname(filepath)
|
return new Promise(async (resolve, reject) => {
|
||||||
await ensureDirectoryExists(dirname)
|
let response: AxiosResponse<any> | undefined
|
||||||
|
let writer: ReturnType<typeof createWriteStream> | undefined
|
||||||
|
|
||||||
const response = await axios.get(url, {
|
const cleanup = (err?: Error) => {
|
||||||
responseType: 'stream',
|
try {
|
||||||
signal,
|
response?.data?.destroy?.()
|
||||||
timeout,
|
} catch {}
|
||||||
})
|
try {
|
||||||
const writer = createWriteStream(filepath)
|
writer?.destroy?.()
|
||||||
response.data.pipe(writer)
|
} catch {}
|
||||||
|
if (err) {
|
||||||
|
try {
|
||||||
|
logger.error(`Download failed for ${url}: ${err.message}`)
|
||||||
|
} catch {}
|
||||||
|
reject(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
try {
|
||||||
writer.on('finish', () => {
|
const dirname = path.dirname(filepath)
|
||||||
resolve(filepath)
|
await ensureDirectoryExists(dirname)
|
||||||
})
|
|
||||||
writer.on('error', (error) => {
|
response = await axios.get(url, {
|
||||||
reject(error)
|
responseType: 'stream',
|
||||||
})
|
signal,
|
||||||
|
timeout,
|
||||||
|
})
|
||||||
|
|
||||||
|
writer = createWriteStream(filepath)
|
||||||
|
response?.data.pipe(writer)
|
||||||
|
|
||||||
|
response?.data.on('error', cleanup)
|
||||||
|
writer?.on('error', cleanup)
|
||||||
|
|
||||||
|
writer?.on('finish', () => {
|
||||||
|
cleanup()
|
||||||
|
resolve(filepath)
|
||||||
|
})
|
||||||
|
|
||||||
|
signal?.addEventListener(
|
||||||
|
'abort',
|
||||||
|
() => {
|
||||||
|
cleanup(new Error('Download aborted'))
|
||||||
|
},
|
||||||
|
{ once: true }
|
||||||
|
)
|
||||||
|
} catch (error) {
|
||||||
|
cleanup(error as Error)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user