mirror of
https://github.com/Crosstalk-Solutions/project-nomad.git
synced 2026-04-03 23:36:17 +02:00
fix: add stall detection to resumable downloader
Axios stream timeout only covers initial connection, not the body. Once streaming starts, a stalled server would hang indefinitely. - Added stallTimeout param (default 60s) to DoResumableDownloadParams - Reset a timer on every received chunk; abort if no data arrives within the window - Treat stall errors as retriable network errors in doResumableDownloadWithRetry so partial downloads can resume from where they left off
This commit is contained in:
parent
8bb8b414f8
commit
191e2b8a83
|
|
@ -18,6 +18,7 @@ export async function doResumableDownload({
|
||||||
url,
|
url,
|
||||||
filepath,
|
filepath,
|
||||||
timeout = 30000,
|
timeout = 30000,
|
||||||
|
stallTimeout = 60000,
|
||||||
signal,
|
signal,
|
||||||
onProgress,
|
onProgress,
|
||||||
onComplete,
|
onComplete,
|
||||||
|
|
@ -88,11 +89,30 @@ export async function doResumableDownload({
|
||||||
let lastProgressTime = Date.now()
|
let lastProgressTime = Date.now()
|
||||||
let lastDownloadedBytes = startByte
|
let lastDownloadedBytes = startByte
|
||||||
|
|
||||||
|
// Stall detection: abort if no data received within stallTimeout ms
|
||||||
|
let lastReceivedBytes = startByte
|
||||||
|
let stallTimer = setTimeout(() => {
|
||||||
|
cleanup(new Error(`Download stalled: no data received for ${stallTimeout}ms from ${url}`))
|
||||||
|
}, stallTimeout)
|
||||||
|
|
||||||
|
const resetStallTimer = () => {
|
||||||
|
clearTimeout(stallTimer)
|
||||||
|
stallTimer = setTimeout(() => {
|
||||||
|
cleanup(new Error(`Download stalled: no data received for ${stallTimeout}ms from ${url}`))
|
||||||
|
}, stallTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
// Progress tracking stream to monitor data flow
|
// Progress tracking stream to monitor data flow
|
||||||
const progressStream = new Transform({
|
const progressStream = new Transform({
|
||||||
transform(chunk: Buffer, _: any, callback: Function) {
|
transform(chunk: Buffer, _: any, callback: Function) {
|
||||||
downloadedBytes += chunk.length
|
downloadedBytes += chunk.length
|
||||||
|
|
||||||
|
// Reset stall timer on every chunk received
|
||||||
|
if (downloadedBytes > lastReceivedBytes) {
|
||||||
|
lastReceivedBytes = downloadedBytes
|
||||||
|
resetStallTimer()
|
||||||
|
}
|
||||||
|
|
||||||
// Update progress tracking
|
// Update progress tracking
|
||||||
const now = Date.now()
|
const now = Date.now()
|
||||||
if (onProgress && now - lastProgressTime >= 500) {
|
if (onProgress && now - lastProgressTime >= 500) {
|
||||||
|
|
@ -118,6 +138,7 @@ export async function doResumableDownload({
|
||||||
|
|
||||||
// Handle errors and cleanup
|
// Handle errors and cleanup
|
||||||
const cleanup = (error?: Error) => {
|
const cleanup = (error?: Error) => {
|
||||||
|
clearTimeout(stallTimer)
|
||||||
progressStream.destroy()
|
progressStream.destroy()
|
||||||
response.data.destroy()
|
response.data.destroy()
|
||||||
writeStream.destroy()
|
writeStream.destroy()
|
||||||
|
|
@ -136,6 +157,7 @@ export async function doResumableDownload({
|
||||||
})
|
})
|
||||||
|
|
||||||
writeStream.on('finish', async () => {
|
writeStream.on('finish', async () => {
|
||||||
|
clearTimeout(stallTimer)
|
||||||
if (onProgress) {
|
if (onProgress) {
|
||||||
onProgress({
|
onProgress({
|
||||||
downloadedBytes,
|
downloadedBytes,
|
||||||
|
|
@ -191,7 +213,10 @@ export async function doResumableDownloadWithRetry({
|
||||||
|
|
||||||
const isAborted = error.name === 'AbortError' || error.code === 'ABORT_ERR'
|
const isAborted = error.name === 'AbortError' || error.code === 'ABORT_ERR'
|
||||||
const isNetworkError =
|
const isNetworkError =
|
||||||
error.code === 'ECONNRESET' || error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT'
|
error.code === 'ECONNRESET' ||
|
||||||
|
error.code === 'ENOTFOUND' ||
|
||||||
|
error.code === 'ETIMEDOUT' ||
|
||||||
|
(error.message as string)?.includes('stalled')
|
||||||
|
|
||||||
onAttemptError?.(error, attempt)
|
onAttemptError?.(error, attempt)
|
||||||
if (isAborted) {
|
if (isAborted) {
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@ export type DoResumableDownloadParams = {
|
||||||
onProgress?: (progress: DoResumableDownloadProgress) => void
|
onProgress?: (progress: DoResumableDownloadProgress) => void
|
||||||
onComplete?: (url: string, path: string) => void | Promise<void>
|
onComplete?: (url: string, path: string) => void | Promise<void>
|
||||||
forceNew?: boolean
|
forceNew?: boolean
|
||||||
|
/** How long (ms) to wait without receiving data before aborting. Defaults to 60000 (60s). */
|
||||||
|
stallTimeout?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & {
|
export type DoResumableDownloadWithRetryParams = DoResumableDownloadParams & {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user