From fd66c04c7c45eef8b1b0445e3f6ebf2bd5e5aca2 Mon Sep 17 00:00:00 2001 From: MrOrz Date: Thu, 29 Feb 2024 00:53:40 +0800 Subject: [PATCH] fix(MediaManager): use different HTTP streams for each variant - So that the variant's transform stream don't block each other by the stream backpressure mechanism, or does not block others when exits writing before reading the whole file. --- src/MediaManager.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/MediaManager.ts b/src/MediaManager.ts index 2aa97dd..d3e5a7d 100644 --- a/src/MediaManager.ts +++ b/src/MediaManager.ts @@ -1,3 +1,4 @@ +import fetch from 'node-fetch'; import { Bucket, Storage, File } from '@google-cloud/storage'; import { pipeline } from 'stream/promises'; import prepareStream from './lib/prepareStream'; @@ -160,9 +161,11 @@ class MediaManager { // Must not await this promise because we still need to pipe body to another writable stream // for hash calculation below const uploadPromise = Promise.all( - variantSettings.map(({ transform, contentType }, i) => + variantSettings.map(async ({ transform, contentType }, i) => pipeline( - body, + // Initiate separate HTTP stream here, as each variant may consume the stream differently, + // or even ends before reading the whole file. + (await fetch(url)).body, transform, tempVariantFiles[i].createWriteStream({ contentType, gzip: 'auto' }) )