Merge pull request #2878 from OpenNeuroOrg/file-fetch-high-water-mark

fix(server): Set 4MB high water mark for streaming file responses
OpenNeuroOrg · Aug 31, 2023 · 5ca344a · 5ca344a
2 parents 1a23918 + 0eca863
commit 5ca344a
Showing 1 changed file with 13 additions and 16 deletions.
diff --git a/packages/openneuro-server/src/handlers/datalad.js b/packages/openneuro-server/src/handlers/datalad.js
@@ -40,23 +40,20 @@ export const getFile = async (req, res) => {
     const uri = snapshotId
       ? `http://${worker}/datasets/${datasetId}/snapshots/${snapshotId}/files/${filename}`
       : `http://${worker}/datasets/${datasetId}/files/${filename}`
-    return (
-      fetch(uri)
-        .then(r => {
-          // Set the content length (allow clients to catch HTTP issues better)
-          res.setHeader(
-            'Content-Length',
-            Number(r.headers.get('content-length')),
-          )
-          return r.body
-        })
+    return fetch(uri)
+      .then(r => {
+        // Set the content length (allow clients to catch HTTP issues better)
+        res.setHeader('Content-Length', Number(r.headers.get('content-length')))
+        return r.body
+      })
+      .then(stream =>
         // @ts-expect-error
-        .then(stream => Readable.fromWeb(stream).pipe(res))
-        .catch(err => {
-          console.error(err)
-          res.status(500).send('Internal error transferring requested file')
-        })
-    )
+        Readable.fromWeb(stream, { highWaterMark: 4194304 }).pipe(res),
+      )
+      .catch(err => {
+        console.error(err)
+        res.status(500).send('Internal error transferring requested file')
+      })
   }
 }