From 28b0f24802eb36fc61898eae2a4ae6f90517cf6b Mon Sep 17 00:00:00 2001
From: dtfiedler <dylan@ar.io>
Date: Tue, 16 Apr 2024 10:48:25 -0600
Subject: [PATCH] fix(s3): fix recursion and add logs

---
 src/system.ts | 49 ++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/src/system.ts b/src/system.ts
index 830801f..b69a6c4 100644
--- a/src/system.ts
+++ b/src/system.ts
@@ -56,7 +56,12 @@ export const bootstrapCache = async () => {
 
   if (SAVE_CACHE_TO_S3) {
     // save cache to S3 every s3CacheIntervalMs
-    setInterval(saveCacheToS3, s3CacheIntervalMs);
+    logger.info('Cache will be saved to S3 on an interval', {
+      intervalMs: s3CacheIntervalMs,
+    });
+    setInterval(async () => {
+      await saveCacheToS3();
+    }, s3CacheIntervalMs);
   }
 };
 
@@ -212,6 +217,12 @@ export const saveCacheToS3 = async () => {
       keyPrefix: string;
     }) => {
       const files = fs.readdirSync(folderPath);
+      logger.debug('Uploading folder to S3', {
+        folderPath,
+        bucket,
+        keyPrefix,
+        files: files.length,
+      });
       await Promise.all(
         files.map(async (file) => {
           // wrap in a pLimit to avoid resource exhaustion
@@ -224,29 +235,45 @@ export const saveCacheToS3 = async () => {
                 keyPrefix,
               });
               const fileStream = fs.createReadStream(filePath);
-              const key = path.basename(filePath);
               const upload = new Upload({
                 client: s3,
                 params: {
                   Bucket: bucket,
-                  Key: key,
+                  Key: filePath,
                   Body: fileStream,
                 },
               });
 
+              const fileStartTime = Date.now();
+
               // catch errors for a single file
-              return upload.done().catch((error: unknown) => {
-                const message =
-                  error instanceof Error ? error : new Error('Unknown error');
-                logger.error('Failed to upload file to S3', {
-                  error: message,
-                  file,
+              return upload
+                .done()
+                .then(() => {
+                  logger.debug('Successfully uploaded file to S3', {
+                    filePath,
+                    bucket,
+                    keyPrefix,
+                    durationMs: Date.now() - fileStartTime,
+                  });
+                })
+                .catch((error: unknown) => {
+                  const message =
+                    error instanceof Error ? error : new Error('Unknown error');
+                  logger.error('Failed to upload file to S3', {
+                    error: message,
+                    file,
+                  });
                 });
-              });
             } else {
+              logger.debug('Recursively uploading folder to S3', {
+                filePath,
+                bucket,
+                keyPrefix,
+              });
               // recursively upload folders
               return uploadFolder({
-                folderPath,
+                folderPath: filePath,
                 bucket,
                 keyPrefix: keyPrefix + file + '/',
               });