parallelizing request in EsploraProvider

* updated config files * linting formatted
Bitshala-Incubator · Aug 21, 2024 · 4f612a7 · 4f612a7
1 parent 4d300a1
commit 4f612a7
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 20 deletions.
diff --git a/config/config.yaml b/config/config.yaml
@@ -11,3 +11,4 @@ app:
 providerType: <providerType>
 esplora:
   url: <host>
+  batchSize: <batchSize>
diff --git a/config/dev.config.yaml b/config/dev.config.yaml
@@ -11,3 +11,4 @@ app:
 providerType: ESPLORA
 esplora:
   url: https://blockstream.info
+  batchSize: 5
diff --git a/src/block-data-providers/esplora.provider.ts b/src/block-data-providers/esplora.provider.ts
@@ -65,6 +65,7 @@ export class EsploraProvider
     protected readonly operationStateKey = 'esplora-operation-state';
     private readonly baseUrl: string;
     private isSyncing = false;
+    private readonly batchSize: number;
 
     constructor(
         private readonly configService: ConfigService,
@@ -73,6 +74,8 @@ export class EsploraProvider
     ) {
         super(indexerService, operationStateService);
 
+        this.batchSize = this.configService.get<number>('esplora.batchSize');
+
         let pathPrefix;
         switch (this.configService.get<BitcoinNetwork>('app.network')) {
             case BitcoinNetwork.TESTNET:
@@ -161,26 +164,30 @@ export class EsploraProvider
     private async processBlock(height: number, hash: string) {
         const txids = await this.getTxidsForBlock(hash);
 
-        // this is not very efficient
-        // at the time of implementation, I'm using a public esplora instance,
-        // so there is a rate limit on the number of requests
-        // in the future, we should add a flag to parallelize this
-        for (let i = 1; i < txids.length; i++) {
-            const txid = txids[i];
-            const tx = await this.getTx(txid);
-            const vin: TransactionInput[] = tx.vin.map((input) => ({
-                txid: input.txid,
-                vout: input.vout,
-                scriptSig: input.scriptsig,
-                prevOutScript: input.prevout.scriptpubkey,
-                witness: input.witness,
-            }));
-            const vout = tx.vout.map((output) => ({
-                scriptPubKey: output.scriptpubkey,
-                value: output.value,
-            }));
-
-            await this.indexTransaction(txid, vin, vout, height, hash);
+        for (let i = 1; i < txids.length; i += this.batchSize) {
+            const batch = txids.slice(
+                i,
+                Math.min(i + this.batchSize, txids.length),
+            );
+
+            await Promise.all(
+                batch.map(async (txid) => {
+                    const tx = await this.getTx(txid);
+                    const vin: TransactionInput[] = tx.vin.map((input) => ({
+                        txid: input.txid,
+                        vout: input.vout,
+                        scriptSig: input.scriptsig,
+                        prevOutScript: input.prevout.scriptpubkey,
+                        witness: input.witness,
+                    }));
+                    const vout = tx.vout.map((output) => ({
+                        scriptPubKey: output.scriptpubkey,
+                        value: output.value,
+                    }));
+
+                    await this.indexTransaction(txid, vin, vout, height, hash);
+                }, this),
+            );
         }
     }
 

diff --git a/src/configuration.model.ts b/src/configuration.model.ts
@@ -57,6 +57,11 @@ class EsploraConfig {
         require_host: true,
     })
     url: string;
+
+    @IsInt()
+    @Min(1)
+    @Max(100)
+    batchSize: number;
 }
 
 export class Config {