Added batchSize configuration option for parallelizing requests in EsploraProvider

config/config.yaml

@@ -11,3 +11,4 @@ app:
 providerType: <providerType>
 esplora:
   url: <host>
+  batchSize: <batchSize>

config/dev.config.yaml

@@ -11,3 +11,4 @@ app:
 providerType: ESPLORA
 esplora:
   url: https://blockstream.info
+  batchSize: 5

src/block-data-providers/esplora.provider.ts

@@ -65,6 +65,7 @@ export class EsploraProvider
     protected readonly operationStateKey = 'esplora-operation-state';
     private readonly baseUrl: string;
     private isSyncing = false;
+    private readonly batchSize: number;
 
     constructor(
         private readonly configService: ConfigService,
@@ -73,6 +74,8 @@ export class EsploraProvider
     ) {
         super(indexerService, operationStateService);
 
+        this.batchSize = this.configService.get<number>('esplora.batchSize');
+
         let pathPrefix;
         switch (this.configService.get<BitcoinNetwork>('app.network')) {
             case BitcoinNetwork.TESTNET:
@@ -161,26 +164,30 @@ export class EsploraProvider
     private async processBlock(height: number, hash: string) {
         const txids = await this.getTxidsForBlock(hash);
 
-        // this is not very efficient
-        // at the time of implementation, I'm using a public esplora instance,
-        // so there is a rate limit on the number of requests
-        // in the future, we should add a flag to parallelize this
-        for (let i = 1; i < txids.length; i++) {
-            const txid = txids[i];
-            const tx = await this.getTx(txid);
-            const vin: TransactionInput[] = tx.vin.map((input) => ({
-                txid: input.txid,
-                vout: input.vout,
-                scriptSig: input.scriptsig,
-                prevOutScript: input.prevout.scriptpubkey,
-                witness: input.witness,
-            }));
-            const vout = tx.vout.map((output) => ({
-                scriptPubKey: output.scriptpubkey,
-                value: output.value,
-            }));
-
-            await this.indexTransaction(txid, vin, vout, height, hash);
+        for (let i = 1; i < txids.length; i += this.batchSize) {
+            const batch = txids.slice(
+                i,
+                Math.min(i + this.batchSize, txids.length),
+            );
+
+            await Promise.all(
+                batch.map(async (txid) => {
+                    const tx = await this.getTx(txid);
+                    const vin: TransactionInput[] = tx.vin.map((input) => ({
+                        txid: input.txid,
+                        vout: input.vout,
+                        scriptSig: input.scriptsig,
+                        prevOutScript: input.prevout.scriptpubkey,
+                        witness: input.witness,
+                    }));
+                    const vout = tx.vout.map((output) => ({
+                        scriptPubKey: output.scriptpubkey,
+                        value: output.value,
+                    }));
+
+                    await this.indexTransaction(txid, vin, vout, height, hash);
+                }, this),
+            );
         }
     }
 

src/configuration.model.ts

@@ -57,6 +57,11 @@ class EsploraConfig {
         require_host: true,
     })
     url: string;
+
+    @IsInt()
+    @Min(1)
+    @Max(100)
+    batchSize: number;
 }
 
 export class Config {