diff --git a/config/config.yaml b/config/config.yaml index 9b8045d..f89d2a6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,3 +11,4 @@ app: providerType: esplora: url: + batchSize: diff --git a/config/dev.config.yaml b/config/dev.config.yaml index ddbfd58..89982eb 100644 --- a/config/dev.config.yaml +++ b/config/dev.config.yaml @@ -11,3 +11,4 @@ app: providerType: ESPLORA esplora: url: https://blockstream.info + batchSize: 5 diff --git a/src/block-data-providers/esplora.provider.ts b/src/block-data-providers/esplora.provider.ts index 95ea691..336308c 100644 --- a/src/block-data-providers/esplora.provider.ts +++ b/src/block-data-providers/esplora.provider.ts @@ -65,6 +65,7 @@ export class EsploraProvider protected readonly operationStateKey = 'esplora-operation-state'; private readonly baseUrl: string; private isSyncing = false; + private readonly batchSize: number; constructor( private readonly configService: ConfigService, @@ -73,6 +74,8 @@ export class EsploraProvider ) { super(indexerService, operationStateService); + this.batchSize = this.configService.get('esplora.batchSize'); + let pathPrefix; switch (this.configService.get('app.network')) { case BitcoinNetwork.TESTNET: @@ -161,26 +164,30 @@ export class EsploraProvider private async processBlock(height: number, hash: string) { const txids = await this.getTxidsForBlock(hash); - // this is not very efficient - // at the time of implementation, I'm using a public esplora instance, - // so there is a rate limit on the number of requests - // in the future, we should add a flag to parallelize this - for (let i = 1; i < txids.length; i++) { - const txid = txids[i]; - const tx = await this.getTx(txid); - const vin: TransactionInput[] = tx.vin.map((input) => ({ - txid: input.txid, - vout: input.vout, - scriptSig: input.scriptsig, - prevOutScript: input.prevout.scriptpubkey, - witness: input.witness, - })); - const vout = tx.vout.map((output) => ({ - scriptPubKey: output.scriptpubkey, - value: output.value, - })); - - await this.indexTransaction(txid, vin, vout, height, hash); + for (let i = 1; i < txids.length; i += this.batchSize) { + const batch = txids.slice( + i, + Math.min(i + this.batchSize, txids.length), + ); + + await Promise.all( + batch.map(async (txid) => { + const tx = await this.getTx(txid); + const vin: TransactionInput[] = tx.vin.map((input) => ({ + txid: input.txid, + vout: input.vout, + scriptSig: input.scriptsig, + prevOutScript: input.prevout.scriptpubkey, + witness: input.witness, + })); + const vout = tx.vout.map((output) => ({ + scriptPubKey: output.scriptpubkey, + value: output.value, + })); + + await this.indexTransaction(txid, vin, vout, height, hash); + }, this), + ); } } diff --git a/src/configuration.model.ts b/src/configuration.model.ts index dc7fe26..65158c9 100644 --- a/src/configuration.model.ts +++ b/src/configuration.model.ts @@ -57,6 +57,11 @@ class EsploraConfig { require_host: true, }) url: string; + + @IsInt() + @Min(1) + @Max(100) + batchSize: number; } export class Config {