implemented a new entity to store cached block state

.gitignore

@@ -29,7 +29,6 @@ lerna-debug.log*
 
 # IDE - VSCode
 .vscode/*
-!.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json

config/dev.config.yaml

@@ -3,7 +3,7 @@ db:
   synchronize: true
 app:
   port: 3000
-  verbose: true
+  verbose: false
   debug: true
   network: regtest
   requestRetry:
@@ -19,4 +19,3 @@ bitcoinCore:
   rpcPass: password
   rpcUser: admin
   rpcPort: 18443
-

src/block-data-providers/base-block-data-provider.abstract.ts

@@ -1,51 +1,25 @@
 import { OperationStateService } from '@/operation-state/operation-state.service';
-import { Logger, OnModuleInit } from '@nestjs/common';
+import { Logger } from '@nestjs/common';
 import {
     IndexerService,
     TransactionInput,
     TransactionOutput,
 } from '@/indexer/indexer.service';
-import { TransactionsService } from '@/transactions/transactions.service';
-import { SchedulerRegistry } from '@nestjs/schedule';
-import { CronJob } from 'cron';
 import { ConfigService } from '@nestjs/config';
+import { BlockStateService } from '@/block-state/block-state.service';
+import { BlockState } from '@/block-state/block-state.entity';
 
-export interface BaseOperationState {
-    indexedBlockHeight: number;
-    indexedBlockHash: string;
-}
-
-export abstract class BaseBlockDataProvider<
-    OperationState extends BaseOperationState,
-> implements OnModuleInit
-{
+export abstract class BaseBlockDataProvider<OperationState> {
     protected abstract readonly logger: Logger;
-    protected readonly cronJobName = 'providerSync';
-    protected readonly schedulerInterval = '*/10 * * * * *';
-    protected emptyHash =
-        '0000000000000000000000000000000000000000000000000000000000000000';
+    protected abstract readonly operationStateKey: string;
 
     protected constructor(
         protected readonly configService: ConfigService,
         private readonly indexerService: IndexerService,
         private readonly operationStateService: OperationStateService,
-        private readonly transactionService: TransactionsService,
-        private readonly schedulerRegistry: SchedulerRegistry,
+        protected readonly blockStateService: BlockStateService,
     ) {}
 
-    onModuleInit() {
-        this.initiateCronJob();
-    }
-
-    abstract sync(): void;
-
-    private initiateCronJob() {
-        const job = new CronJob(this.schedulerInterval, () => this.sync());
-
-        this.schedulerRegistry.addCronJob(this.cronJobName, job);
-        job.start();
-    }
-
     async indexTransaction(
         txid: string,
         vin: TransactionInput[],
@@ -63,42 +37,45 @@ export abstract class BaseBlockDataProvider<
     }
 
     async getState(): Promise<OperationState> {
-        const state =
-            await this.operationStateService.getCurrentOperationState();
-        return state as unknown as Promise<OperationState>;
+        return (
+            await this.operationStateService.getOperationState(
+                this.operationStateKey,
+            )
+        )?.state;
     }
 
-    async setState(futureState: Partial<OperationState>): Promise<void> {
-        await this.operationStateService.setOperationState(futureState);
+    async setState(
+        state: OperationState,
+        blockState: BlockState,
+    ): Promise<void> {
+        await this.operationStateService.setOperationState(
+            this.operationStateKey,
+            state,
+        );
+
+        await this.blockStateService.addBlockState(blockState);
     }
 
     abstract getBlockHash(height: number): Promise<string>;
 
     async traceReorg(): Promise<number> {
-        let state = await this.operationStateService.getCurrentOperationState();
+        let state = await this.blockStateService.getCurrentBlockState();
 
-        if (state.indexedBlockHash === this.emptyHash) {
-            return state.indexedBlockHeight;
-        }
+        if (!state) return null;
 
-        while (true) {
-            if (state === null) {
-                throw new Error('Reorgs levels deep');
-            }
+        while (state) {
+            const fetchedBlockHash = await this.getBlockHash(state.blockHeight);
 
-            const fetchedBlockHash = await this.getBlockHash(
-                state.indexedBlockHeight,
-            );
+            if (state.blockHash === fetchedBlockHash) return state.blockHeight;
 
-            if (state.indexedBlockHash === fetchedBlockHash) {
-                return state.indexedBlockHeight;
-            }
+            await this.blockStateService.removeState(state);
 
-            await this.transactionService.deleteTransactionByBlockHash(
-                state.indexedBlockHash,
+            this.logger.log(
+                `Reorg found at height: ${state.blockHeight}, Wrong hash: ${state.blockHash}, Correct hash: ${fetchedBlockHash}`,
             );
-
-            state = await this.operationStateService.dequeue_operation_state();
+            state = await this.blockStateService.getCurrentBlockState();
         }
+
+        throw new Error('Cannot Reorgs, blockchain state exhausted');
     }
 }

src/block-data-providers/bitcoin-core/interfaces.ts

@@ -1,5 +1,4 @@
 import { TransactionInput, TransactionOutput } from '@/indexer/indexer.service';
-import { BaseOperationState } from '@/block-data-providers/base-block-data-provider.abstract';
 
 export interface Block {
     height: number;
@@ -40,7 +39,9 @@ export interface Output {
     };
 }
 
-export type BitcoinCoreOperationState = BaseOperationState;
+export type BitcoinCoreOperationState = {
+    indexedBlockHeight: number;
+};
 
 export type Transaction = {
     txid: string;

src/block-data-providers/bitcoin-core/provider.spec.ts

@@ -12,8 +12,7 @@ import {
     rawTransactions,
 } from '@/block-data-providers/bitcoin-core/provider-fixtures';
 import { Test, TestingModule } from '@nestjs/testing';
-import { TransactionsService } from '@/transactions/transactions.service';
-import { SchedulerRegistry } from '@nestjs/schedule';
+import { BlockStateService } from '@/block-state/block-state.service';
 
 describe('Bitcoin Core Provider', () => {
     let provider: BitcoinCoreProvider;
@@ -49,11 +48,7 @@ describe('Bitcoin Core Provider', () => {
                     },
                 },
                 {
-                    provide: TransactionsService,
-                    useClass: jest.fn(),
-                },
-                {
-                    provide: SchedulerRegistry,
+                    provide: BlockStateService,
                     useClass: jest.fn(),
                 },
             ],

src/block-data-providers/bitcoin-core/provider.ts

@@ -7,7 +7,7 @@ import {
     SATS_PER_BTC,
     TAPROOT_ACTIVATION_HEIGHT,
 } from '@/common/constants';
-import { SchedulerRegistry } from '@nestjs/schedule';
+import { Cron, CronExpression } from '@nestjs/schedule';
 import {
     IndexerService,
     TransactionInput,
@@ -28,7 +28,7 @@ import {
 import { AxiosRequestConfig } from 'axios';
 import * as currency from 'currency.js';
 import { AxiosRetryConfig, makeRequest } from '@/common/request';
-import { TransactionsService } from '@/transactions/transactions.service';
+import { BlockStateService } from '@/block-state/block-state.service';
 
 @Injectable()
 export class BitcoinCoreProvider
@@ -45,15 +45,13 @@ export class BitcoinCoreProvider
         configService: ConfigService,
         indexerService: IndexerService,
         operationStateService: OperationStateService,
-        transactionService: TransactionsService,
-        schedulerRegistry: SchedulerRegistry,
+        blockStateService: BlockStateService,
     ) {
         super(
             configService,
             indexerService,
             operationStateService,
-            transactionService,
-            schedulerRegistry,
+            blockStateService,
         );
 
         const { protocol, rpcPort, rpcHost } =
@@ -75,19 +73,27 @@ export class BitcoinCoreProvider
             );
         } else {
             this.logger.log('No previous state found. Starting from scratch.');
-            const updatedState: BitcoinCoreOperationState = {
-                indexedBlockHash: this.emptyHash,
-                indexedBlockHeight:
-                    this.configService.get<BitcoinNetwork>('app.network') ===
-                    BitcoinNetwork.MAINNET
-                        ? TAPROOT_ACTIVATION_HEIGHT - 1
-                        : 0,
-            };
-
-            await this.setState(updatedState);
+
+            const blockHeight =
+                this.configService.get<BitcoinNetwork>('app.network') ===
+                BitcoinNetwork.MAINNET
+                    ? TAPROOT_ACTIVATION_HEIGHT - 1
+                    : 0;
+            const blockHash = await this.getBlockHash(blockHeight);
+
+            await this.setState(
+                {
+                    indexedBlockHeight: blockHeight,
+                },
+                {
+                    blockHash,
+                    blockHeight,
+                },
+            );
         }
     }
 
+    @Cron(CronExpression.EVERY_10_SECONDS)
     async sync() {
         if (this.isSyncing) return;
         this.isSyncing = true;
@@ -112,7 +118,8 @@ export class BitcoinCoreProvider
             const networkInfo = await this.getNetworkInfo();
             const verbosityLevel = this.versionToVerbosity(networkInfo.version);
 
-            let height = (await this.traceReorg()) + 1;
+            let height =
+                ((await this.traceReorg()) ?? state.indexedBlockHeight) + 1;
 
             for (height; height <= tipHeight; height++) {
                 const [transactions, blockHash] = await this.processBlock(
@@ -132,9 +139,10 @@ export class BitcoinCoreProvider
                     );
                 }
 
-                await this.setState({
-                    indexedBlockHeight: height,
-                    indexedBlockHash: blockHash,
+                state.indexedBlockHeight = height;
+                await this.setState(state, {
+                    blockHash: blockHash,
+                    blockHeight: height,
                 });
             }
         } finally {

src/block-data-providers/block-provider.module.ts

@@ -7,16 +7,15 @@ import { IndexerService } from '@/indexer/indexer.service';
 import { ProviderType } from '@/common/enum';
 import { BitcoinCoreProvider } from '@/block-data-providers/bitcoin-core/provider';
 import { EsploraProvider } from '@/block-data-providers/esplora/provider';
-import { TransactionsService } from '@/transactions/transactions.service';
-import { TransactionsModule } from '@/transactions/transactions.module';
-import { SchedulerRegistry } from '@nestjs/schedule';
+import { BlockStateService } from '@/block-state/block-state.service';
+import { BlockStateModule } from '@/block-state/block-state.module';
 
 @Module({
     imports: [
         OperationStateModule,
         IndexerModule,
         ConfigModule,
-        TransactionsModule,
+        BlockStateModule,
     ],
     controllers: [],
     providers: [
@@ -26,32 +25,28 @@ import { SchedulerRegistry } from '@nestjs/schedule';
                 ConfigService,
                 IndexerService,
                 OperationStateService,
-                TransactionsService,
-                SchedulerRegistry,
+                BlockStateService,
             ],
             useFactory: (
                 configService: ConfigService,
                 indexerService: IndexerService,
                 operationStateService: OperationStateService,
-                transactionService: TransactionsService,
-                schedulerRegistry: SchedulerRegistry,
+                blockStateService: BlockStateService,
             ) => {
                 switch (configService.get<ProviderType>('providerType')) {
                     case ProviderType.ESPLORA:
                         return new EsploraProvider(
                             configService,
                             indexerService,
                             operationStateService,
-                            transactionService,
-                            schedulerRegistry,
+                            blockStateService,
                         );
                     case ProviderType.BITCOIN_CORE_RPC:
                         return new BitcoinCoreProvider(
                             configService,
                             indexerService,
                             operationStateService,
-                            transactionService,
-                            schedulerRegistry,
+                            blockStateService,
                         );
                     default:
                         throw Error('unrecognised provider type in config');

src/block-data-providers/esplora/interface.ts

@@ -1,10 +1,7 @@
-import { BaseOperationState } from '@/block-data-providers/base-block-data-provider.abstract';
-
-export interface EsploraOperationState extends BaseOperationState {
-    providerState: {
-        currentBlockHeight: number;
-        lastProcessedTxIndex: number;
-    };
+export interface EsploraOperationState {
+    currentBlockHeight: number;
+    indexedBlockHeight: number;
+    lastProcessedTxIndex: number;
 }
 
 type EsploraTransactionInput = {