From 5c87696a4a32339492055128f25db7fbb2150d4c Mon Sep 17 00:00:00 2001 From: Joshua Aruokhai Date: Tue, 22 Oct 2024 16:17:02 +0100 Subject: [PATCH] chore: Improved E2E Test Structure Draft: Improved E2E Test implemented improved e2e test structure --- .github/workflows/actions-test.yml | 21 +-- e2e/Makefile | 55 ------- e2e/README.md | 39 ----- e2e/file-logger.ts | 54 +++++++ e2e/heath-check.e2e-spec.ts | 17 -- e2e/helpers/common.helper.ts | 77 ++++----- e2e/helpers/docker/docker-compose.yml | 22 --- e2e/helpers/rpc.helper.ts | 40 +++++ e2e/helpers/wallet.helper.ts | 87 +++++++++-- e2e/indexer.e2e-spec.ts | 110 +++++-------- e2e/setup.ts | 121 ++++++++++++++ package-lock.json | 174 ++++++++++++++++++++- package.json | 6 +- src/configuration.ts | 4 +- src/silent-blocks/silent-blocks.service.ts | 2 +- 15 files changed, 535 insertions(+), 294 deletions(-) delete mode 100644 e2e/Makefile delete mode 100644 e2e/README.md create mode 100644 e2e/file-logger.ts delete mode 100644 e2e/heath-check.e2e-spec.ts delete mode 100644 e2e/helpers/docker/docker-compose.yml create mode 100644 e2e/setup.ts diff --git a/.github/workflows/actions-test.yml b/.github/workflows/actions-test.yml index c85acdb..73cbc8f 100644 --- a/.github/workflows/actions-test.yml +++ b/.github/workflows/actions-test.yml @@ -43,8 +43,6 @@ jobs: with: node-version: 20.12.x cache: npm - - name: Start test containers - run: docker compose -f "./e2e/helpers/docker/docker-compose.yml" up -d - name: Cache node modules id: cache-node-modules uses: actions/cache@v4 @@ -54,20 +52,11 @@ jobs: - name: Install Dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' run: npm install - - name: Start indexer - run: npm run start:e2e > indexer.log 2>&1 & - - name: Wait for indexer to become available - run: | - for i in {1..24}; do - curl --fail -X GET http://localhost:3000/health && break || sleep 2 - done - - name: Run e2e tests + - name: Run E2E Test run: npm run test:e2e - - name: Fetch bitcoin core logs + - name: Fetch Indexer logs if: always() - run: docker compose -f "./e2e/helpers/docker/docker-compose.yml" logs bitcoin - - name: Fetch indexer logs + run: cat e2e/.logs/indexer.log + - name: Fetch Bitcoind logs if: always() - run: cat indexer.log - - name: Stop test containers - run: docker compose -f "./e2e/helpers/docker/docker-compose.yml" down \ No newline at end of file + run: cat e2e/.logs/bitcoind.log \ No newline at end of file diff --git a/e2e/Makefile b/e2e/Makefile deleted file mode 100644 index 7ada8f4..0000000 --- a/e2e/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -# Variables -DOCKER_COMPOSE_FILE = helpers/docker/docker-compose.yml -NESTJS_SERVICE_PORT = 3000 -INDEXER_TEST_CMD = npm run test:e2e -MAX_RETRIES = 10 # Number of retries to check if the inddexer service is up -RETRY_INTERVAL = 5 # Seconds between retries -PM2_CMD = npx pm2 -NPM_CMD = npm -APP_NAME = silent-indexer -LOG_DIR = .logs -INDEXER_OUTPUT_LOG = $(LOG_DIR)/$(APP_NAME)-out.log -INDEXER_ERROR_LOG = $(LOG_DIR)/$(APP_NAME)-error.log - -# Targets -.PHONY: all start-bitcoind start-indexer setup test clean - -# Default target -all: setup test - @$(MAKE) clean - -# Start Docker Compose services -start-bitcoind: - @echo "Starting Docker Compose services..." - docker compose -f $(DOCKER_COMPOSE_FILE) up -d - -# Start the Indexer service -start-indexer: - @echo "Starting the Indexer service with PM2..." - $(PM2_CMD) start $(NPM_CMD) --name $(APP_NAME) --output $(INDEXER_OUTPUT_LOG) --error $(INDEXER_ERROR_LOG) -- run start:e2e - -# Wait for the Indexer service to be available before running tests -setup: start-bitcoind start-indexer - @echo "Waiting for Indexer service to be ready on port $(NESTJS_SERVICE_PORT)..." - @count=0; \ - while ! nc -z localhost $(NESTJS_SERVICE_PORT) && [ $$count -lt $(MAX_RETRIES) ]; do \ - count=$$((count + 1)); \ - echo "Waiting for service... ($$count/$(MAX_RETRIES))"; \ - sleep $(RETRY_INTERVAL); \ - done; \ - if ! nc -z localhost $(NESTJS_SERVICE_PORT); then \ - echo "Error: Service not available after $(MAX_RETRIES) retries, exiting."; \ - exit 1; \ - fi - -# Run the end-to-end tests -test: - @echo "Running end-to-end tests..." - $(INDEXER_TEST_CMD) || echo "Warning: E2E tests failed, but continuing..." - -# Stop all services and delete all files -clean: - @echo "Stopping and removing Docker Compose services..." - docker compose -f $(DOCKER_COMPOSE_FILE) down -v - $(PM2_CMD) delete $(APP_NAME) || echo "Warning: PM2 delete command failed, but continuing..." - @echo "Clean up completed." \ No newline at end of file diff --git a/e2e/README.md b/e2e/README.md deleted file mode 100644 index 5b487ea..0000000 --- a/e2e/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# End-to-End Testing Setup - -This guide explains how to run end-to-end (E2E) tests for our Silent Indexer application using a `Makefile`. The steps include installing `make`, setting up PM2, and executing the tests. - -## Prerequisites - -- Node.js and npm must be installed on your machine. You can download them from [Node.js official website](https://nodejs.org/). - -## Step 1: Install `make` - -### On Linux - -Most Linux distributions come with `make` pre-installed. If not, you can install it using your package manager. - -For **Debian/Ubuntu**: - -```bash -sudo apt-get update -sudo apt-get install make -``` - -## Step 2: Install Dependencies - -1. Run the following command to install the npm dependencies specified in package.json: - - ```bash - npm install - ``` -This command will download all the packages listed in dependencies and devDependencies from the npm registry. - - -## Step 3: Run e2e -Ensure you are in __e2e__ directory and that port 3000 is free. - -1. **Run the following command** to run e2e test using make: - - ```bash - make - ``` diff --git a/e2e/file-logger.ts b/e2e/file-logger.ts new file mode 100644 index 0000000..f23d17e --- /dev/null +++ b/e2e/file-logger.ts @@ -0,0 +1,54 @@ +import { LoggerService } from '@nestjs/common'; +import { createWriteStream, existsSync, mkdirSync, WriteStream } from 'fs'; + +export class FileLogger implements LoggerService { + private logFolderPath = `${__dirname}/.logs`; + private logFilePath: string; + private logFileStream: WriteStream; + + constructor(logFile: string) { + if (!existsSync(this.logFolderPath)) { + mkdirSync(this.logFolderPath, { recursive: true }); + } + + this.logFilePath = `${this.logFolderPath}/${logFile}.log`; + this.logFileStream = createWriteStream(this.logFilePath, { + flags: 'a', + }); + } + + private writeLog(level: string, message: string, trace?: string) { + const timestamp = new Date().toISOString(); + let logEntry = `[${timestamp}] [${level}] ${message}\n`; + + if (trace) { + logEntry += `Trace: ${trace}\n`; + } + + this.logFileStream.write(logEntry); + } + + log(message: string) { + this.writeLog('LOG', message); + } + + error(message: string, trace: string) { + this.writeLog('ERROR', message, trace); + } + + warn(message: string) { + this.writeLog('WARN', message); + } + + debug(message: string) { + this.writeLog('DEBUG', message); + } + + verbose(message: string) { + this.writeLog('VERBOSE', message); + } + + getWriteStream() { + return this.logFileStream; + } +} diff --git a/e2e/heath-check.e2e-spec.ts b/e2e/heath-check.e2e-spec.ts deleted file mode 100644 index 698a39a..0000000 --- a/e2e/heath-check.e2e-spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { ApiHelper } from '@e2e/helpers/api.helper'; -import { ServiceStatus } from '@/common/enum'; - -describe('Health check', () => { - let apiHelper: ApiHelper; - - beforeAll(() => { - apiHelper = new ApiHelper(); - }); - - it('should check health of indexer', async () => { - const { data, status } = await apiHelper.get(`/health`); - - expect(status).toBe(200); - expect(data).toEqual(ServiceStatus.HEALTHY); - }); -}); diff --git a/e2e/helpers/common.helper.ts b/e2e/helpers/common.helper.ts index 9a80407..32f138c 100644 --- a/e2e/helpers/common.helper.ts +++ b/e2e/helpers/common.helper.ts @@ -1,14 +1,16 @@ import { Payment, Transaction } from 'bitcoinjs-lib'; import { IndexerService } from '@/indexer/indexer.service'; +import { + Transaction as TransactionEntity, + TransactionOutput, +} from '@/transactions/transaction.entity'; import { SATS_PER_BTC } from '@/common/constants'; import * as currency from 'currency.js'; -import { varIntSize } from '@/common/common'; -export function generateScanTweak( +export function generateScanTweakAndOutputEntity( transaction: Transaction, outputs: Payment[], - indexerService: IndexerService, -): string { +): [string, TransactionOutput[]] { const txins = transaction.ins.map((input, index) => { const isWitness = transaction.hasWitnesses(); @@ -27,9 +29,29 @@ export function generateScanTweak( value: output.value, })); - const [scanTweak] = indexerService.computeScanTweak(txins, txouts); + const [scanTweak, outputEntity] = new IndexerService().computeScanTweak( + txins, + txouts, + ); - return scanTweak.toString('hex'); + return [scanTweak.toString('hex'), outputEntity]; +} + +export function transactionToEntity( + transaction: Transaction, + txid: string, + blockHash: string, + blockHeight: number, + outputs: Payment[], +): TransactionEntity { + const entityTransaction = new TransactionEntity(); + entityTransaction.blockHash = blockHash; + entityTransaction.blockHeight = blockHeight; + entityTransaction.isSpent = false; + [entityTransaction.scanTweak, entityTransaction.outputs] = + generateScanTweakAndOutputEntity(transaction, outputs); + entityTransaction.id = txid; + return entityTransaction; } export const readVarInt = (data: Buffer, cursor: number): number => { @@ -50,49 +72,6 @@ export interface SilentBlockTransaction { scanTweak: string; } -export interface SilentBlock { - type: number; - transactions: SilentBlockTransaction[]; -} - -export const parseSilentBlock = (data: Buffer): SilentBlock => { - const type = data.readUInt8(0); - const transactions: SilentBlockTransaction[] = []; - - let cursor = 1; - const count = readVarInt(data, cursor); - cursor += varIntSize(count); - - for (let i = 0; i < count; i++) { - const txid = data.subarray(cursor, cursor + 32).toString('hex'); - cursor += 32; - - const outputs = []; - const outputCount = readVarInt(data, cursor); - cursor += varIntSize(outputCount); - - for (let j = 0; j < outputCount; j++) { - const value = Number(data.readBigUInt64BE(cursor)); - cursor += 8; - - const pubkey = data.subarray(cursor, cursor + 32).toString('hex'); - cursor += 32; - - const vout = data.readUint32BE(cursor); - cursor += 4; - - outputs.push({ value, pubkey, vout }); - } - - const scanTweak = data.subarray(cursor, cursor + 33).toString('hex'); - cursor += 33; - - transactions.push({ txid, outputs, scanTweak }); - } - - return { type, transactions }; -}; - export const btcToSats = (amount: number): number => { return currency(amount, { precision: 8 }).multiply(SATS_PER_BTC).value; }; diff --git a/e2e/helpers/docker/docker-compose.yml b/e2e/helpers/docker/docker-compose.yml deleted file mode 100644 index 097c202..0000000 --- a/e2e/helpers/docker/docker-compose.yml +++ /dev/null @@ -1,22 +0,0 @@ -services: - bitcoin: - image: btcpayserver/bitcoin:24.0.1-1 - environment: - BITCOIN_NETWORK: regtest - BITCOIN_EXTRA_ARGS: | - server=1 - rest=1 - rpcbind=0.0.0.0:18443 - rpcallowip=0.0.0.0/0 - rpcuser=polaruser - rpcpassword=password - debug=1 - logips=1 - logtimemicros=1 - blockmintxfee=0 - deprecatedrpc=signrawtransaction - listenonion=0 - fallbackfee=0.00001 - txindex=1 - ports: - - '18443:18443' \ No newline at end of file diff --git a/e2e/helpers/rpc.helper.ts b/e2e/helpers/rpc.helper.ts index e9363f1..6169d22 100644 --- a/e2e/helpers/rpc.helper.ts +++ b/e2e/helpers/rpc.helper.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'fs'; import * as yaml from 'js-yaml'; import axios, { AxiosRequestConfig } from 'axios'; +import { setTimeout } from 'timers/promises'; type PartialUtxo = { value: number; @@ -32,6 +33,18 @@ export class BitcoinRPCUtil { }; } + public async waitForBitcoind(): Promise { + for (let i = 0; i < 10; i++) { + try { + await this.getBlockchainInfo(); + return; + } catch (error) { + await setTimeout(2000); + } + } + throw new Error('Bitcoind refused to start'); + } + public async request(config: AxiosRequestConfig): Promise { try { const response = await axios.request({ @@ -60,6 +73,15 @@ export class BitcoinRPCUtil { }); } + loadWallet(walletName: string): Promise { + return this.request({ + data: { + method: 'createwallet', + params: [walletName], + }, + }); + } + getNewAddress(): Promise { return this.request({ data: { @@ -69,6 +91,24 @@ export class BitcoinRPCUtil { }); } + getBlockCount(): Promise { + return this.request({ + data: { + method: 'getblockcount', + params: [], + }, + }); + } + + getBlockchainInfo(): Promise { + return this.request({ + data: { + method: 'getblockchaininfo', + params: [], + }, + }); + } + mineToAddress(numBlocks: number, address: string): Promise { return this.request({ data: { diff --git a/e2e/helpers/wallet.helper.ts b/e2e/helpers/wallet.helper.ts index d9072f2..93eed9b 100644 --- a/e2e/helpers/wallet.helper.ts +++ b/e2e/helpers/wallet.helper.ts @@ -11,6 +11,7 @@ import { import { btcToSats } from '@e2e/helpers/common.helper'; import { randomBytes } from 'crypto'; import { toXOnly } from 'bitcoinjs-lib/src/psbt/bip371'; +import { BitcoinRPCUtil } from '@e2e/helpers/rpc.helper'; initEccLib(ecc); @@ -21,11 +22,62 @@ export type UTXO = { rawTx: string; }; +export type SentTransactionDetails = { + transaction: Transaction; + txid: string; + blockhash: string; +}; + export class WalletHelper { - private root: BIP32Interface; + private readonly root: BIP32Interface; + private readonly bitcoinRPCUtil: BitcoinRPCUtil; constructor() { this.root = fromSeed(randomBytes(64), networks.regtest); + this.bitcoinRPCUtil = new BitcoinRPCUtil(); + } + + async initializeWallet() { + await this.bitcoinRPCUtil.createWallet('test_wallet'); + await this.bitcoinRPCUtil.loadWallet('test_wallet'); + // ensures 5000 BTC are initially available + await this.mineBlock(200); + } + + async getBlockCount() { + return this.bitcoinRPCUtil.getBlockCount(); + } + + async mineBlock(numOfBlocks: number): Promise { + const walletAddress = await this.bitcoinRPCUtil.getNewAddress(); + return this.bitcoinRPCUtil.mineToAddress(numOfBlocks, walletAddress); + } + + async addFundToUTXO(payment: Payment, amount): Promise { + const txid = await this.bitcoinRPCUtil.sendToAddress( + payment.address, + amount, + ); + + for (let vout = 0; vout < 2; vout++) { + const utxo = await this.bitcoinRPCUtil.getTxOut(txid, vout); + if ( + utxo && + utxo.scriptPubKey && + utxo.scriptPubKey.address === payment.address + ) { + return { + txid, + vout: vout, + value: btcToSats(utxo.value), + rawTx: await this.bitcoinRPCUtil.getRawTransaction(txid), + }; + } + } + + throw new Error( + `Cannot find transaction for txid: ${txid}, address: ${payment.address}`, + ); } generateAddresses(count: number, type: 'p2wpkh' | 'p2tr'): Payment[] { @@ -57,14 +109,12 @@ export class WalletHelper { return outputs; } - /** - * Craft and sign a transaction sending 5.999 BTC to the provided Taproot address. - * - * @param utxos - Array of UTXOs to spend from. - * @param taprootOutput - The Taproot output to send to. - * @returns {Transaction} The raw signed transaction. - */ - craftTransaction(utxos: UTXO[], taprootOutput: Payment): Transaction { + async craftAndSendTransaction( + utxos: UTXO[], + taprootOutput: Payment, + outputValue: number, + fee: number, + ): Promise { const psbt = new Psbt({ network: networks.regtest }); utxos.forEach((utxo) => { @@ -75,33 +125,36 @@ export class WalletHelper { }); }); - // Add the output to the Taproot address (6 BTC) const totalInputValue = utxos.reduce( (acc, utxo) => acc + utxo.value, 0, ); - const outputValue = btcToSats(5.999); - const fee = btcToSats(0.001); - - if (totalInputValue < outputValue + fee) { + if (totalInputValue < btcToSats(outputValue) + btcToSats(fee)) { throw new Error('Insufficient funds'); } psbt.addOutput({ address: taprootOutput.address, tapInternalKey: taprootOutput.internalPubkey, - value: outputValue, + value: btcToSats(outputValue), }); // Sign the inputs with the corresponding private keys - utxos.forEach((utxo, index) => { + utxos.forEach((_, index) => { const keyPair = this.root.derivePath(`m/84'/0'/0'/0/${index}`); psbt.signInput(index, keyPair); }); psbt.finalizeAllInputs(); - return psbt.extractTransaction(true); + const transaction = psbt.extractTransaction(true); + + const txid = await this.bitcoinRPCUtil.sendRawTransaction( + transaction.toHex(), + ); + const blockhash = (await this.mineBlock(1))[0]; + + return { transaction, txid, blockhash }; } } diff --git a/e2e/indexer.e2e-spec.ts b/e2e/indexer.e2e-spec.ts index cd6f245..a6bb564 100644 --- a/e2e/indexer.e2e-spec.ts +++ b/e2e/indexer.e2e-spec.ts @@ -1,102 +1,66 @@ -import { WalletHelper, UTXO } from '@e2e/helpers/wallet.helper'; -import { BitcoinRPCUtil } from '@e2e/helpers/rpc.helper'; +import { UTXO, WalletHelper } from '@e2e/helpers/wallet.helper'; +import { transactionToEntity } from '@e2e/helpers/common.helper'; +import { initialiseDep } from '@e2e/setup'; import { ApiHelper } from '@e2e/helpers/api.helper'; -import { parseSilentBlock, SilentBlock } from '@e2e/helpers/common.helper'; -import { generateScanTweak } from '@e2e/helpers/common.helper'; -import { btcToSats } from '@e2e/helpers/common.helper'; -import { IndexerService } from '@/indexer/indexer.service'; +import { SilentBlocksService } from '@/silent-blocks/silent-blocks.service'; describe('Indexer', () => { let apiHelper: ApiHelper; - let blockHash: string; - let expectedSilentBlock: SilentBlock; + let walletHelper: WalletHelper; + let shutdownDep: () => Promise; beforeAll(async () => { - const walletHelper = new WalletHelper(); - const bitcoinRPCUtil = new BitcoinRPCUtil(); - const indexerService = new IndexerService(); + shutdownDep = await initialiseDep(); + walletHelper = new WalletHelper(); apiHelper = new ApiHelper(); - await bitcoinRPCUtil.createWallet('test_wallet'); - const initialAddress = await bitcoinRPCUtil.getNewAddress(); - const taprootOutput = walletHelper.generateAddresses(1, 'p2tr')[0]; - const p2wkhOutputs = walletHelper.generateAddresses(6, 'p2wpkh'); - await bitcoinRPCUtil.mineToAddress(101, initialAddress); + await walletHelper.initializeWallet(); + }); - const txidList = []; - for (const output of p2wkhOutputs) { - const txid = await bitcoinRPCUtil.sendToAddress(output.address, 1); - txidList.push(txid); - } - await bitcoinRPCUtil.mineToAddress(6, initialAddress); + afterAll(async () => { + await shutdownDep(); + }); + it('p2wpkh - should ensure that the correct silent block is fetched', async () => { + const taprootOutput = walletHelper.generateAddresses(1, 'p2tr')[0]; + const p2wkhOutputs = walletHelper.generateAddresses(6, 'p2wpkh'); const utxos: UTXO[] = []; - for (let i = 0; i < 6; i++) { - for (let vout = 0; vout < 2; vout++) { - const utxo = await bitcoinRPCUtil.getTxOut(txidList[i], vout); - if ( - utxo && - utxo.scriptPubKey && - utxo.scriptPubKey.address === p2wkhOutputs[i].address - ) { - utxos.push({ - txid: txidList[i], - vout: vout, - value: btcToSats(utxo.value), - rawTx: await bitcoinRPCUtil.getRawTransaction( - txidList[i], - ), - }); - break; - } - } + for (const output of p2wkhOutputs) { + const utxo = await walletHelper.addFundToUTXO(output, 1); + utxos.push(utxo); } - const transaction = walletHelper.craftTransaction( - utxos, - taprootOutput, // Send 5.999 BTC to taproot address with .001 BTC fee - ); - const txid = await bitcoinRPCUtil.sendRawTransaction( - transaction.toHex(), - ); - blockHash = (await bitcoinRPCUtil.mineToAddress(1, initialAddress))[0]; + const { transaction, txid, blockhash } = + await walletHelper.craftAndSendTransaction( + utxos, + taprootOutput, + 5.999, + 0.001, + ); - const expectedScanTweak = generateScanTweak( + const blockCount = await walletHelper.getBlockCount(); + const transformedTransaction = transactionToEntity( transaction, + txid, + blockhash, + blockCount, p2wkhOutputs, - indexerService, ); - expectedSilentBlock = { - type: 0, - transactions: [ - { - txid: txid, - outputs: [ - { - value: btcToSats(5.999), - pubkey: taprootOutput.pubkey.toString('hex'), - vout: 0, - }, - ], - scanTweak: expectedScanTweak, - }, - ], - }; + const silentBlock = new SilentBlocksService( + {} as any, + {} as any, + ).encodeSilentBlock([transformedTransaction]); await new Promise((resolve) => setTimeout(resolve, 15000)); - }); - - it('should ensure that the correct silent block is fetched', async () => { const response = await apiHelper.get( - `/silent-block/hash/${blockHash}`, + `/silent-block/hash/${blockhash}`, { responseType: 'arraybuffer', }, ); - const decodedBlock = parseSilentBlock(response.data); - expect(decodedBlock).toMatchObject(expectedSilentBlock); + expect(response.data).toEqual(silentBlock); }); }); diff --git a/e2e/setup.ts b/e2e/setup.ts new file mode 100644 index 0000000..2c4d96e --- /dev/null +++ b/e2e/setup.ts @@ -0,0 +1,121 @@ +import { INestApplication, Logger } from '@nestjs/common'; +import { AppModule } from '@/app.module'; +import * as Docker from 'dockerode'; +import { NestFactory } from '@nestjs/core'; +import { ConfigService } from '@nestjs/config'; +import { readFileSync } from 'fs'; +import * as yaml from 'js-yaml'; +import { BitcoinRPCUtil } from '@e2e/helpers/rpc.helper'; +import { FileLogger } from '@e2e/file-logger'; + +async function startBitcoinD( + configPath = './config/e2e.config.yaml', +): Promise { + const logger = new Logger('Bitcoind'); + let container: Docker.Container; + + const config = yaml.load(readFileSync(configPath, 'utf8')) as Record< + string, + any + >; + + const user = config.bitcoinCore.rpcUser; + const password = config.bitcoinCore.rpcPass; + const network = config.app.network; + const port = config.bitcoinCore.rpcPort; + + try { + const docker = new Docker(); + const imageName = 'btcpayserver/bitcoin:24.0.1-1'; + + const images = await docker.listImages(); + const imageExists = images.some( + (image) => image.RepoTags && image.RepoTags.includes(imageName), + ); + + // Pull the image if it's not available + if (!imageExists) { + logger.log(`Image ${imageName} not found locally. Pulling...`); + const stream = await docker.pull(imageName); + await new Promise((resolve, reject) => { + docker.modem.followProgress(stream, (err, output) => + err ? reject(err) : resolve(output), + ), + (event) => logger.log(event.status); + }); + } + + // Create and start the container + container = await docker.createContainer({ + Image: imageName, + ExposedPorts: { [`${port}/tcp`]: {} }, // Expose the Bitcoin RPC and P2P ports + HostConfig: { + PortBindings: { + [`${port}/tcp`]: [{ HostPort: `${port}` }], + }, + }, + Env: [ + `BITCOIN_NETWORK=${network}`, + `BITCOIN_EXTRA_ARGS=server=1\n + rest=1\n + rpcbind=0.0.0.0:${port}\n + rpcallowip=0.0.0.0/0\n + rpcuser=${user}\n + rpcpassword=${password}\n + debug=0\n + logips=1\n + logtimemicros=1\n + blockmintxfee=0\n + deprecatedrpc=signrawtransaction\n + listenonion=0\n + fallbackfee=0.00001\n + txindex=1`, + ], + }); + + logger.log('Starting bitcoind container...'); + await container.start(); + + // Pipe the container logs to the file + const logs = await container.logs({ + follow: true, + stdout: true, + stderr: true, + }); + logs.pipe(new FileLogger('bitcoind').getWriteStream()); + + return container; + } catch (error) { + logger.error('Error starting bitcoind container:', error); + await container.remove({ v: true, force: true }); + throw new Error(error); + } +} + +async function setupTestApp(): Promise { + const bitcoinRpc = new BitcoinRPCUtil(); + await bitcoinRpc.waitForBitcoind(); + + new Logger('Indexer').log('Starting Indexer...'); + const app = await NestFactory.create(AppModule, { + logger: new FileLogger('indexer'), + }); + + const configService = app.get(ConfigService); + const port = configService.get('app.port'); + + await app.listen(port); + + return app; +} + +export async function initialiseDep() { + const bitcoind = await startBitcoinD(); + const app = await setupTestApp(); + + return async function shutdownDep() { + await app.close(); + await bitcoind.stop(); + await bitcoind.remove({ v: true, force: true }); + }; +} diff --git a/package-lock.json b/package-lock.json index 07d0ac9..9907eb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "@nestjs/cli": "^10.4.5", "@nestjs/schematics": "^9.0.0", "@nestjs/testing": "^10.3.7", + "@types/dockerode": "^3.3.31", "@types/express": "^4.17.13", "@types/jest": "29.5.0", "@types/js-yaml": "^4.0.9", @@ -48,6 +49,7 @@ "bitcoinjs-lib": "^6.1.6-rc.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "dockerode": "^4.0.2", "eslint": "^8.0.1", "eslint-config-prettier": "^8.3.0", "eslint-plugin-import": "^2.26.0", @@ -59,7 +61,7 @@ "prettier": "^2.3.2", "run-script-webpack-plugin": "^0.2.0", "source-map-support": "^0.5.20", - "supertest": "^6.1.3", + "supertest": "^6.3.4", "tiny-secp256k1": "^2.2.3", "ts-jest": "29.0.5", "ts-loader": "^9.2.3", @@ -885,6 +887,12 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true + }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", @@ -2757,6 +2765,27 @@ "@types/node": "*" } }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.31", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.31.tgz", + "integrity": "sha512-42R9eoVqJDSvVspV89g7RwRqfNExgievLNWoHkg7NoWIqAmavIbgQBb4oc0qRtHkxE+I3Xxvqv7qVXFABKPBTg==", + "dev": true, + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", @@ -2925,6 +2954,15 @@ "@types/send": "*" } }, + "node_modules/@types/ssh2": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.1.tgz", + "integrity": "sha512-ZIbEqKAsi5gj35y4P4vkJYly642wIbY6PqoN0xiyQGshKUGXR9WQjF/iF9mXBQ8uBKy3ezfsCkcoHKhd0BzuDA==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18" + } + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -3751,6 +3789,15 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", "dev": true }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/ast-types": { "version": "0.13.4", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", @@ -3966,6 +4013,15 @@ "node": ">=10.0.0" } }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/bech32": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/bech32/-/bech32-2.0.0.tgz", @@ -4299,6 +4355,16 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -5026,6 +5092,21 @@ } } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-hash": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", @@ -5405,6 +5486,67 @@ "node": ">=8" } }, + "node_modules/docker-modem": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.3.tgz", + "integrity": "sha512-89zhop5YVhcPEt5FpUFGr3cDyceGhq/F9J+ZndQ4KfqNvfbJpPMfgeixFgUj5OjCYAboElqODxY5Z1EBsSa6sg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.2.tgz", + "integrity": "sha512-9wM1BVpVMFr2Pw3eJNXrYYt6DT9k0xMcsSCjtPvyQ+xa1iPg/Mo3T/gUcwI0B2cczqCeCYRPF8yFYDwtFXT0+w==", + "dev": true, + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "docker-modem": "^5.0.3", + "tar-fs": "~2.0.1" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", + "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "dev": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -12201,6 +12343,12 @@ "node": ">=0.10.0" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -12245,6 +12393,24 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" }, + "node_modules/ssh2": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", + "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.20.0" + } + }, "node_modules/ssri": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", @@ -13084,6 +13250,12 @@ "node": ">= 0.8.0" } }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true + }, "node_modules/tx2": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tx2/-/tx2-1.0.5.tgz", diff --git a/package.json b/package.json index 332c2aa..affae41 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "node --trace-warnings node_modules/.bin/jest --runInBand --no-cache --forceExit --config jest-e2e.config.ts", + "test:e2e": " NODE_ENV=e2e node --trace-warnings node_modules/.bin/jest --runInBand --no-cache --forceExit --config jest-e2e.config.ts", "migration:generate": "typeorm-ts-node-commonjs migration:generate ./migrations/migrations -d migrations/data-source.ts", "migration:run": "typeorm-ts-node-commonjs migration:run -d migrations/data-source.ts -t each", "migration:revert": "typeorm-ts-node-commonjs migration:revert -d migrations/data-source.ts" @@ -54,6 +54,7 @@ "@nestjs/cli": "^10.4.5", "@nestjs/schematics": "^9.0.0", "@nestjs/testing": "^10.3.7", + "@types/dockerode": "^3.3.31", "@types/express": "^4.17.13", "@types/jest": "29.5.0", "@types/js-yaml": "^4.0.9", @@ -67,6 +68,7 @@ "bitcoinjs-lib": "^6.1.6-rc.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "dockerode": "^4.0.2", "eslint": "^8.0.1", "eslint-config-prettier": "^8.3.0", "eslint-plugin-import": "^2.26.0", @@ -78,7 +80,7 @@ "prettier": "^2.3.2", "run-script-webpack-plugin": "^0.2.0", "source-map-support": "^0.5.20", - "supertest": "^6.1.3", + "supertest": "^6.3.4", "tiny-secp256k1": "^2.2.3", "ts-jest": "29.0.5", "ts-loader": "^9.2.3", diff --git a/src/configuration.ts b/src/configuration.ts index 89aa3b0..0e3977a 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -6,13 +6,13 @@ import { Config } from '@/configuration.model'; import { validateSync } from 'class-validator'; import { plainToClass } from 'class-transformer'; -const getConfigFilePath = () => { +const getConfigFilePath = (): string => { switch (process.env.NODE_ENV) { case 'dev': case 'test': return join(__dirname, 'config', 'dev.config.yaml'); case 'e2e': - return join(__dirname, 'config', 'e2e.config.yaml'); + return join(__dirname, '..', 'config', 'e2e.config.yaml'); default: return join(__dirname, 'config', 'config.yaml'); } diff --git a/src/silent-blocks/silent-blocks.service.ts b/src/silent-blocks/silent-blocks.service.ts index 71a4d08..117daf0 100644 --- a/src/silent-blocks/silent-blocks.service.ts +++ b/src/silent-blocks/silent-blocks.service.ts @@ -34,7 +34,7 @@ export class SilentBlocksService { return length; } - private encodeSilentBlock(transactions: Transaction[]): Buffer { + public encodeSilentBlock(transactions: Transaction[]): Buffer { const block = Buffer.alloc(this.getSilentBlockLength(transactions)); let cursor = 0;