Add CI job to reproduce panic for CSUB-552

.github/workflows/ci.yml

@@ -420,6 +420,104 @@ jobs:
           # dangling processes
           killall -9 creditcoin-node
 
+  integration-test-csub-552:
+    needs:
+      - build-creditcoin-node
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download creditcoin-node binary
+        uses: actions/download-artifact@v3
+        with:
+          name: creditcoin-node
+          path: target/release
+
+      - name: Import blocks
+        run: |
+          chmod a+x ./target/release/creditcoin-node
+          ./target/release/creditcoin-node import-blocks \
+            --chain ./integration-tests/csub-552/reproSpec.json \
+            --base-path ./integration-tests/csub-552/chaindata/node1 \
+            --state-pruning archive --blocks-pruning archive \
+            --database paritydb ./integration-tests/csub-552/repro.blocks
+
+      - name: Start Node 1
+        run: |
+          ./target/release/creditcoin-node \
+            --chain ./integration-tests/csub-552/reproSpec.json \
+            --validator --alice --mining-threads 1 --pruning archive \
+            --mining-key 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY \
+            --node-key 844794b5f1da387970d769debc3a30f729f3515841121ccecebed2582723e04d \
+            --base-path ./integration-tests/csub-552/chaindata/node1 >creditcoin-node1.log 2>&1 &
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - run: npm install -g yarn
+
+      - name: switchToPos()
+        run: |
+          pushd ./creditcoin-js/ && yarn install && yarn pack && popd || exit 1
+          yarn --cwd ./scripts/js upgrade 'creditcoin-js'
+          yarn --cwd ./scripts/js switchToPos ws://127.0.0.1:9944 //Alice
+
+      - name: Wait for finalized blocks to catch-up
+        run: |
+          ./scripts/wait-for-finalized-blocks.sh
+
+      - name: Store info about the last finalized block
+        id: last-block-info
+        run: |
+          # store info about the last finalized block before the fork
+          LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
+               -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getFinalizedHead", "params": [] }' \
+               http://127.0.0.1:9933 | jq -r .result)
+
+          while true; do
+              LAST_BLOCK_NUMBER=$(curl --silent -H "Content-Type: application/json" \
+                   -d "{\"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"chain_getBlock\", \"params\": [\"$LAST_BLOCK\"] }" \
+                   http://127.0.0.1:9933 | jq -r .result | jq -r .block.header.number)
+
+              if [ "$LAST_BLOCK_NUMBER" != "null" ]; then
+                break
+              else
+                echo "INFO: retry fetching block info for $LAST_BLOCK"
+                sleep 10
+              fi
+          done
+          echo "last_block_number=$LAST_BLOCK_NUMBER" >> "$GITHUB_OUTPUT"
+
+      - name: Start Node 2
+        run: |
+          ./target/release/creditcoin-node \
+            --base-path integration-tests/csub-552/chaindata/node2 \
+            --chain integration-tests/csub-552/reproSpec.json \
+            --validator --mining-threads 1 --pruning archive \
+            --mining-key 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY \
+            --bootnodes '/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWSrv5vZAh2Xr97BpyHLZ8scGqcaPr8CLYvm4EVtLwKiXy' \
+            --port 30332 --ws-port 9942 --rpc-port 9932 >creditcoin-node2.log 2>&1 &
+
+      - name: Wait for creditcoin-node to sync past block number ${{ steps.last-block-info.outputs.last_block_number }}
+        run: |
+          # Note: node2 will crash if a panic occurs and this script would fail
+          ./scripts/wait-for-sync.sh ${{ steps.last-block-info.outputs.last_block_number }}
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: csub-552-logs
+          path: "*.log"
+
+      - name: Kill SUT
+        run: |
+          # if all went well kill the node. Otherwise GitHub Actions would exit on the
+          # previous step killing everything and we don't have to worry about
+          # dangling processes
+          killall -9 creditcoin-node
+
   javascript-lint:
     name: javascript-lint / ${{ matrix.directory }}
     runs-on: ubuntu-22.04

.github/workflows/runtime-upgrade.yml

@@ -61,7 +61,7 @@ jobs:
         id: last-block-info
         run: |
           # store info about the last finalized block before the fork
-          # WARNING: using getBlockHash() instead of getFinalizedBlock() b/c PoW doesn't have finalization
+          # WARNING: using getBlockHash() instead of getFinalizedHead() b/c PoW doesn't have finalization
           LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
                -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getBlockHash", "params": [] }' \
                ${{ env.HTTPS_RPC_URL }} | jq -r .result)
@@ -204,7 +204,7 @@ jobs:
         id: last-block-info
         run: |
           # store info about the last finalized block before the fork
-          # WARNING: using getBlockHash() instead of getFinalizedBlock() b/c PoW doesn't have finalization
+          # WARNING: using getBlockHash() instead of getFinalizedHead() b/c PoW doesn't have finalization
           LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
                -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getBlockHash", "params": [] }' \
                http://127.0.0.1:9933 | jq -r .result)

scripts/wait-for-finalized-blocks.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# shellcheck disable=SC2309
+set -euo pipefail
+
+# Wait until existing blocks are finalized
+RPC_URL=${1:-http://127.0.0.1:9933}
+
+LAST_BLOCK_NUMBER=0xff
+FINAL_BLOCK_NUMBER=0x0
+
+# wait until finalized block is within 5 blocks of the last one
+while [[ $((LAST_BLOCK_NUMBER - FINAL_BLOCK_NUMBER)) -gt 0x5 ]]; do
+# Bash seems to support hex math & comparisons if numbers are prefixed with 0x
+# https://stackoverflow.com/a/13503150/1431647
+
+    echo "INFO: Block Number from chain: $LAST_BLOCK_NUMBER; finalized: $FINAL_BLOCK_NUMBER"
+
+    # WARNING: using getBlockHash() instead of getFinalizedHead() b/c PoW doesn't have finalization
+    LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
+                  -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getBlockHash", "params": [] }' \
+                    "$RPC_URL" | jq -r '.result'
+    )
+
+    LAST_BLOCK_NUMBER=$(curl --silent -H "Content-Type: application/json" \
+         -d "{\"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"chain_getBlock\", \"params\": [\"$LAST_BLOCK\"] }" \
+        "$RPC_URL" | jq -r '.result.block.header.number')
+
+    if [ "$LAST_BLOCK_NUMBER" == "null" ]; then
+        LAST_BLOCK_NUMBER="0xff"
+        echo "FAIL: fetching block info for $LAST_BLOCK"
+    fi
+
+    FINAL_BLOCK=$(curl --silent -H "Content-Type: application/json" \
+                  -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getFinalizedHead", "params": [] }' \
+                    "$RPC_URL" | jq -r '.result'
+    )
+
+    FINAL_BLOCK_NUMBER=$(curl --silent -H "Content-Type: application/json" \
+         -d "{\"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"chain_getBlock\", \"params\": [\"$FINAL_BLOCK\"] }" \
+        "$RPC_URL" | jq -r '.result.block.header.number')
+
+    if [ "$FINAL_BLOCK_NUMBER" == "null" ]; then
+        FINAL_BLOCK_NUMBER="0x0"
+        echo "FAIL: fetching block info for $FINAL_BLOCK"
+    fi
+
+    sleep 10
+done
+
+echo "DONE: blocks are finalized"
+echo "INFO: Block Number from chain: $LAST_BLOCK_NUMBER; finalized: $FINAL_BLOCK_NUMBER"
+
+exit 0

scripts/wait-for-sync.sh

@@ -20,11 +20,11 @@ echo "INFO: will start comparing blocks numbers ......"
 
 
 LAST_BLOCK_NUMBER=0x0
-# Bash seems to support hex comparisons if numbers are prefixed wwith 0x
+# Bash seems to support hex comparisons if numbers are prefixed with 0x
 # https://stackoverflow.com/a/13503150/1431647
 while [[ $LAST_BLOCK_NUMBER -lt $WAIT_FOR_BLOCK_NUMBER ]]; do
     echo "INFO: Block Number from chain: $LAST_BLOCK_NUMBER < target: $WAIT_FOR_BLOCK_NUMBER"
-    # WARNING: using getBlockHash() instead of getFinalizedBlock() b/c PoW doesn't have finalization
+    # WARNING: using getBlockHash() instead of getFinalizedHead() b/c PoW doesn't have finalization
     LAST_BLOCK=$(curl --silent -H "Content-Type: application/json" \
                   -d '{"id": 1, "jsonrpc": "2.0", "method": "chain_getBlockHash", "params": [] }' \
                     "$RPC_URL" | jq -r '.result'