firedancer: leader pipeline

contrib/test/setup_fd_cluster.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -euxo pipefail
+IFS=$'\n\t'
+
+PRIMARY_IP=$(ip -o -4 addr show scope global | awk '{ print $4 }' | cut -d/ -f1)
+RPC_URL="http://$PRIMARY_IP:8899/"
+
+mkdir ../test-ledger
+cd ../test-ledger
+
+echo "Creating mint and stake authority keys..."
+solana-keygen new --no-bip39-passphrase --force -o faucet-keypair.json > /dev/null
+solana-keygen new --no-bip39-passphrase --force -o authority.json > /dev/null
+
+# Create bootstrap validator keys
+echo "Creating keys for Validator"
+solana-keygen new --no-bip39-passphrase --force -o validator-keypair.json > id.seed
+solana-keygen new --no-bip39-passphrase --force -o vote-account-keypair.json > vote.seed
+solana-keygen new --no-bip39-passphrase --force -o stake-account-keypair.json > stake.seed
+cd ..
+
+# Genesis
+echo "Running Genesis..."
+
+GENESIS_OUTPUT=$(solana-genesis \
+    --cluster-type mainnet-beta \
+    --ledger test-ledger \
+    --bootstrap-validator test-ledger/validator-keypair.json test-ledger/vote-account-keypair.json test-ledger/stake-account-keypair.json \
+    --bootstrap-stake-authorized-pubkey test-ledger/validator-keypair.json \
+    --bootstrap-validator-lamports 11000000000000000 \
+    --bootstrap-validator-stake-lamports 10000000000000000 \
+    --faucet-pubkey test-ledger/faucet-keypair.json --faucet-lamports 1000000000000000000 \
+    --slots-per-epoch 200 \
+    --hashes-per-tick 128 \
+    --ticks-per-slot 64)
+
+# Start validator
+echo "Starting Bootstarp Validator..."
+
+# Start the bootstrap validator
+GENESIS_HASH=$(echo $GENESIS_OUTPUT | grep -o -P '(?<=Genesis hash:).*(?=Shred version:)' | xargs)
+SHRED_VERSION=$(echo $GENESIS_OUTPUT | grep -o -P '(?<=Shred version:).*(?=Ticks per slot:)' | xargs)
+_PRIMARY_INTERFACE=$(ip route show default | awk '/default/ {print $5}')
+
+RUST_LOG=debug solana-validator \
+    --identity test-ledger/validator-keypair.json \
+    --ledger test-ledger \
+    --limit-ledger-size 100000000 \
+    --no-genesis-fetch \
+    --no-snapshot-fetch \
+    --no-poh-speed-test \
+    --no-os-network-limits-test \
+    --vote-account $(solana-keygen pubkey test-ledger/vote-account-keypair.json) \
+    --expected-shred-version $SHRED_VERSION \
+    --expected-genesis-hash $GENESIS_HASH \
+    --no-wait-for-vote-to-start-leader \
+    --no-incremental-snapshots \
+    --full-snapshot-interval-slots 100 \
+    --rpc-port 8899 \
+    --gossip-port 8001 \
+    --gossip-host $PRIMARY_IP \
+    --dynamic-port-range 8100-10000 \
+    --full-rpc-api \
+    --allow-private-addr \
+    --rpc-faucet-address 127.0.0.1:9900 \
+    --log test-ledger/validator.log

contrib/test/setup_fd_cluster_stakes.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euxo pipefail
+IFS=$'\n\t'
+
+cd ../test-ledger
+
+PRIMARY_IP=$(ip -o -4 addr show scope global | awk '{ print $4 }' | cut -d/ -f1)
+RPC_URL="http://$PRIMARY_IP:8899/"
+RPC_URL="http://localhost:8899/"
+
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-identity-keypair.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-stake-keypair.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-vote-keypair.json
+solana-keygen new --no-bip39-passphrase --silent --outfile fd-withdrawer-keypair.json
+
+solana -u $RPC_URL --keypair faucet-keypair.json transfer --allow-unfunded-recipient fd-identity-keypair.json 4000000
+solana -u $RPC_URL --keypair fd-identity-keypair.json create-vote-account fd-vote-keypair.json fd-identity-keypair.json fd-withdrawer-keypair.json
+solana -u $RPC_URL --keypair fd-identity-keypair.json create-stake-account fd-stake-keypair.json 3000000
+solana -u $RPC_URL --keypair fd-identity-keypair.json delegate-stake fd-stake-keypair.json fd-vote-keypair.json
+
+solana -u $RPC_URL --keypair fd-identity-keypair.json vote-account fd-vote-keypair.json
+solana -u $RPC_URL --keypair fd-identity-keypair.json stake-account fd-stake-keypair.json

contrib/test/test_firedancer.sh

@@ -31,7 +31,7 @@ fi
 
 # if fd_frank_ledger is not on path then use the one in the home directory
 if ! command -v fddev > /dev/null; then
-  PATH="$FD_DIR/build/native/gcc/bin":$PATH
+  PATH="$FD_DIR/build/native/$CC/bin":$PATH
 fi
 
 #fddev configure fini all >/dev/null 2>&1
@@ -94,6 +94,9 @@ done
 FULL_SNAPSHOT=$(wget -c -nc -S --trust-server-names http://$PRIMARY_IP:8899/snapshot.tar.bz2 |& grep 'location:' | cut -d/ -f2)
 
 echo "
+[layout]
+    affinity = \"1-32\"
+    bank_tile_count = 1
 [gossip]
     port = 8700
 [tiles]
@@ -106,13 +109,13 @@ echo "
         repair_serve_listen_port = 8702
     [tiles.replay]
         snapshot = \"$FULL_SNAPSHOT\"
-        tpool_thread_count = 10
+        tpool_thread_count = 8
         funk_sz_gb = 8
         funk_rec_max = 100000
         funk_txn_max = 1024
 [log]
-  path = \"fddev.log\"
-  level_stderr = \"NOTICE\"
+    path = \"fddev.log\"
+    level_stderr = \"NOTICE\"
 [development]
     topology = \"firedancer\"
 " > fddev.toml

contrib/test/test_firedancer_leader.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euxo pipefail
+IFS=$'\n\t'
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+cd ../test-ledger/
+
+cleanup() {
+  sudo killall -9 -q fddev || true
+  fddev configure fini all >/dev/null 2>&1 || true
+}
+
+trap cleanup EXIT SIGINT SIGTERM
+
+FD_DIR="$SCRIPT_DIR/../.."
+
+sudo killall -9 -q fddev || true
+
+# if fd_frank_ledger is not on path then use the one in the home directory
+if ! command -v fddev > /dev/null; then
+  PATH="$FD_DIR/build/native/$CC/bin":$PATH
+fi
+
+_PRIMARY_INTERFACE=$(ip route show default | awk '/default/ {print $5}')
+PRIMARY_IP=$(ip addr show $_PRIMARY_INTERFACE | awk '/inet / {print $2}' | cut -d/ -f1 | head -n1)
+
+while [ $(solana -u localhost epoch-info --output json | jq .blockHeight) -le 150 ]; do
+  sleep 1
+done
+
+FULL_SNAPSHOT=$(wget -c -nc -S --trust-server-names http://$PRIMARY_IP:8899/snapshot.tar.bz2 |& grep 'location:' | cut -d/ -f2)
+
+echo "
+[layout]
+    affinity = \"1-32\"
+    bank_tile_count = 1
+[gossip]
+    port = 8700
+[tiles]
+    [tiles.gossip]
+        entrypoints = [\"$PRIMARY_IP\"]
+        peer_ports = [8001]
+        gossip_listen_port = 8700
+    
+    [tiles.repair]
+        repair_intake_listen_port = 8701
+        repair_serve_listen_port = 8702
+    [tiles.replay]
+        snapshot = \"$FULL_SNAPSHOT\"
+        tpool_thread_count = 8
+        funk_sz_gb = 32
+        funk_rec_max = 10000000
+        funk_txn_max = 1024
+[log]
+    path = \"fddev.log\"
+    level_stderr = \"INFO\"
+    level_flush = \"ERR\"
+[development]
+    topology = \"firedancer\"
+[consensus]
+    identity_path = \"fd-identity-keypair.json\"
+" > fddev.toml
+
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init kill --config $(readlink -f fddev.toml)
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init hugetlbfs --config $(readlink -f fddev.toml)
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init xdp --config $(readlink -f fddev.toml)
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init ethtool --config $(readlink -f fddev.toml)
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init keys --config $(readlink -f fddev.toml)
+sudo $FD_DIR/build/native/$CC/bin/fddev configure init workspace --config $(readlink -f fddev.toml)
+
+sudo gdb -ex r --args $FD_DIR/build/native/$CC/bin/fddev dev --no-configure --log-path $(readlink -f fddev.log) --config $(readlink -f fddev.toml) --no-solana --no-sandbox --no-clone

contrib/test/test_firedancer_testnet.sh

@@ -7,13 +7,15 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 FD_DIR="$SCRIPT_DIR/../.."
 
 # create temporary files in the user's home directory because it's likely to be on a large disk
-TMPDIR=$(mktemp --directory --tmpdir="$HOME" tmp-test-tvu-testnet.XXXXXX)
+# TMPDIR=$(mktemp --directory --tmpdir="$HOME" tmp-test-tvu-testnet.XXXXXX)
+TMPDIR=/home/lheeger/tmp-test-tvu-testnet
+mkdir -p $TMPDIR
 cd $TMPDIR
 
 cleanup() {
   sudo killall fddev || true
   fddev configure fini all >/dev/null 2>&1 || true
-  rm -rf "$TMPDIR"
+  # rm -rf "$TMPDIR"
 }
 
 download_snapshot() {
@@ -22,7 +24,7 @@ download_snapshot() {
   local s
   for _ in $(seq 1 $num_tries); do
     s=$(curl -s --max-redirs 0 $url)
-    if ! wget -q --trust-server-names $url; then
+    if ! wget -nc -q --trust-server-names $url; then
       sleep 1
     else
       echo "${s:1}"
@@ -66,6 +68,9 @@ if ! is_ip "$ENTRYPOINT"; then
 fi
 
 echo "
+[layout]
+    affinity = \"1-32\"
+    bank_tile_count = 1
 [gossip]
     port = 8720
 [tiles]

src/app/fdctl/Local.mk

@@ -43,6 +43,7 @@ $(call add-objs,run/tiles/fd_gossip,fd_fdctl)
 $(call add-objs,run/tiles/fd_store_int,fd_fdctl)
 $(call add-objs,run/tiles/fd_replay,fd_fdctl)
 $(call add-objs,run/tiles/fd_replay_thread,fd_fdctl)
+$(call add-objs,run/tiles/fd_poh_int,fd_fdctl)
 endif
 
 # fdctl topologies

src/app/fdctl/run/tiles/fd_gossip.c

@@ -43,6 +43,8 @@
    TODO: update */
 #define SCRATCH_DEPTH (16UL)
 
+static volatile ulong * fd_shred_version;
+
 static int
 fd_pubkey_eq( fd_pubkey_t const * key1, fd_pubkey_t const * key2 ) {
   return memcmp( key1->key, key2->key, sizeof(fd_pubkey_t) ) == 0;
@@ -264,10 +266,13 @@ gossip_deliver_fun( fd_crds_data_t * data, void * arg ) {
     if( verify_vote_txn( gossip_vote ) != 0 ) {
       return;
     }
+
     uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->pack_out_mem, ctx->pack_out_chunk );
     ulong vote_txn_sz    = gossip_vote->txn.raw_sz;
     memcpy( vote_txn_msg, gossip_vote->txn.raw, vote_txn_sz );
-    ulong sig = 1UL;
+    vote_txn_msg += vote_txn_sz;
+
+    ulong sig = 1UL; 
     fd_mcache_publish( ctx->pack_out_mcache, ctx->pack_out_depth, ctx->pack_out_seq, sig, ctx->pack_out_chunk,
       vote_txn_sz, 0UL, 0, 0 );
     ctx->pack_out_seq   = fd_seq_inc( ctx->pack_out_seq, 1UL );
@@ -445,7 +450,11 @@ after_credit( void * _ctx,
       ctx->repair_contact_out_chunk = fd_dcache_compact_next( ctx->repair_contact_out_chunk, repair_contact_sz, ctx->repair_contact_out_chunk0, ctx->repair_contact_out_wmark );
     }
   }
-
+
+  ushort shred_version = fd_gossip_get_shred_version( ctx->gossip );
+  if( shred_version!=0U ) {
+    *fd_shred_version = shred_version;
+  }
   fd_gossip_settime( ctx->gossip, now );
   fd_gossip_continue( ctx->gossip );
 }
@@ -543,9 +552,9 @@ unprivileged_init( fd_topo_t *      topo,
   ctx->gossip_config.my_addr       = ctx->gossip_my_addr;
   ctx->gossip_config.my_version = (fd_gossip_version_v2_t){
     .from = ctx->identity_public_key,
-    .major = 1337U,
-    .minor = 1337U,
-    .patch = 1337U,
+    .major = 42U,
+    .minor = 42U,
+    .patch = 42U,
     .commit = 0U,
     .has_commit = 0U,
     .feature_set = 0U,
@@ -628,6 +637,11 @@ unprivileged_init( fd_topo_t *      topo,
   if( FD_UNLIKELY( scratch_top>( (ulong)scratch + scratch_footprint( tile ) ) ) )
     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
 
+  ulong poh_shred_obj_id = fd_pod_query_ulong( topo->props, "poh_shred", ULONG_MAX );
+  FD_TEST( poh_shred_obj_id!=ULONG_MAX );
+
+  fd_shred_version = fd_fseq_join( fd_topo_obj_laddr( topo, poh_shred_obj_id ) );
+  FD_TEST( fd_shred_version );
 }
 
 static ulong