From 712203d0baa8283276b79c1a9271fd8ee28fb5ef Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 18:20:27 +0000 Subject: [PATCH 01/10] ci: rename test_firedancer to test_firedancer_localnet.yml --- .../{test_firedancer.yml => test_firedancer_localnet.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test_firedancer.yml => test_firedancer_localnet.yml} (100%) diff --git a/.github/workflows/test_firedancer.yml b/.github/workflows/test_firedancer_localnet.yml similarity index 100% rename from .github/workflows/test_firedancer.yml rename to .github/workflows/test_firedancer_localnet.yml From ad9ad3ab431a4d063b8966ccd434c1e80b4f9feb Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 18:32:08 +0000 Subject: [PATCH 02/10] updates --- .github/workflows/test_firedancer_localnet.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 6093da27fd..88244ba51c 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -1,4 +1,4 @@ -name: Firedancer +name: Firedancer Localnet on: workflow_call: workflow_dispatch: @@ -6,7 +6,7 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - firedancer-tests: + integration-test: runs-on: [self-hosted, 512G] if: github.event.pull_request.draft == false env: @@ -19,6 +19,9 @@ jobs: - uses: ./.github/actions/deps + - name: run + run: pwd + - name: build run: | ./contrib/make-j fddev @@ -31,10 +34,10 @@ jobs: - uses: ./.github/actions/hugepages with: count_gigantic: 128 - count_huge: 500 # TODO: this is required until we can handle anonymouse workspaces and loose huge pages in fddev + count_huge: 100 # TODO: this is required until we can handle anonymouse workspaces and loose huge pages in fddev - name: test firedancer - local cluster run: | sudo prlimit --pid=$$ --nofile=1048576 sudo prlimit --pid=$$ --memlock=unlimited - ./contrib/test/test_firedancer.sh + # ./contrib/test/test_firedancer.sh From a01e0ce544503fefa711ede648911210278e9916 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 18:33:15 +0000 Subject: [PATCH 03/10] updates --- .github/workflows/test_firedancer_localnet.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 88244ba51c..fde012b13a 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -2,6 +2,8 @@ name: Firedancer Localnet on: workflow_call: workflow_dispatch: + pull_request: + types: [synchronize, opened, reopened, ready_for_review] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true From 0bfe03bca6e9c4ca04b41ff3ba1392b7acc24711 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 18:41:33 +0000 Subject: [PATCH 04/10] updates --- .github/workflows/on_pull_request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index a40a565d61..38aaac5666 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -1,8 +1,8 @@ name: On Pull Request on: merge_group: - pull_request: - types: [synchronize, opened, reopened, ready_for_review] + # pull_request: + # types: [synchronize, opened, reopened, ready_for_review] concurrency: group: on-pull-request_${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true From f57fb3f2ff67ab2f14f92640af6a3aff1268d479 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 20:20:44 +0000 Subject: [PATCH 05/10] updates --- .github/workflows/test_firedancer_localnet.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index fde012b13a..07cc3a23c4 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -21,8 +21,19 @@ jobs: - uses: ./.github/actions/deps - - name: run - run: pwd + - name: clone agave + run: | + git clone --depth 1 --branch v1.18.15 https://github.com/anza-xyz/agave.git + + - name: build agave + run: | + cd agave + ./cargo build --release \ + --package solana-genesis \ + --package agave-validator \ + --package solana-cli \ + --package solana-keygen + cd .. - name: build run: | From 29cf961f7c19d166113e2cadd95078bc156859a2 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 20:26:48 +0000 Subject: [PATCH 06/10] updates --- .github/workflows/test_firedancer_localnet.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 07cc3a23c4..7a06450b25 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -21,6 +21,9 @@ jobs: - uses: ./.github/actions/deps + - name: install rust toolchain + uses: actions-rs/toolchain@v1 + - name: clone agave run: | git clone --depth 1 --branch v1.18.15 https://github.com/anza-xyz/agave.git From 11e23343cde82e021ec1644b86102fea560baaca Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 20:37:30 +0000 Subject: [PATCH 07/10] updates --- .github/workflows/test_firedancer_localnet.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 7a06450b25..3f64cbab9d 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -23,6 +23,8 @@ jobs: - name: install rust toolchain uses: actions-rs/toolchain@v1 + with: + toolcahin: stable - name: clone agave run: | From 921d498004f91d0f1aa0c06fb9b80e885fabd94e Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 20:41:03 +0000 Subject: [PATCH 08/10] updates --- .github/workflows/test_firedancer_localnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 3f64cbab9d..51784735c4 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -24,7 +24,7 @@ jobs: - name: install rust toolchain uses: actions-rs/toolchain@v1 with: - toolcahin: stable + toolchain: stable - name: clone agave run: | From 35e5757339f2bd08bbdd494a12e7f16e3ff79070 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Fri, 5 Jul 2024 21:31:31 +0000 Subject: [PATCH 09/10] updates --- .github/workflows/test_firedancer_localnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_firedancer_localnet.yml b/.github/workflows/test_firedancer_localnet.yml index 51784735c4..b782337ddd 100644 --- a/.github/workflows/test_firedancer_localnet.yml +++ b/.github/workflows/test_firedancer_localnet.yml @@ -40,7 +40,7 @@ jobs: --package solana-keygen cd .. - - name: build + - name: build firedancer run: | ./contrib/make-j fddev From d0bdf5053e3008ee35c20bd815089d72002a0e98 Mon Sep 17 00:00:00 2001 From: Liam Heeger Date: Mon, 8 Jul 2024 15:04:28 +0000 Subject: [PATCH 10/10] updates --- src/app/fdctl/run/tiles/fd_replay.c | 37 +++++++++++++------------ src/app/fdctl/run/topos/fd_firedancer.c | 4 +-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/app/fdctl/run/tiles/fd_replay.c b/src/app/fdctl/run/tiles/fd_replay.c index 72ab003794..18cdccaecf 100644 --- a/src/app/fdctl/run/tiles/fd_replay.c +++ b/src/app/fdctl/run/tiles/fd_replay.c @@ -497,6 +497,7 @@ after_frag( void * _ctx, fd_fork_t * fork = fd_fork_frontier_ele_query( ctx->forks->frontier, &ctx->curr_slot, NULL, ctx->forks->pool ); + if( fork == NULL ) { long prepare_time_ns = -fd_log_wallclock(); @@ -515,6 +516,24 @@ after_frag( void * _ctx, // fork is advancing FD_LOG_DEBUG(( "new block execution - slot: %lu, parent_slot: %lu", ctx->curr_slot, ctx->parent_slot )); + /* if it is an epoch boundary, push out stake weights */ + int is_new_epoch = 0; + if( fork->slot_ctx.slot_bank.slot != 0 ) { + ulong slot_idx; + fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( fork->slot_ctx.epoch_ctx ); + ulong prev_epoch = fd_slot_to_epoch( &epoch_bank->epoch_schedule, fork->slot_ctx.slot_bank.prev_slot, &slot_idx ); + ulong new_epoch = fd_slot_to_epoch( &epoch_bank->epoch_schedule, fork->slot_ctx.slot_bank.slot, &slot_idx ); + + if( prev_epoch < new_epoch || slot_idx == 0 ) { + FD_LOG_DEBUG(("Epoch boundary")); + is_new_epoch = 1; + } + } + + if( is_new_epoch ) { + publish_stake_weights( ctx, mux, &fork->slot_ctx ); + } + fork->slot_ctx.slot_bank.prev_slot = fork->slot_ctx.slot_bank.slot; fork->slot_ctx.slot_bank.slot = ctx->curr_slot; @@ -533,26 +552,8 @@ after_frag( void * _ctx, FD_LOG_ERR(( "txn publishing failed" )); } - /* if it is an epoch boundary, push out stake weights */ - int is_new_epoch = 0; - if( fork->slot_ctx.slot_bank.slot != 0 ) { - ulong slot_idx; - fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( fork->slot_ctx.epoch_ctx ); - ulong prev_epoch = fd_slot_to_epoch( &epoch_bank->epoch_schedule, fork->slot_ctx.slot_bank.prev_slot, &slot_idx ); - ulong new_epoch = fd_slot_to_epoch( &epoch_bank->epoch_schedule, fork->slot_ctx.slot_bank.slot, &slot_idx ); - - if( prev_epoch < new_epoch || slot_idx == 0 ) { - FD_LOG_DEBUG(("Epoch boundary")); - is_new_epoch = 1; - } - } - res = fd_runtime_block_execute_prepare( &fork->slot_ctx ); - if( is_new_epoch ) { - publish_stake_weights( ctx, mux, &fork->slot_ctx ); - } - if( res != FD_RUNTIME_EXECUTE_SUCCESS ) { FD_LOG_ERR(( "block prep execute failed" )); } diff --git a/src/app/fdctl/run/topos/fd_firedancer.c b/src/app/fdctl/run/topos/fd_firedancer.c index 73285627db..3b452234dd 100644 --- a/src/app/fdctl/run/topos/fd_firedancer.c +++ b/src/app/fdctl/run/topos/fd_firedancer.c @@ -331,14 +331,14 @@ fd_topo_firedancer( config_t * _config ) { /**/ fd_topob_tile_in( topo, "sender", 0UL, "metric_in", "sign_voter", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "voter_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "dedup_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ + /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "poh_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "gossip_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ /**/ fd_topob_tile_in( topo, "bhole", 0UL, "metric_in", "replay_notif", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ /**/ fd_topob_tile_in( topo, "pohi", 0UL, "metric_in", "replay_poh", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ /**/ fd_topob_tile_in( topo, "pohi", 0UL, "metric_in", "stake_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ - /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "dedup_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */ /**/ fd_topob_tile_in( topo, "pohi", 0UL, "metric_in", "pack_replay", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_in( topo, "pack", 0UL, "metric_in", "poh_pack", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); fd_topob_tile_out( topo, "pohi", 0UL, "poh_pack", 0UL ); /* Hacky: Reserve a ulong to allow net0 to pass its PID to its neighbors */