From 99a2fdc58170f2d4164ddc3a7890949463456045 Mon Sep 17 00:00:00 2001 From: Austin Adams Date: Thu, 9 Feb 2023 12:04:24 -0600 Subject: [PATCH] Performance and Congestion improvements. 1.14 (#40) --- .github/workflows/release.yml | 170 ++- .github/workflows/test.yml | 6 +- Cargo.lock | 1034 ++++++++++++++++- Cargo.toml | 3 +- README.md | 30 +- plerkle/Cargo.toml | 21 +- plerkle/src/geyser_plugin_nft.rs | 327 ++++-- plerkle_messenger/Cargo.toml | 2 +- plerkle_messenger/src/plerkle_messenger.rs | 9 +- plerkle_messenger/src/redis_messenger.rs | 224 ++-- plerkle_serialization/Cargo.toml | 6 +- plerkle_serialization/src/lib.rs | 4 +- .../src/serializer/serializer_stable.rs | 12 +- .../solana_geyser_plugin_interface_shims.rs | 4 +- plerkle_snapshot/Cargo.toml | 21 + plerkle_snapshot/README.md | 5 + plerkle_snapshot/p.json | 14 + plerkle_snapshot/src/app_tracing.rs | 11 + plerkle_snapshot/src/config.rs | 29 + plerkle_snapshot/src/error.rs | 32 + plerkle_snapshot/src/main.rs | 56 + plerkle_snapshot/src/plugin.rs | 138 +++ plerkle_snapshot/src/process.rs | 2 + rust-toolchain.toml | 2 +- 24 files changed, 1784 insertions(+), 378 deletions(-) create mode 100644 plerkle_snapshot/Cargo.toml create mode 100644 plerkle_snapshot/README.md create mode 100644 plerkle_snapshot/p.json create mode 100644 plerkle_snapshot/src/app_tracing.rs create mode 100644 plerkle_snapshot/src/config.rs create mode 100644 plerkle_snapshot/src/error.rs create mode 100644 plerkle_snapshot/src/main.rs create mode 100644 plerkle_snapshot/src/plugin.rs create mode 100644 plerkle_snapshot/src/process.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36e5b74..2c728ca2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,17 +2,14 @@ on: push: tags: - "v*" - pull_request: - paths: - - ".github/workflows/release.yml" env: CARGO_TERM_COLOR: always IMAGE_NAME: plerkle-test-validator - RUST_VERSION: 1.59.0 - SOLANA_VERSION_STABLE: v1.13.6 - RUST_VERSION_REGRET: 1.64.0 - SOLANA_VERSION_REGRET: v1.14.13 + RUST_VERSION: 1.64.0 + SOLANA_VERSION_STABLE: v1.14.14 + RUST_VERSION_REGRET: 1.66.0 + SOLANA_VERSION_REGRET: v1.15.0 jobs: release-stable: runs-on: buildjet-8vcpu-ubuntu-2004 @@ -67,59 +64,60 @@ jobs: rust ${{ env.RUST_VERSION }} files: | ${{ env.GEYSER_PLUGIN_NAME }}-release-* - release-regret: - runs-on: buildjet-8vcpu-ubuntu-2004 - steps: - - uses: actions/checkout@v2 - - name: Set env vars - run: | - source ci/env.sh - echo "GEYSER_PLUGIN_NAME=$plugin_name" | tee -a $GITHUB_ENV - echo "GEYSER_PLUGIN_LIB=lib${plugin_lib_name}" | tee -a $GITHUB_ENV - - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get upgrade - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" - sudo apt-get update - sudo apt-get install -y libudev-dev libssl-dev libsasl2-dev libzstd-dev - sudo apt-get install -y openssl --allow-unauthenticated - sudo apt-get install -y libssl1.1 --allow-unauthenticated - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env.RUST_VERSION_REGRET }} - override: true - profile: minimal - components: rustfmt - - name: Build Plugin - run: | - echo "CI_TAG=${GITHUB_REF#refs/*/}" >> "$GITHUB_ENV" - echo "CI_OS_NAME=linux" >> "$GITHUB_ENV" - cargo build --release - - name: Build release tarball - run: ./ci/create-tarball.sh - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env.RUST_VERSION_REGRET }} - override: true - profile: minimal - components: rustfmt - - name: Build release tarball - run: ./ci/create-tarball.sh - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - name: UNSTABLE ${{ env.CI_TAG }} - body: | - ## UNSTABLE VERSION: - Reccomended for Devnet, Testnet - ${{ env.GEYSER_PLUGIN_NAME }} ${{ env.CI_TAG }} - solana ${{ env.SOLANA_VERSION_REGRET }} - rust ${{ env.RUST_VERSION_REGRET }} - files: | - ${{ env.GEYSER_PLUGIN_NAME }}-release-* + # release-regret: + # runs-on: buildjet-8vcpu-ubuntu-2004 + # steps: + # - uses: actions/checkout@v2 + # - name: Set env vars + # run: | + # source ci/env.sh + # echo "GEYSER_PLUGIN_NAME=$plugin_name" | tee -a $GITHUB_ENV + # echo "GEYSER_PLUGIN_LIB=lib${plugin_lib_name}" | tee -a $GITHUB_ENV + # - if: runner.os == 'Linux' + # run: | + # sudo apt-get update + # sudo apt-get upgrade + # wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + # sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" + # sudo apt-get update + # sudo apt-get install -y libudev-dev libssl-dev libsasl2-dev libzstd-dev + # sudo apt-get install -y openssl --allow-unauthenticated + # sudo apt-get install -y libssl1.1 --allow-unauthenticated + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: ${{ env.RUST_VERSION_REGRET }} + # override: true + # profile: minimal + # components: rustfmt + # - name: Build Plugin + # run: | + # rm plerkle-release-x86_64-unknown-linux-gnu.tar.bz2 + # echo "CI_TAG=${GITHUB_REF#refs/*/}" >> "$GITHUB_ENV" + # echo "CI_OS_NAME=linux" >> "$GITHUB_ENV" + # cargo build --release + # - name: Build release tarball + # run: ./ci/create-tarball.sh + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: ${{ env.RUST_VERSION_REGRET }} + # override: true + # profile: minimal + # components: rustfmt + # - name: Build release tarball + # run: ./ci/create-tarball.sh + # - name: Release + # uses: softprops/action-gh-release@v1 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # name: UNSTABLE ${{ env.CI_TAG }} + # body: | + # ## UNSTABLE VERSION: + # Reccomended for Devnet, Testnet + # ${{ env.GEYSER_PLUGIN_NAME }} ${{ env.CI_TAG }} + # solana ${{ env.SOLANA_VERSION_REGRET }} + # rust ${{ env.RUST_VERSION_REGRET }} + # files: | + # ${{ env.GEYSER_PLUGIN_NAME }}-release-* push-stable: runs-on: ubuntu-latest needs: release-stable @@ -146,29 +144,29 @@ jobs: echo VERSION=$TAG docker tag $IMAGE_NAME $IMAGE_ID:$TAG docker push $IMAGE_ID:$TAG - push-regret: - runs-on: ubuntu-latest - needs: release-regret - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v3 - - name: Build image - run: docker build --build-arg RUST_VERSION=${{ env.RUST_VERSION_REGRET }} --build-arg SOLANA_VERSION=v${{ env.SOLANA_VERSION_REGRET }} . --file Solana.Dockerfile --tag $IMAGE_NAME --label 'runnumber=${GITHUB_RUN_ID}' - - name: Log in to registry - run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u $ --password-stdin - - name: Push image Stable - if: startsWith(github.ref, 'refs/tags/') - run: | - CI_TAG=${GITHUB_REF#refs/*/} - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # Strip git ref prefix from version - VERSION=$CI_TAG - echo IMAGE_ID=$IMAGE_ID - export TAG=$VERSION-${{ env.RUST_VERSION_REGRET }}-${{ env.SOLANA_VERSION_REGRET }} - echo VERSION=$TAG - docker tag $IMAGE_NAME $IMAGE_ID:$TAG - docker push $IMAGE_ID:$TAG + # push-regret: + # runs-on: ubuntu-latest + # needs: release-regret + # permissions: + # packages: write + # contents: read + # steps: + # - uses: actions/checkout@v3 + # - name: Build image + # run: docker build --build-arg RUST_VERSION=${{ env.RUST_VERSION_REGRET }} --build-arg SOLANA_VERSION=v${{ env.SOLANA_VERSION_REGRET }} . --file Solana.Dockerfile --tag $IMAGE_NAME --label 'runnumber=${GITHUB_RUN_ID}' + # - name: Log in to registry + # run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u $ --password-stdin + # - name: Push image Stable + # if: startsWith(github.ref, 'refs/tags/') + # run: | + # CI_TAG=${GITHUB_REF#refs/*/} + # IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + # # Change all uppercase to lowercase + # IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + # # Strip git ref prefix from version + # VERSION=$CI_TAG + # echo IMAGE_ID=$IMAGE_ID + # export TAG=$VERSION-${{ env.RUST_VERSION_REGRET }}-${{ env.SOLANA_VERSION_REGRET }} + # echo VERSION=$TAG + # docker tag $IMAGE_NAME $IMAGE_ID:$TAG + # docker push $IMAGE_ID:$TAG diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e951936d..690584da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,10 +12,10 @@ on: env: CARGO_TERM_COLOR: always IMAGE_NAME: plerkle-test-validator - RUST_VERSION: 1.59.0 - SOLANA_VERSION_STABLE: v1.13.6 + RUST_VERSION: 1.64.0 + SOLANA_VERSION_STABLE: v1.14.14 RUST_VERSION_REGRET: 1.64.0 - SOLANA_VERSION_REGRET: v1.14.13 + SOLANA_VERSION_REGRET: v1.14.14 jobs: test-stable: diff --git a/Cargo.lock b/Cargo.lock index ca4d4b79..b4cb99bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -98,6 +104,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" + [[package]] name = "arc-swap" version = "1.6.0" @@ -335,6 +347,18 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -383,6 +407,27 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cadence" version = "0.29.0" @@ -451,6 +496,45 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -475,6 +559,19 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.42.0", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -535,6 +632,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -565,10 +676,20 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.14" @@ -604,6 +725,28 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -671,6 +814,30 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", + "rayon", +] + +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown 0.12.3", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "derivation-path" version = "0.2.0" @@ -697,6 +864,21 @@ dependencies = [ "subtle", ] +[[package]] +name = "dir-diff" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2860407d7d7e2e004bb2128510ad9e8d669e76fa005ccf567977b5d71b8b4a0b" +dependencies = [ + "walkdir", +] + +[[package]] +name = "eager" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" + [[package]] name = "ed25519" version = "1.5.3" @@ -738,6 +920,12 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -749,18 +937,18 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +checksum = "2953d1df47ac0eb70086ccabf0275aa8da8591a28bd358ee2b52bd9f9e3ff9e9" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ "proc-macro2", "quote", @@ -786,6 +974,18 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "1.8.0" @@ -816,6 +1016,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.45.0", +] + [[package]] name = "flatbuffers" version = "22.12.6" @@ -996,8 +1208,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1033,6 +1247,24 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1096,7 +1328,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.5", ] [[package]] @@ -1143,7 +1375,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.5", "pin-project-lite", "socket2", "tokio", @@ -1165,6 +1397,19 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -1215,6 +1460,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "index_list" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9d968042a4902e08810946fc7cd5851eb75e80301342305af755ca06cb82ce" + [[package]] name = "indexmap" version = "1.9.2" @@ -1225,6 +1476,18 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indicatif" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1264,6 +1527,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.5" @@ -1288,6 +1557,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "keccak" version = "0.1.3" @@ -1367,6 +1647,17 @@ dependencies = [ "libsecp256k1-core", ] +[[package]] +name = "libsqlite3-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "link-cplusplus" version = "1.0.8" @@ -1395,6 +1686,35 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.5.0" @@ -1410,6 +1730,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -1458,6 +1787,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "modular-bitfield" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53d79ba8304ac1c4f9eb3b9d281f21f7be9d4626f72ce7df4ad8fbde4f38a74" +dependencies = [ + "modular-bitfield-impl", + "static_assertions", +] + +[[package]] +name = "modular-bitfield-impl" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1476,6 +1826,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-derive" version = "0.3.3" @@ -1537,6 +1897,12 @@ dependencies = [ "syn", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.17.0" @@ -1595,10 +1961,45 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.12.1" +name = "os_str_bytes" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + +[[package]] +name = "ouroboros" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbb50b356159620db6ac971c6d5c9ab788c9cc38a6f49619fca2a27acb062ca" +dependencies = [ + "aliasable", + "ouroboros_macro", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0d9d1a6191c4f391f87219d1ea42b23f09ee84d64763cd05ee6ea88d9f384d" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core", @@ -1619,9 +2020,18 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.10.1" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216eaa586a190f0a738f2f918511eecfa90f13295abec0e457cdebcceda80cbd" +dependencies = [ + "crypto-mac", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.6", ] @@ -1655,6 +2065,50 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "pest" +version = "2.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac3922aac69a40733080f53c1ce7f91dcf57e1a5f6c52f421fadec7fbdc4b69" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d06646e185566b5961b4058dd107e0a7f56e77c3f484549fb119867773c0f202" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6f60b2ba541577e2a0c307c8f39d1439108120eb7903adeb6497fa880c59616" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.6", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1675,15 +2129,17 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "plerkle" -version = "1.3.10" +version = "1.4.0" dependencies = [ "async-trait", - "base64 0.13.1", + "base64 0.21.0", "bs58", "bytemuck", "cadence", "cadence-macros", "chrono", + "crossbeam", + "dashmap 5.4.0", "figment", "flatbuffers", "hex", @@ -1706,7 +2162,7 @@ dependencies = [ [[package]] name = "plerkle_messenger" -version = "1.3.10" +version = "1.4.0" dependencies = [ "async-mutex", "async-trait", @@ -1721,7 +2177,7 @@ dependencies = [ [[package]] name = "plerkle_serialization" -version = "1.3.10" +version = "1.4.0" dependencies = [ "chrono", "flatbuffers", @@ -1730,6 +2186,26 @@ dependencies = [ "solana-transaction-status", ] +[[package]] +name = "plerkle_snapshot" +version = "1.4.0" +dependencies = [ + "figment", + "indicatif", + "json5", + "libloading", + "reqwest", + "serde", + "serde_json", + "solana-geyser-plugin-interface", + "solana-runtime", + "solana-snapshot-etl", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "polyval" version = "0.5.3" @@ -1742,6 +2218,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1768,11 +2250,35 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -1901,7 +2407,7 @@ dependencies = [ "combine", "futures", "futures-util", - "itoa", + "itoa 1.0.5", "native-tls", "percent-encoding", "pin-project-lite", @@ -1933,6 +2439,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.28" @@ -1965,10 +2477,12 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -1978,6 +2492,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower-service", @@ -2004,6 +2519,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "rusqlite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2046,6 +2581,15 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -2117,9 +2661,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" dependencies = [ "serde", ] @@ -2137,11 +2681,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ - "itoa", + "itoa 1.0.5", "ryu", "serde", ] @@ -2153,7 +2697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.5", "ryu", "serde", ] @@ -2210,6 +2754,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -2262,9 +2815,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd16f344fe90259904100d960a347db6026f83ebca89b34e03f9754b10ed6bb" +checksum = "3476a9ecc99e122f37ed91e6e3e907840fac95a813c4231eef6dae1646b12a2f" dependencies = [ "Inflector", "base64 0.13.1", @@ -2275,6 +2828,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "solana-address-lookup-table-program", "solana-config-program", "solana-sdk", "solana-vote-program", @@ -2284,11 +2838,57 @@ dependencies = [ "zstd", ] +[[package]] +name = "solana-address-lookup-table-program" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536eb4cfe1e6c0699d45222eb0a326be634deff9f1d67ff0e8c303af46265fce" +dependencies = [ + "bincode", + "bytemuck", + "log", + "num-derive", + "num-traits", + "rustc_version", + "serde", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-program", + "solana-program-runtime", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-bucket-map" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf65b7fd9c5d79778ca25845aca653f786ac998a9a72591b2c6889cb9a814d7" +dependencies = [ + "log", + "memmap2", + "modular-bitfield", + "rand", + "solana-measure", + "solana-sdk", + "tempfile", +] + +[[package]] +name = "solana-compute-budget-program" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7873d2593d6357378d8e6a2c3772e355c5f07be67a0df5436c1b374d24f41439" +dependencies = [ + "solana-program-runtime", + "solana-sdk", +] + [[package]] name = "solana-config-program" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7896f0b395f048d95fd37690ff9f8caa634660bd955c2172bfa847cc44cf8cc" +checksum = "b5a2b03767f3916c4697a1f60e1aa3a47424406933db5c80dd4502564443a6ab" dependencies = [ "bincode", "chrono", @@ -2300,31 +2900,43 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f845c62a8aee31ad250c08345818a7b0da10ebf7e1e5fdf5f5e735b6dee766" +checksum = "72e131e5e67830c24dea3a916e8bcb8404de36febe015b569c1843282283896b" dependencies = [ + "ahash", + "blake3", + "block-buffer 0.9.0", "bs58", "bv", + "byteorder", + "cc", + "either", "generic-array", + "getrandom 0.1.16", + "hashbrown 0.12.3", "im", "lazy_static", "log", "memmap2", + "once_cell", + "rand_core 0.6.4", "rustc_version", "serde", "serde_bytes", "serde_derive", + "serde_json", "sha2 0.10.6", "solana-frozen-abi-macro", + "subtle", "thiserror", ] [[package]] name = "solana-frozen-abi-macro" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ebd2c533f11262885f1131e52ea6136e7c72fffb18c858ffee05964c5d2beb7" +checksum = "3ce6b1cbbc9a929eaebb8f009e54d351e411b85f622040e50f9b2d8d0f4a8649" dependencies = [ "proc-macro2", "quote", @@ -2334,9 +2946,9 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-interface" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615c43289ff7b371fcacad0e560911eb3a5d95d491dd8b65622d534fa8463bb0" +checksum = "b942f5b94c7aa6b5c9bc0f71b3c3cf8c9683c7b861d75990b37aca8de3fdeae5" dependencies = [ "log", "solana-sdk", @@ -2346,9 +2958,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6829405e67e41d2598ad3b5fb3fab83798332897d08b6a43df1b124e12aae43" +checksum = "202ab12577144fe5573f3368dcb49246455fd0861aba76bac8a42904366d0313" dependencies = [ "env_logger", "lazy_static", @@ -2357,9 +2969,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ba179113437b5d1dded5075f954d5a9220fe2fad67726a096be396e7cb76ab" +checksum = "e98bf5c9183f9aaa9a7cdb3baa6e34c505b85c51bec24a4c5e3d877c8d55ea9f" dependencies = [ "log", "solana-sdk", @@ -2367,9 +2979,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c49cc205c220e613ec3345444494227bac59ad859f6666d7407b3179b65b279" +checksum = "f87de747c0fc1965d1fbc42192478d51f2119cc523dec219ee4877026222223f" dependencies = [ "crossbeam-channel", "gethostname", @@ -2381,9 +2993,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b218d6f58a793dfd2a7df80c5e9d289d1d78a22dc4975aa9962b9726f5c182ae" +checksum = "32c5a1723637282e578a21f138eaef6d44fd234ac267525e3ec01569d06c278a" dependencies = [ "base64 0.13.1", "bincode", @@ -2394,41 +3006,49 @@ dependencies = [ "bs58", "bv", "bytemuck", + "cc", "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.1.16", + "getrandom 0.2.8", "itertools", "js-sys", "lazy_static", + "libc", "libsecp256k1", "log", + "memoffset 0.6.5", "num-derive", "num-traits", "parking_lot", "rand", + "rand_chacha", "rustc_version", "rustversion", "serde", "serde_bytes", "serde_derive", + "serde_json", "sha2 0.10.6", "sha3 0.10.6", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-sdk-macro", "thiserror", + "tiny-bip39", "wasm-bindgen", + "zeroize", ] [[package]] name = "solana-program-runtime" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6571d34ae29ddff6aa5e99e5f22163e4194ca93be7ab5f61a4d86a0579b0d90" +checksum = "d807ad5c1d4fcb72c26d002c7da1ee72e7c7353865f89e426843dcf0f8ab0f3f" dependencies = [ "base64 0.13.1", "bincode", + "eager", "enum-iterator", "itertools", "libc", @@ -2436,20 +3056,93 @@ dependencies = [ "log", "num-derive", "num-traits", + "rand", + "rustc_version", + "serde", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-measure", + "solana-metrics", + "solana-sdk", + "thiserror", +] + +[[package]] +name = "solana-rayon-threadlimit" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d1ce0d493fc7fce14ea85fdbf413c90911ad38f2ed97abd66358fc7cf408e6" +dependencies = [ + "lazy_static", + "num_cpus", +] + +[[package]] +name = "solana-runtime" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f3c2189b45ddaef98411f71873416c7d827279e400c4a4189655b2353b0cd" +dependencies = [ + "arrayref", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder", + "bzip2", + "crossbeam-channel", + "dashmap 4.0.2", + "dir-diff", + "flate2", + "fnv", + "im", + "index_list", + "itertools", + "lazy_static", + "log", + "lru", + "lz4", + "memmap2", + "num-derive", + "num-traits", + "num_cpus", + "once_cell", + "ouroboros", + "rand", + "rayon", + "regex", "rustc_version", "serde", + "serde_derive", + "solana-address-lookup-table-program", + "solana-bucket-map", + "solana-compute-budget-program", + "solana-config-program", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-measure", + "solana-metrics", + "solana-program-runtime", + "solana-rayon-threadlimit", "solana-sdk", + "solana-stake-program", + "solana-vote-program", + "solana-zk-token-proof-program", + "solana-zk-token-sdk", + "strum", + "strum_macros", + "symlink", + "tar", + "tempfile", "thiserror", + "zstd", ] [[package]] name = "solana-sdk" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad11e736b23ba40f3e245c2d558a2bb57e733a2c6bb6874d48f3efc6c16b435" +checksum = "7c2284232a7da506454d1e41f348d667119faf9e9c60211f5c62d0bbfb405d8a" dependencies = [ "assert_matches", "base64 0.13.1", @@ -2474,7 +3167,7 @@ dependencies = [ "memmap2", "num-derive", "num-traits", - "pbkdf2", + "pbkdf2 0.11.0", "qstring", "rand", "rand_chacha", @@ -2498,9 +3191,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74ea5bf7eca00bc00e34453e11ee0f35cc15dab08c7167b5b01ef88623628246" +checksum = "66b1eae3692dcafb7e02ea1a463b7e387ed4ada840a324d3857fd7541c14cb0c" dependencies = [ "bs58", "proc-macro2", @@ -2509,11 +3202,66 @@ dependencies = [ "syn", ] +[[package]] +name = "solana-snapshot-etl" +version = "0.3.0" +source = "git+https://github.com/austbot/solana-snapshot-etl#2042b4b4e9c90f17921f2ed07b33bce6f131caed" +dependencies = [ + "bincode", + "borsh", + "clap", + "crossbeam", + "csv", + "env_logger", + "indicatif", + "itertools", + "json5", + "libloading", + "log", + "memmap2", + "num_cpus", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "solana-geyser-plugin-interface", + "solana-program", + "solana-runtime", + "solana-sdk", + "spl-token", + "tar", + "thiserror", + "zstd", +] + +[[package]] +name = "solana-stake-program" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5af4559a4a3180f3ef3e7aa827ec700b50b4f8113799b2fb6d12eb692e09e47" +dependencies = [ + "bincode", + "log", + "num-derive", + "num-traits", + "rustc_version", + "serde", + "serde_derive", + "solana-config-program", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-metrics", + "solana-program-runtime", + "solana-sdk", + "solana-vote-program", + "thiserror", +] + [[package]] name = "solana-transaction-status" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2416f64ddcd0dd8177242156d31c477a87f08b521423c12834b85a68446606ff" +checksum = "cbb7a4689fb7a1938b2956cd01cba081ebfb02bccda3b7926c5a9090e68b630b" dependencies = [ "Inflector", "base64 0.13.1", @@ -2526,6 +3274,7 @@ dependencies = [ "serde_derive", "serde_json", "solana-account-decoder", + "solana-address-lookup-table-program", "solana-measure", "solana-metrics", "solana-sdk", @@ -2539,9 +3288,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d63290a58e00fb083547e29e05165bcd2d32f3287da0c2edf997c1c284e78470" +checksum = "098571b9ad25da26b1e5811bd15173cd0a0b9c6d724768236feebf1bf28e6d98" dependencies = [ "bincode", "log", @@ -2558,11 +3307,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-zk-token-proof-program" +version = "1.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a10082c3f5e76e377b7b024a8872a98a2070dedc09aa28ebf1c092b8237e58" +dependencies = [ + "bytemuck", + "getrandom 0.1.16", + "num-derive", + "num-traits", + "solana-program-runtime", + "solana-sdk", + "solana-zk-token-sdk", +] + [[package]] name = "solana-zk-token-sdk" -version = "1.13.6" +version = "1.14.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f631034507984b0199e6e6be15d8c8bda33607b523094139b6ece91cd267c733" +checksum = "3fa419f14c8fb7d0c775cbd202377a77f10e80b1b4ac39a8c56aec1910b5c374" dependencies = [ "aes-gcm-siv", "arrayref", @@ -2573,6 +3337,7 @@ dependencies = [ "cipher 0.4.3", "curve25519-dalek", "getrandom 0.1.16", + "itertools", "lazy_static", "merlin", "num-derive", @@ -2596,9 +3361,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spl-associated-token-account" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a33ecc83137583902c3e13c02f34151c8b2f2b74120f9c2b3ff841953e083d" +checksum = "fbc000f0fdf1f12f99d77d398137c1751345b18c88258ce0f99b7872cf6c9bd6" dependencies = [ "assert_matches", "borsh", @@ -2636,9 +3401,9 @@ dependencies = [ [[package]] name = "spl-token-2022" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a97cbf60b91b610c846ccf8eecca96d92a24a19ffbf9fe06cd0c84e76ec45e" +checksum = "0edb869dbe159b018f17fb9bfa67118c30f232d7f54a73742bc96794dff77ed8" dependencies = [ "arrayref", "bytemuck", @@ -2652,12 +3417,52 @@ dependencies = [ "thiserror", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.107" @@ -2681,6 +3486,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -2704,6 +3520,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.38" @@ -2724,6 +3546,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "time" version = "0.1.45" @@ -2735,6 +3566,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "tiny-bip39" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc59cb9dfc85bb312c3a78fd6aa8a8582e310b0fa885d5bb877f6dcc601839d" +dependencies = [ + "anyhow", + "hmac 0.8.1", + "once_cell", + "pbkdf2 0.4.0", + "rand", + "rustc-hash", + "sha2 0.9.9", + "thiserror", + "unicode-normalization", + "wasm-bindgen", + "zeroize", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2783,9 +3633,9 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -2861,6 +3711,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -2875,6 +3751,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + [[package]] name = "uncased" version = "0.9.7" @@ -2954,6 +3836,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2966,6 +3854,17 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -3210,6 +4109,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + [[package]] name = "yansi" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 27d8f9b6..96e4c455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,6 @@ members = [ "plerkle_messenger", "plerkle", - "plerkle_serialization" + "plerkle_serialization", + "plerkle_snapshot" ] \ No newline at end of file diff --git a/README.md b/README.md index 979af56c..d33780ee 100644 --- a/README.md +++ b/README.md @@ -79,11 +79,9 @@ The process running the validator must have access to environment variables. Tho ```bash RUST_LOG=warn -PLUGIN_CONFIG_RELOAD_TTL=300 PLUGIN_MESSENGER_CONFIG='{ messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } }' ``` -The PLUGIN_CONFIG_TTL_RELOAD tells the plugin how long to keep the geyser plugin file cached in seconds. This allows hot reloading of what programs you are listening to without restarting the validator. The PLUGIN_MESSENGER_CONFIG determins which compiled messenger to select and a specific configuration for the messenger. @@ -93,6 +91,9 @@ The PLUGIN_MESSENGER_CONFIG determins which compiled messenger to select and a s - "pipeline_size_bytes" - Maximum command size, roughly equates to the payload size. This setting locally buffers bytes in a queue to be flushed when the buffere grows past the desired amount. Default is 512mb(max redis command size) / 100, maximum is 512mb(max redis command size) / 100. You should test your optimal size to avoid high send latency and avoid RTT. - "local_buffer_max_window" - Maximum time to wait for the buffer to fill be for flushing. For lower traffic you dont want to be waiting around so set a max window and it will send at a minumum of every X milliseconds . Default 1000 +- "confirmation_level" - Can be one of "Processed", "Confirmed", "Rooted". Defaults to Processed this is the level we wait for before sending. "Processed" is essentially when we first see it which can on rare cases be reverted. "Confirmed" has extremley low likley hood of being reverted but takes longer (~1k ms in our testing) to show up. "Rooted" is impossible to revert but takes the longest. +- "num_workers" - This is the number of workers who will pickup notifications from the plugin and send them to the messenger. Default is 5 + ``` Lower Scale Low network latency @@ -116,10 +117,33 @@ PLUGIN_MESSENGER_CONFIG='{pipeline_size_bytes=50000000,local_buffer_max_window=5 ``` -PLUGIN_MESSENGER_CONFIG='{batch_size=100,message_wait_timeout=5,retries=5, consumer_id="random_string",messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } }' +PLUGIN_MESSENGER_CONFIG='{batch_size=1000,message_wait_timeout=5,retries=5, consumer_id="random_string",messenger_type="Redis", connection_config={ redis_connection_str="redis://redis" } }' ``` + +*** Hardcoded Configuration *** +We are still tuning some fo the default values for max stream size but this is what we have currently +``` +msg.set_buffer_size(ACCOUNT_STREAM,100_000_000).await; +msg.set_buffer_size(SLOT_STREAM, 100_000).await; +msg.set_buffer_size(TRANSACTION_STREAM, 10_000_000).await; +msg.set_buffer_size(BLOCK_STREAM, 100_000).await; + +``` + +NOTE: in 1.4.0 we are not sending to slot status. + + +### Metrics +The plugin exposes the following statsd metrics +count plugin.startup -> times the plugin started +time message_send_queue_time -> time spent on messenger internal buffer +time message_send_latency -> rtt time to messenger bus +count account_seen_event , tags: owner , is_startup -> number of account events filtered and seen +time startup.timer -> startup flush timer +count transaction_seen_event tags slot-idx -> number of filtered txns seen + ### Building With Docker This repo contains a docker File that allows you to run an test the plerkle plugin using a test validator. diff --git a/plerkle/Cargo.toml b/plerkle/Cargo.toml index e3f201d5..32cc2b11 100644 --- a/plerkle/Cargo.toml +++ b/plerkle/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "plerkle" description = "Geyser plugin with dynamic config reloading, message bus agnostic abstractions and a whole lot of fun." -version = "1.3.11" +version = "1.4.0" authors = ["Metaplex Developers "] repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" license = "AGPL-3.0" @@ -14,12 +14,12 @@ crate-type = ["cdylib", "rlib"] [dependencies] log = "0.4.11" async-trait = "0.1.53" -solana-sdk = { version ="=1.13.6" } -solana-transaction-status = { version = "=1.13.6" } -solana-geyser-plugin-interface = { version = "=1.13.6" } -solana-logger = { version = "=1.13.6" } +solana-sdk = { version ="=1.14.14" } +solana-transaction-status = { version = "=1.14.14" } +solana-geyser-plugin-interface = { version = "=1.14.14" } +solana-logger = { version = "=1.14.14" } thiserror = "1.0.30" -base64 = "0.13.0" +base64 = "0.21.0" lazy_static = "1.4.0" bs58 = "0.4.0" bytemuck = "1.7.2" @@ -31,14 +31,13 @@ cadence-macros = "0.29.0" chrono = "0.4.19" tracing = "0.1.35" hex = "0.4.3" -plerkle_messenger = { path = "../plerkle_messenger", version = "1.3.11", features = ["redis"] } +plerkle_messenger = { path = "../plerkle_messenger", version = "1.4.0", features = ["redis"] } flatbuffers = "22.10.26" -plerkle_serialization = { path = "../plerkle_serialization", version = "1.3.11" } +plerkle_serialization = { path = "../plerkle_serialization", version = "1.4.0" } tokio = { version = "1.23.0", features = ["full"] } figment = { version = "0.10.6", features = ["env", "test"] } - - - +dashmap = {version = "5.4.0"} +crossbeam = {version = "0.8.2"} [dependencies.num-integer] version = "0.1.44" diff --git a/plerkle/src/geyser_plugin_nft.rs b/plerkle/src/geyser_plugin_nft.rs index 6a8252a5..7c38a78d 100644 --- a/plerkle/src/geyser_plugin_nft.rs +++ b/plerkle/src/geyser_plugin_nft.rs @@ -4,6 +4,8 @@ use crate::{ }; use cadence::{BufferedUdpMetricSink, QueuingMetricSink, StatsdClient}; use cadence_macros::*; +use crossbeam::channel::{unbounded, Sender}; +use dashmap::DashMap; use figment::{providers::Env, Figment}; use flatbuffers::FlatBufferBuilder; use log::*; @@ -12,26 +14,28 @@ use plerkle_messenger::{ TRANSACTION_STREAM, }; use plerkle_serialization::serializer::{ - serialize_account, serialize_block, serialize_slot_status, serialize_transaction, + serialize_account, serialize_block, serialize_transaction, }; use serde::Deserialize; use solana_geyser_plugin_interface::geyser_plugin_interface::{ - GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, - ReplicaBlockInfoVersions, ReplicaTransactionInfoVersions, Result, - SlotStatus, + GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, + ReplicaTransactionInfoVersions, Result, SlotStatus, }; use solana_sdk::{message::AccountKeys, pubkey::Pubkey}; use std::{ + collections::BTreeSet, fmt::{Debug, Formatter}, fs::File, io::Read, net::UdpSocket, + ops::Bound::Included, + ops::RangeBounds, + sync::Arc, }; use tokio::{ self as tokio, runtime::{Builder, Runtime}, - sync::mpsc::{self as mpsc, Sender}, time::Instant, }; @@ -41,6 +45,49 @@ struct SerializedData<'a> { seen_at: Instant, } +#[derive(Default)] +pub struct SlotStore { + parents: BTreeSet, +} +const SLOT_EXPIRY: u64 = 600 * 2; +impl SlotStore { + pub fn new() -> Self { + SlotStore { + parents: BTreeSet::new(), + } + } + + pub fn has_children(&self, slot: u64) -> bool { + self.parents.contains(&slot) + } + + pub fn needs_purge(&self, current_slot: u64) -> Option> { + if current_slot <= SLOT_EXPIRY { + //just in case we do some testing + return None; + } + + let rng = self + .parents + .range((Included(0), Included(current_slot - SLOT_EXPIRY))) + .cloned() + .collect(); + Some(rng) + } + + pub fn insert(&mut self, parent: u64) { + self.parents.insert(parent); + } + + pub fn remove(&mut self, slot: u64) { + self.parents.remove(&slot); + } + + pub fn remove_range(&mut self, range: impl RangeBounds) { + self.parents.retain(|slot| range.contains(slot)); + } +} + #[derive(Default)] pub(crate) struct Plerkle<'a> { runtime: Option, @@ -49,19 +96,64 @@ pub(crate) struct Plerkle<'a> { sender: Option>>, started_at: Option, handle_startup: bool, + slots_seen: SlotStore, + account_event_cache: Arc)>>>, + conf_level: Option, +} + +#[derive(Deserialize, PartialEq, Debug)] +pub enum ConfirmationLevel { + Processed, + Rooted, + Confirmed, } +impl Into for ConfirmationLevel { + fn into(self) -> SlotStatus { + match self { + ConfirmationLevel::Processed => SlotStatus::Processed, + ConfirmationLevel::Rooted => SlotStatus::Rooted, + ConfirmationLevel::Confirmed => SlotStatus::Confirmed, + } + } +} + + #[derive(Deserialize, PartialEq, Debug)] pub struct PluginConfig { pub messenger_config: MessengerConfig, + pub num_workers: Option, pub config_reload_ttl: Option, + pub confirmation_level: Option } -const MSG_BUFFER_SIZE: usize = 1000000; +const NUM_WORKERS: usize = 5; impl<'a> Plerkle<'a> { pub fn new() -> Self { - Self::default() + Plerkle { + runtime: None, + accounts_selector: None, + transaction_selector: None, + sender: None, + started_at: None, + handle_startup: false, + slots_seen: SlotStore::new(), + account_event_cache: Arc::new(DashMap::new()), + conf_level: None, + } + } + + fn send( + sender: Sender>, + runtime: &tokio::runtime::Runtime, + data: SerializedData<'static>, + ) -> Result<()> { + // Send account info over channel. + runtime.spawn(async move { + let _ = sender.send(data); + }); + Ok(()) } fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector { @@ -142,6 +234,10 @@ impl<'a> Plerkle<'a> { } } + fn get_confirmation_level(&self) ->SlotStatus { + self.conf_level.unwrap_or(SlotStatus::Processed) + } + // Currently not used but may want later. pub fn _txn_contains_program<'b>(keys: AccountKeys, program: &Pubkey) -> bool { keys.iter() @@ -216,51 +312,61 @@ impl GeyserPlugin for Plerkle<'static> { msg: format!("Could not create tokio runtime: {:?}", err), })?; - let (sender, mut receiver) = mpsc::channel::(MSG_BUFFER_SIZE); - self.sender = Some(sender); + let (sender, mut receiver) = unbounded::(); + let config: PluginConfig = Figment::new() .join(Env::prefixed("PLUGIN_")) .extract() .map_err(|config_error| GeyserPluginError::ConfigFileReadError { msg: format!("Could not read messenger config: {:?}", config_error), })?; + self.conf_level = config.confirmation_level.map(|c| c.into()); + let workers_num = config.num_workers.unwrap_or(NUM_WORKERS); runtime.spawn(async move { - // Create new Messenger connection. - if let Ok(mut messenger) = select_messenger(config.messenger_config).await { - if messenger.add_stream(ACCOUNT_STREAM).await.is_err() { - error!("Error adding ACCOUNT stream"); - } - - if messenger.add_stream(SLOT_STREAM).await.is_err() { - error!("Error adding SLOT stream"); - } - - if messenger.add_stream(TRANSACTION_STREAM).await.is_err() { - error!("Error adding TRANSACTION stream"); - } - - if messenger.add_stream(BLOCK_STREAM).await.is_err() { - error!("Error adding BLOCK stream"); - } + let mut messenger_workers = Vec::new(); + for _ in 0..workers_num { + let mut msg = select_messenger(config.messenger_config.clone()) + .await + .unwrap(); // We want to fail if the messenger is not configured correctly. + msg.add_stream(ACCOUNT_STREAM).await; + msg.add_stream(SLOT_STREAM).await; + msg.add_stream(TRANSACTION_STREAM).await; + msg.add_stream(BLOCK_STREAM).await; + msg.set_buffer_size(ACCOUNT_STREAM,100_000_000).await; + msg.set_buffer_size(SLOT_STREAM, 100_000).await; + msg.set_buffer_size(TRANSACTION_STREAM, 10_000_000).await; + msg.set_buffer_size(BLOCK_STREAM, 100_000).await; + // Idempotent call to add streams. + + messenger_workers.push(msg); + } - messenger.set_buffer_size(ACCOUNT_STREAM, 10_000_000).await; - messenger.set_buffer_size(SLOT_STREAM, 100_000).await; - messenger - .set_buffer_size(TRANSACTION_STREAM, 10_000_000) - .await; - messenger.set_buffer_size(BLOCK_STREAM, 100_000).await; - // Receive messages in a loop as long as at least one Sender is in scope. - while let Some(data) = receiver.recv().await { - let start = Instant::now(); - let bytes = data.builder.finished_data(); - let _ = messenger.send(data.stream, bytes).await; - safe_metric(|| { - statsd_time!("message_send_queue_time", data.seen_at.elapsed()); - statsd_time!("message_send_latency", start.elapsed()); - }) - } + for mut worker in messenger_workers.into_iter() { + let receiver = receiver.clone(); + tokio::spawn(async move { + while let Ok(data) = receiver.recv() { + let start = Instant::now(); + let bytes = data.builder.finished_data(); + safe_metric(|| { + statsd_time!( + "message_send_queue_time", + data.seen_at.elapsed().as_millis() as u64, + "stream" => data.stream + ); + }); + let _ = worker.send(data.stream, bytes).await; + safe_metric(|| { + statsd_time!( + "message_send_latency", + start.elapsed().as_millis() as u64, + "stream" => data.stream + ); + }) + } + }); } }); + self.sender = Some(sender); self.runtime = Some(runtime); Ok(()) } @@ -275,13 +381,24 @@ impl GeyserPlugin for Plerkle<'static> { slot: u64, is_startup: bool, ) -> solana_geyser_plugin_interface::geyser_plugin_interface::Result<()> { - let seen = Instant::now(); if !self.handle_startup && is_startup { return Ok(()); } let rep: plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2; let account = match account { - //ReplicaAccountInfoVersions::V0_0_2(ai) => ai, + ReplicaAccountInfoVersions::V0_0_2(ai) => { + rep = plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2 { + pubkey: ai.pubkey, + lamports: ai.lamports, + owner: ai.owner, + executable: ai.executable, + rent_epoch: ai.rent_epoch, + data: ai.data, + write_version: ai.write_version, + txn_signature: ai.txn_signature, + }; + &rep + } ReplicaAccountInfoVersions::V0_0_1(ai) => { rep = plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaAccountInfoV2 { pubkey: ai.pubkey, @@ -305,27 +422,48 @@ impl GeyserPlugin for Plerkle<'static> { msg: "Accounts selector not initialized".to_string(), }); } + let seen = Instant::now(); // Get runtime and sender channel. - let runtime = self.get_runtime()?; - let sender = self.get_sender_clone()?; - // Serialize data. let builder = FlatBufferBuilder::new(); let builder = serialize_account(builder, account, slot, is_startup); let owner = bs58::encode(account.owner).into_string(); - // Send account info over channel. - runtime.spawn(async move { - let data = SerializedData { - stream: ACCOUNT_STREAM, - builder, - seen_at: seen.clone(), - }; - let _ = sender.send(data).await; - safe_metric(|| { - let s = is_startup.to_string(); - statsd_count!("account_seen_event", 1, "owner" => &owner, "is_startup" => &s); - }); + safe_metric(|| { + let s = is_startup.to_string(); + statsd_count!("account_seen_event", 1, "owner" => &owner, "is_startup" => &s); }); + let data = SerializedData { + stream: ACCOUNT_STREAM, + builder, + seen_at: seen, + }; + let runtime = self.get_runtime()?; + let sender = self.get_sender_clone()?; + + if is_startup { + Plerkle::send(sender, runtime, data)?; + } else { + let account_key = Pubkey::new(account.pubkey); + let cache = self.account_event_cache.get_mut(&slot); + if let Some(cache) = cache { + if cache.contains_key(&account_key) { + cache.alter(&account_key, |_, v| { + if account.write_version > v.0 { + return (account.write_version, data); + } else { + v + } + }); + } else { + cache.insert(account_key, (account.write_version, data)); + } + } else { + let pubkey_cache = DashMap::new(); + pubkey_cache.insert(account_key, (account.write_version, data)); + self.account_event_cache.insert(slot, pubkey_cache); + } + } + Ok(()) } @@ -345,30 +483,36 @@ impl GeyserPlugin for Plerkle<'static> { parent: Option, status: SlotStatus, ) -> solana_geyser_plugin_interface::geyser_plugin_interface::Result<()> { - let seen = Instant::now(); - // Get runtime and sender channel. - let runtime = self.get_runtime()?; - let sender = self.get_sender_clone()?; - - // Serialize data. - let builder = FlatBufferBuilder::new(); - let status = match status { - SlotStatus::Rooted => plerkle_serialization::solana_geyser_plugin_interface_shims::SlotStatus::Rooted, - SlotStatus::Processed => plerkle_serialization::solana_geyser_plugin_interface_shims::SlotStatus::Processed, - SlotStatus::Confirmed => plerkle_serialization::solana_geyser_plugin_interface_shims::SlotStatus::Confirmed, - }; - let builder = serialize_slot_status(builder, slot, parent, status); - - // Send slot status over channel. - runtime.spawn(async move { - let data = SerializedData { - stream: SLOT_STREAM, - builder, - seen_at: seen.clone(), - }; - let _ = sender.send(data).await; - }); - + info!("Slot status update: {:?} {:?}", slot, status); + if status == SlotStatus::Processed && parent.is_some() { + self.slots_seen.insert(parent.unwrap()); + } + if status == self.get_confirmation_level() { // playing with this value here + let slot_map = self.account_event_cache.remove(&slot); + if let Some((_, events)) = slot_map { + info!("Sending Account events for SLOT: {:?}", slot); + for (_, event) in events.into_iter() { + info!("Sending Account event for stream: {:?}", event.1.stream); + let sender = self.get_sender_clone()?; + let runtime = self.get_runtime()?; + Plerkle::send(sender, runtime, event.1)?; + } + } + let seen = &mut self.slots_seen; + let slots_to_purge = seen.needs_purge(slot); + if let Some(purgable) = slots_to_purge { + debug!("Purging slots: {:?}", purgable); + for slot in &purgable { + seen.remove(*slot); + } + let cl = self.account_event_cache.clone(); + self.get_runtime()?.spawn(async move { + for s in purgable { + cl.remove(&s); + } + }); + } + } Ok(()) } @@ -380,7 +524,16 @@ impl GeyserPlugin for Plerkle<'static> { let seen = Instant::now(); let rep: plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2; let transaction_info = match transaction_info { - //ReplicaTransactionInfoVersions::V0_0_2(ti) => ti, + ReplicaTransactionInfoVersions::V0_0_2(ti) => { + rep = plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2 { + signature: ti.signature, + is_vote: ti.is_vote, + transaction: ti.transaction, + transaction_status_meta: ti.transaction_status_meta, + index: ti.index, + }; + &rep + } ReplicaTransactionInfoVersions::V0_0_1(ti) => { rep = plerkle_serialization::solana_geyser_plugin_interface_shims::ReplicaTransactionInfoV2 { signature: ti.signature, @@ -423,7 +576,7 @@ impl GeyserPlugin for Plerkle<'static> { builder, seen_at: seen.clone(), }; - let _ = sender.send(data).await; + let _ = sender.send(data); }); safe_metric(|| { statsd_count!("transaction_seen_event", 1, "slot-idx" => &slt_idx); @@ -454,7 +607,7 @@ impl GeyserPlugin for Plerkle<'static> { block_height: block_info.block_height, executed_transaction_count: 0, }; - + let builder = serialize_block(builder, &block_info); // Send block info over channel. @@ -464,7 +617,7 @@ impl GeyserPlugin for Plerkle<'static> { builder, seen_at: seen.clone(), }; - let _ = sender.send(data).await; + let _ = sender.send(data); }); } } diff --git a/plerkle_messenger/Cargo.toml b/plerkle_messenger/Cargo.toml index b84577e3..e039bf25 100644 --- a/plerkle_messenger/Cargo.toml +++ b/plerkle_messenger/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "plerkle_messenger" description = "Metaplex Messenger trait for Geyser plugin producer/consumer patterns." -version = "1.3.11" +version = "1.4.0" authors = ["Metaplex Developers "] repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" license = "AGPL-3.0" diff --git a/plerkle_messenger/src/plerkle_messenger.rs b/plerkle_messenger/src/plerkle_messenger.rs index e6bbfa35..605fc526 100644 --- a/plerkle_messenger/src/plerkle_messenger.rs +++ b/plerkle_messenger/src/plerkle_messenger.rs @@ -31,6 +31,13 @@ impl RecvData { } } +#[derive(Debug, PartialEq, Eq)] +pub enum ConsumptionType { + New, + Redeliver, + All, +} + #[async_trait] pub trait Messenger: Sync + Send { async fn new(config: MessengerConfig) -> Result @@ -40,7 +47,7 @@ pub trait Messenger: Sync + Send { async fn add_stream(&mut self, stream_key: &'static str) -> Result<(), MessengerError>; async fn set_buffer_size(&mut self, stream_key: &'static str, max_buffer_size: usize); async fn send(&mut self, stream_key: &'static str, bytes: &[u8]) -> Result<(), MessengerError>; - async fn recv(&mut self, stream_key: &'static str) -> Result, MessengerError>; + async fn recv(&mut self, stream_key: &'static str, consumption_type: ConsumptionType) -> Result, MessengerError>; async fn stream_size(&mut self, stream_key: &'static str) -> Result; // Ack-ing messages is made a bit awkward by the current interface layout because diff --git a/plerkle_messenger/src/redis_messenger.rs b/plerkle_messenger/src/redis_messenger.rs index f95a059a..c46ef86f 100644 --- a/plerkle_messenger/src/redis_messenger.rs +++ b/plerkle_messenger/src/redis_messenger.rs @@ -1,4 +1,6 @@ -use crate::{error::MessengerError, Messenger, MessengerConfig, MessengerType, RecvData}; +use crate::{ + error::MessengerError, ConsumptionType, Messenger, MessengerConfig, MessengerType, RecvData, +}; use async_trait::async_trait; use log::*; @@ -28,7 +30,7 @@ pub const MESSAGE_WAIT_TIMEOUT: usize = 10; pub const IDLE_TIMEOUT: usize = 5000; pub const REDIS_MAX_BYTES_COMMAND: usize = 536870912; pub const PIPELINE_SIZE_BYTES: usize = REDIS_MAX_BYTES_COMMAND / 100; -pub const PIPELINE_MAX_TIME: u64 = 1000; +pub const PIPELINE_MAX_TIME: u64 = 10; pub struct RedisMessenger { connection: ConnectionManager, @@ -58,12 +60,6 @@ impl RedisMessenger { stream_key: &'static str, ) -> Result, MessengerError> { let mut id = "0-0".to_owned(); - // We need to call `XAUTOCLAIM` repeatedly because it will (according to the docs) - // only look at up to 10 * `count` PEL entries each time, and `id` is used to - // know where we left off to continue from next call. - - // The `redis` crate doesn't appear to support this command so we have - // to call it via the lower level primitives it provides. let mut xauto = cmd("XAUTOCLAIM"); xauto .arg(stream_key) @@ -75,10 +71,7 @@ impl RedisMessenger { .arg("COUNT") .arg(self.batch_size); - // Before Redis 7 (we're using 6.2.x presently), `XAUTOCLAIM` returns an array of - // two items: an id to be used for the next call to continue scanning the PEL, - // and a list of successfully claimed messages in the same format as `XRANGE`. - let result: (String, StreamRangeReply) = xauto + let result: (String, StreamRangeReply, Vec) = xauto .query_async(&mut self.connection) .await .map_err(|e| MessengerError::AutoclaimError { msg: e.to_string() })?; @@ -92,60 +85,66 @@ impl RedisMessenger { } let mut retained_ids = Vec::new(); + let f = range_reply.ids.first().unwrap(); + let l = range_reply.ids.last().unwrap(); // We need to use `xpending_count` to get a `StreamPendingCountReply` which - // contains information about the number of times a message has been - // delivered. - + // // contains information about the number of times a message has been + // // delivered. + let pending_result: StreamPendingCountReply = self + .connection + .xpending_count( + stream_key, + self.consumer_group_name.clone(), + &f.id.clone(), + &l.id.clone(), + range_reply.ids.len(), + ) + .await + .map_err(|e| { + error!("Redis receive error: {e}"); + MessengerError::ReceiveError { msg: e.to_string() } + })?; + let mut pending = HashMap::new(); + let mut ack_list = Vec::new(); + let prs = pending_result.ids.into_iter(); + for pr in prs { + pending.insert(pr.id.clone(), pr); + } for sid in range_reply.ids { - let pending_result: RedisResult = self - .connection - .xpending_count( - stream_key, - self.consumer_group_name.clone(), - &sid.id, - &sid.id, - 1, - ) - .await; - - match pending_result { - Ok(reply) => { - if reply.ids.is_empty() { - error!("Missing pending message information for id {}", id); - } else { - let info = reply.ids.first().unwrap(); - let StreamId { id, map } = sid; - let data = if let Some(data) = map.get(DATA_KEY) { - data - } else { - println!("No Data was stored in Redis for ID {id}"); - continue; - }; - // Get data from map. - - let bytes = match data { - Value::Data(bytes) => bytes, - _ => { - println!("Redis data for ID {id} in wrong format"); - continue; - } - }; - - if info.times_delivered > self.retries { - self.ack_msg(stream_key, &[id.clone()]).await?; - error!("Message has reached maximum retries {} for id", id); - continue; - } - retained_ids.push(RecvData::new_retry( - id, - bytes.to_vec(), - info.times_delivered, - )); - } + let StreamId { id, map } = sid; + let info = if let Some(info) = pending.get(&id) { + info + } else { + println!("No pending info for ID {id}"); + continue; + }; + let data = if let Some(data) = map.get(DATA_KEY) { + data + } else { + println!("No Data was stored in Redis for ID {id}"); + continue; + }; + // Get data from map. + + let bytes = match data { + Value::Data(bytes) => bytes, + _ => { + println!("Redis data for ID {id} in wrong format"); + continue; } - Err(e) => error!("Redis xpending_count error {} for id {}", e, id), + }; + + if info.times_delivered > self.retries { + error!("Message has reached maximum retries {} for id", id); + ack_list.push(id.clone()); + continue; } + retained_ids.push(RecvData::new_retry( + id, + bytes.to_vec(), + info.times_delivered, + )); } Ok(retained_ids) @@ -201,6 +200,7 @@ impl Messenger for RedisMessenger { .get("message_wait_timeout") .and_then(|r| r.clone().to_u128().map(|n| n as usize)) .unwrap_or(MESSAGE_WAIT_TIMEOUT); + let consumer_group_name = config .get("consumer_group_name") .and_then(|r| r.clone().into_string()) @@ -208,11 +208,7 @@ impl Messenger for RedisMessenger { let pipeline_size = config .get("pipeline_size_bytes") - .and_then(|r| { - r.clone() - .to_u128() - .map(|n| n as usize) - }) + .and_then(|r| r.clone().to_u128().map(|n| n as usize)) .unwrap_or(PIPELINE_SIZE_BYTES); let pipeline_max_time = config @@ -302,10 +298,11 @@ impl Messenger for RedisMessenger { && stream.local_buffer_last_flush.elapsed() <= Duration::from_millis(self.pipeline_max_time as u64) { - debug!( - "Redis local buffer bytes {} and message pipeline size {} ", + info!( + "Redis local buffer bytes {} and message pipeline size {} elapsed time {}ms", stream.local_buffer_total, - stream.local_buffer.len() + stream.local_buffer.len(), + stream.local_buffer_last_flush.elapsed().as_millis() ); return Ok(()); } else { @@ -329,47 +326,62 @@ impl Messenger for RedisMessenger { Ok(()) } - async fn recv(&mut self, stream_key: &'static str) -> Result, MessengerError> { - let xauto_reply = self.xautoclaim(stream_key).await?; - let mut pending_messages = xauto_reply; - let opts = StreamReadOptions::default() - .block(self.message_wait_timeout) - .count(self.batch_size) - .group(self.consumer_group_name.as_str(), self.consumer_id.as_str()); - - // Read on stream key and save the reply. Log but do not return errors. - let reply: StreamReadReply = self - .connection - .xread_options(&[stream_key], &[">"], &opts) - .await - .map_err(|e| { - error!("Redis receive error: {e}"); - MessengerError::ReceiveError { msg: e.to_string() } - })?; - - let mut data_vec = Vec::new(); - data_vec.append(&mut pending_messages); - // Parse data in stream read reply and store in Vec to return to caller. - for StreamKey { key: _, ids } in reply.keys.into_iter() { - for StreamId { id, map } in ids { - // Get data from map. - let data = if let Some(data) = map.get(DATA_KEY) { - data - } else { - println!("No Data was stored in Redis for ID {id}"); - continue; - }; - let bytes = match data { - Value::Data(bytes) => bytes, - _ => { - println!("Redis data for ID {id} in wrong format"); + async fn recv( + &mut self, + stream_key: &'static str, + consumption_type: ConsumptionType, + ) -> Result, MessengerError> { + let mut data_vec = Vec::with_capacity(self.batch_size * 2); + if consumption_type == ConsumptionType::New || consumption_type == ConsumptionType::All { + let opts = StreamReadOptions::default() + //.block(self.message_wait_timeout) + .count(self.batch_size) + .group(self.consumer_group_name.as_str(), self.consumer_id.as_str()); + + // Read on stream key and save the reply. Log but do not return errors. + let reply: StreamReadReply = self + .connection + .xread_options(&[stream_key], &[">"], &opts) + .await + .map_err(|e| { + error!("Redis receive error: {e}"); + MessengerError::ReceiveError { msg: e.to_string() } + })?; + // Parse data in stream read reply and store in Vec to return to caller. + for StreamKey { key: _, ids } in reply.keys.into_iter() { + for StreamId { id, map } in ids { + // Get data from map. + let data = if let Some(data) = map.get(DATA_KEY) { + data + } else { + println!("No Data was stored in Redis for ID {id}"); continue; - } - }; + }; + let bytes = match data { + Value::Data(bytes) => bytes, + _ => { + println!("Redis data for ID {id} in wrong format"); + continue; + } + }; - data_vec.push(RecvData::new(id.clone(), bytes.to_vec())); + data_vec.push(RecvData::new(id, bytes.to_vec())); + } } } + if consumption_type == ConsumptionType::Redeliver || consumption_type == ConsumptionType::All{ + let xauto_reply = self.xautoclaim(stream_key).await; + match xauto_reply { + Ok(reply) => { + let mut pending_messages = reply; + data_vec.append(&mut pending_messages); + } + Err(e) => { + error!("XPENDING ERROR {e}"); + } + } + } + Ok(data_vec) } diff --git a/plerkle_serialization/Cargo.toml b/plerkle_serialization/Cargo.toml index 97ec5032..56b085eb 100644 --- a/plerkle_serialization/Cargo.toml +++ b/plerkle_serialization/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "plerkle_serialization" description = "Metaplex Flatbuffers Plerkle Serialization for Geyser plugin producer/consumer patterns." -version = "1.3.11" +version = "1.4.0" authors = ["Metaplex Developers "] repository = "https://github.com/metaplex-foundation/digital-asset-validator-plugin" license = "AGPL-3.0" @@ -12,7 +12,7 @@ readme = "Readme.md" flatbuffers = "22.10.26" chrono = "0.4.22" serde = { version = "1.0.149"} -solana-sdk = { version = ">=1.13.6" } -solana-transaction-status = { version = ">=1.13.6" } +solana-sdk = { version = "1.14.14" } +solana-transaction-status = { version = "1.14.14" } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/plerkle_serialization/src/lib.rs b/plerkle_serialization/src/lib.rs index ad1678c0..e2a6d553 100644 --- a/plerkle_serialization/src/lib.rs +++ b/plerkle_serialization/src/lib.rs @@ -19,8 +19,6 @@ pub use compiled_instruction_generated::*; pub use slot_status_info_generated::*; pub use transaction_info_generated::*; - - // ---- SHIMS #[allow(unused_imports)] -pub mod solana_geyser_plugin_interface_shims; \ No newline at end of file +pub mod solana_geyser_plugin_interface_shims; diff --git a/plerkle_serialization/src/serializer/serializer_stable.rs b/plerkle_serialization/src/serializer/serializer_stable.rs index 3cc7cdd1..dd5d6fa6 100644 --- a/plerkle_serialization/src/serializer/serializer_stable.rs +++ b/plerkle_serialization/src/serializer/serializer_stable.rs @@ -1,14 +1,14 @@ +use crate::solana_geyser_plugin_interface_shims::{ + ReplicaAccountInfoV2, ReplicaBlockInfoV2, ReplicaTransactionInfoV2, SlotStatus, +}; use crate::{ AccountInfo, AccountInfoArgs, BlockInfo, BlockInfoArgs, CompiledInstruction, CompiledInstructionArgs, InnerInstructions, InnerInstructionsArgs, Pubkey as FBPubkey, Pubkey, - SlotStatusInfo, SlotStatusInfoArgs, - Status as FBSlotStatus, TransactionInfo, TransactionInfoArgs, + SlotStatusInfo, SlotStatusInfoArgs, Status as FBSlotStatus, TransactionInfo, + TransactionInfoArgs, }; use chrono::Utc; use flatbuffers::FlatBufferBuilder; -use crate::solana_geyser_plugin_interface_shims::{ - ReplicaAccountInfoV2, ReplicaBlockInfoV2, ReplicaTransactionInfoV2, SlotStatus, -}; pub fn serialize_account<'a>( mut builder: FlatBufferBuilder<'a>, account: &ReplicaAccountInfoV2, @@ -199,7 +199,7 @@ pub fn serialize_block<'a>( let blockhash = Some(builder.create_string(block_info.blockhash)); // Serialize rewards. - let rewards =None; + let rewards = None; // Serialize everything into Block Info table. let seen_at = Utc::now(); diff --git a/plerkle_serialization/src/solana_geyser_plugin_interface_shims.rs b/plerkle_serialization/src/solana_geyser_plugin_interface_shims.rs index 9a1ad7f6..5385a56e 100644 --- a/plerkle_serialization/src/solana_geyser_plugin_interface_shims.rs +++ b/plerkle_serialization/src/solana_geyser_plugin_interface_shims.rs @@ -1,4 +1,3 @@ - use solana_sdk::{clock::UnixTimestamp, signature::Signature, transaction::SanitizedTransaction}; use solana_transaction_status::TransactionStatusMeta; #[derive(Debug, Clone, PartialEq, Eq)] @@ -34,7 +33,6 @@ pub struct ReplicaAccountInfoV2<'a> { pub txn_signature: Option<&'a Signature>, } - #[derive(Clone, Debug)] pub struct ReplicaTransactionInfoV2<'a> { /// The first signature of the transaction, used for identifying the transaction. @@ -86,4 +84,4 @@ impl SlotStatus { SlotStatus::Rooted => "rooted", } } -} \ No newline at end of file +} diff --git a/plerkle_snapshot/Cargo.toml b/plerkle_snapshot/Cargo.toml new file mode 100644 index 00000000..e6617676 --- /dev/null +++ b/plerkle_snapshot/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "plerkle_snapshot" +version = "1.4.0" +edition = "2021" +description = "Cli tool to load a snapshot into a wile plerkle" + +[dependencies] +reqwest = "0.11.4" +tokio = { version = "1.25.0", features = ["full"] } +solana-snapshot-etl = { git = "https://github.com/austbot/solana-snapshot-etl", features=["parallel","standalone"] } +solana-runtime="1.14.14" +figment = { version = "0.10.6", features = ["env"] } +indicatif = "0.17.3" +thiserror = "1.0.29" +tracing = "0.1.37" +tracing-subscriber = "0.3.16" +serde = { version = "1.0.152", features = ["derive"] } +solana-geyser-plugin-interface = "1.14.14" +libloading = "0.7.4" +serde_json = "1.0.93" +json5 = "0.4.1" \ No newline at end of file diff --git a/plerkle_snapshot/README.md b/plerkle_snapshot/README.md new file mode 100644 index 00000000..552c7bb2 --- /dev/null +++ b/plerkle_snapshot/README.md @@ -0,0 +1,5 @@ +This bin contains alot of code lifted from https://github.com/terorie/solana-snapshot-etl/ + +terorie is an inspiration to the masses and I hope to be as good of a dev as this anon someday. + +IGNORE ME FOR NOW as we need to fix something in the underlying library from terrorie to have slot \ No newline at end of file diff --git a/plerkle_snapshot/p.json b/plerkle_snapshot/p.json new file mode 100644 index 00000000..cd3515c7 --- /dev/null +++ b/plerkle_snapshot/p.json @@ -0,0 +1,14 @@ +{ + "libpath": "/media/austbot/development1/digital-asset-validator-plugin/target/release/libplerkle.so", + "accounts_selector": { + "owners": ["metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s","TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA","BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY"] + }, + "transaction_selector" : { + "mentions": ["BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY"] + }, + "env": "mainnet", + "enable_metrics": true, + "metrics_uri": "127.0.0.1", + "metrics_port": 8125, + "handle_startup": false +} diff --git a/plerkle_snapshot/src/app_tracing.rs b/plerkle_snapshot/src/app_tracing.rs new file mode 100644 index 00000000..18e7acf3 --- /dev/null +++ b/plerkle_snapshot/src/app_tracing.rs @@ -0,0 +1,11 @@ +use crate::config::Config; +use tracing::Level; +use tracing_subscriber::FmtSubscriber; + +pub fn enable_tracing(config: Config) { + let subscriber = FmtSubscriber::builder() + .with_max_level(config.level.unwrap_or(Level::INFO)) + .finish(); + + tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); +} diff --git a/plerkle_snapshot/src/config.rs b/plerkle_snapshot/src/config.rs new file mode 100644 index 00000000..b0258067 --- /dev/null +++ b/plerkle_snapshot/src/config.rs @@ -0,0 +1,29 @@ +use std::str; + +use figment::{Figment, providers::Env}; +use tracing::Level; + +use crate::error::SnappError; +use serde::{Deserialize}; + +#[derive(Default, Debug, Deserialize)] +pub struct Config { + pub snapshot_url: String, + pub snapshot_storage_path: String, + pub plugin_path: String, + #[serde(skip)] + pub level: Option, +} + +pub fn extract_config() -> Result { + Figment::new() + .join(Env::prefixed("SNAP_")) + .extract::() + .map(|mut op|{ + op.level = Some(Level::INFO); + op + }) + .map_err(|config_error| SnappError::ConfigFileReadError { + msg: config_error.to_string(), + }) +} \ No newline at end of file diff --git a/plerkle_snapshot/src/error.rs b/plerkle_snapshot/src/error.rs new file mode 100644 index 00000000..2f969046 --- /dev/null +++ b/plerkle_snapshot/src/error.rs @@ -0,0 +1,32 @@ +use solana_snapshot_etl::SnapshotError; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum SnappError { + #[error("Error reading config file: ({msg})")] + ConfigFileReadError { msg: String }, + #[error("Error Snapshot Download Error: ({msg})")] + SnapshotDownloadError { msg: String }, + #[error("General Error: ({msg})")] + GeneralError { msg: String }, + #[error("Plugin Load Error: ({msg})")] + PluginLoadError { msg: String }, +} + +impl From for SnappError { + fn from(error: SnapshotError) -> Self { + match error { + SnapshotError::IOError(e) => SnappError::GeneralError { msg: e.to_string() }, + SnapshotError::BincodeError(e) => SnappError::GeneralError { msg: e.to_string() }, + SnapshotError::NoStatusCache => SnappError::GeneralError { + msg: "No Status Cache".to_string(), + }, + SnapshotError::NoSnapshotManifest => SnappError::GeneralError { + msg: "No Manifest".to_string(), + }, + SnapshotError::UnexpectedAppendVec => SnappError::GeneralError { + msg: "Unexpected Append Vec".to_string(), + }, + } + } +} diff --git a/plerkle_snapshot/src/main.rs b/plerkle_snapshot/src/main.rs new file mode 100644 index 00000000..39111a93 --- /dev/null +++ b/plerkle_snapshot/src/main.rs @@ -0,0 +1,56 @@ +use solana_snapshot_etl::{ + archived::ArchiveSnapshotExtractor, parallel::AppendVecConsumer, SnapshotExtractor, +}; +use tracing::info; + +use crate::plugin::{load_plugin, GeyserDumper}; + +mod app_tracing; +pub mod config; +pub mod error; +pub mod plugin; + +#[tokio::main] +async fn main() { + match run().await { + Ok(_) => { + println!("Done") + } + Err(e) => { + println!("Error: {}", e); + } + } +} + +async fn run() -> Result<(), error::SnappError> { + let c = config::extract_config()?; + app_tracing::enable_tracing(c); + let resp = reqwest::blocking::get("https:://api.mainnet-beta.solana.com/snapshot.tar.bz2") + .map_err(|p| { + error::SnappError::SnapshotDownloadError { + msg: p.to_string(), + } + })?; + let mut loader = ArchiveSnapshotExtractor::from_reader(resp)?; + info!("Streaming snapshot from HTTP"); + let plugin = unsafe { + load_plugin(&"/media/austbot/development1/digital-asset-validator-plugin/target/release/libplerkle.so") + .map_err(|p| { + error::SnappError::PluginLoadError { + msg: "Failed to load plugin".to_string(), + } + })? + }; + assert!( + plugin.account_data_notifications_enabled(), + "Geyser plugin does not accept account data notifications" + ); + let mut dumper = GeyserDumper::new(plugin); + for append_vec in loader.iter() { + let av = append_vec?; + dumper.on_append_vec(av).unwrap(); + } + drop(dumper); + println!("Done!"); + Ok(()) +} diff --git a/plerkle_snapshot/src/plugin.rs b/plerkle_snapshot/src/plugin.rs new file mode 100644 index 00000000..01ec2583 --- /dev/null +++ b/plerkle_snapshot/src/plugin.rs @@ -0,0 +1,138 @@ + +// Copyright 2022 Solana Foundation. +// Copyright 2022 Richard Patel (terorie). +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Lifted from https://github.com/terorie/solana-snapshot-etl +use indicatif::{ProgressBar, ProgressStyle}; +use solana_geyser_plugin_interface::geyser_plugin_interface::{ + GeyserPlugin, ReplicaAccountInfoV2, ReplicaAccountInfoVersions, +}; +use solana_snapshot_etl::append_vec::{AppendVec, StoredAccountMeta}; +use solana_snapshot_etl::append_vec_iter; +use solana_snapshot_etl::parallel::{AppendVecConsumer, GenericResult}; +use std::error::Error; +use std::rc::Rc; + +use libloading::{Library, Symbol}; +use std::path::{Path, PathBuf}; + +/// # Safety +/// +/// This function loads the dynamically linked library specified in the config file. +/// +/// Causes memory corruption/UB on mismatching rustc or Solana versions, or if you look at the wrong way. +pub unsafe fn load_plugin( + config_file: &str, +) -> Result, Box> { + let config_path = PathBuf::from(config_file); + + let config_content = std::fs::read_to_string(config_file)?; + let config: serde_json::Value = json5::from_str(&config_content)?; + + let libpath = config["libpath"] + .as_str() + .ok_or("Missing libpath param in Geyser config")?; + let mut libpath = PathBuf::from(libpath); + if libpath.is_relative() { + let config_dir = config_path + .parent() + .expect("failed to resolve parent of Geyser config file"); + libpath = config_dir.join(libpath); + } + + load_plugin_inner(&libpath, &config_path.to_string_lossy()) +} + +unsafe fn load_plugin_inner( + libpath: &Path, + config_file: &str, +) -> Result, Box> { + type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin; + // Load library and leak, as we never want to unload it. + let lib = Box::leak(Box::new(Library::new(libpath)?)); + let constructor: Symbol = lib.get(b"_create_plugin")?; + // Unsafe call down to library. + let plugin_raw = constructor(); + let mut plugin = Box::from_raw(plugin_raw); + plugin.on_load(config_file)?; + Ok(plugin) +} + +pub(crate) struct GeyserDumper { + accounts_spinner: ProgressBar, + plugin: Box, + accounts_count: u64, +} + +impl AppendVecConsumer for GeyserDumper { + fn on_append_vec(&mut self, append_vec: AppendVec) -> GenericResult<()> { + for account in append_vec_iter(Rc::new(append_vec)) { + let account = account.access().unwrap(); + self.dump_account(account)?; + } + Ok(()) + } +} + +impl GeyserDumper { + pub(crate) fn new(plugin: Box) -> Self { + // TODO dedup spinner definitions + let spinner_style = ProgressStyle::with_template( + "{prefix:>10.bold.dim} {spinner} rate={per_sec}/s total={human_pos}", + ) + .unwrap(); + let accounts_spinner = ProgressBar::new_spinner() + .with_style(spinner_style) + .with_prefix("accs"); + + Self { + accounts_spinner, + plugin, + accounts_count: 0, + } + } + + pub(crate) fn dump_account( + &mut self, + account: StoredAccountMeta, + ) -> Result<(), Box> { + let slot = 0u64; // TODO fix slot number + self.plugin.update_account( + ReplicaAccountInfoVersions::V0_0_2(&ReplicaAccountInfoV2 { + pubkey: account.meta.pubkey.as_ref(), + lamports: account.account_meta.lamports, + owner: account.account_meta.owner.as_ref(), + executable: account.account_meta.executable, + rent_epoch: account.account_meta.rent_epoch, + data: account.data, + write_version: account.meta.write_version, + txn_signature: None, + }), + slot, + /* is_startup */ false, + )?; + self.accounts_count += 1; + if self.accounts_count % 1024 == 0 { + self.accounts_spinner.set_position(self.accounts_count); + } + Ok(()) + } +} + +impl Drop for GeyserDumper { + fn drop(&mut self) { + self.accounts_spinner.finish(); + } +} \ No newline at end of file diff --git a/plerkle_snapshot/src/process.rs b/plerkle_snapshot/src/process.rs new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/plerkle_snapshot/src/process.rs @@ -0,0 +1,2 @@ + + diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 35cd8762..7cbcbb0e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.59.0" \ No newline at end of file +channel = "1.64.0" \ No newline at end of file