From 54558e2c2a5bccd1d420a06a44c23a028b83693a Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Tue, 13 Sep 2022 09:16:38 -0700 Subject: [PATCH 1/6] add integrations test harness The integrations crate provides a module test harness, calling the substreams cli to run each of the modules and cache the results to be compared on the next run. Cached results are saved in the integrations/target/ folder and can be run locally. The cached results are not committed to the repository, instead they can be restored in the github ci workflow for comparison. NOTE: There is some beginning work on calling the substreams grpc blocks endpoint directly using the substreams playground example as a guide, however, I have not been able to call the endpoint successfully and reverting to using the substreams CLI in the meantime. Signed-off-by: Ryan Tate --- .github/workflows/ci.yaml | 6 +- Cargo.lock | 1034 ++++++++++++++++- Cargo.toml | 1 + Makefile | 11 +- erc20-market-cap/Makefile | 2 +- erc20-market-cap/src/pb.rs | 2 +- erc20-market-cap/substreams.yaml | 2 +- erc20-price/Cargo.toml | 2 +- erc20-price/build.rs | 4 +- erc721/build.rs | 4 +- integrations/.cargo/config.toml | 2 + integrations/Cargo.toml | 18 + integrations/Makefile | 3 + integrations/build.rs | 58 + integrations/src/errors.rs | 17 + integrations/src/lib.rs | 9 + integrations/src/module.rs | 88 ++ integrations/src/pb/messari.erc20_price.v1.rs | 54 + integrations/src/tests.rs | 81 ++ integrations/substreams.yaml | 21 + substreams-helper/build.rs | 1 + uniswap-v2/build.rs | 4 +- 22 files changed, 1395 insertions(+), 29 deletions(-) create mode 100644 integrations/.cargo/config.toml create mode 100644 integrations/Cargo.toml create mode 100644 integrations/Makefile create mode 100644 integrations/build.rs create mode 100644 integrations/src/errors.rs create mode 100644 integrations/src/lib.rs create mode 100644 integrations/src/module.rs create mode 100644 integrations/src/pb/messari.erc20_price.v1.rs create mode 100644 integrations/src/tests.rs create mode 100644 integrations/substreams.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 59c7132e..508358a0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: install buf uses: bufbuild/buf-setup-action@v1.7.0 - - name: install rust-toolchain + - name: rust-toolchain uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable @@ -40,9 +40,11 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - target/ + **/target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: make erc20-price spkg run: make -C erc20-price build && make erc20-price-spkg - name: build all run: make build-all + - name: Integrations + run: make integrations-linux diff --git a/Cargo.lock b/Cargo.lock index d8891303..c1f57b03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,12 +23,95 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -67,6 +150,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + [[package]] name = "byte-slice-cast" version = "1.2.1" @@ -127,6 +216,22 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "crunchy" version = "0.2.2" @@ -159,9 +264,18 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "erc20-price-substreams" -version = "0.0.1" +version = "0.1.0" dependencies = [ "anyhow", "ethabi", @@ -258,12 +372,90 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -285,6 +477,25 @@ dependencies = [ "wasi", ] +[[package]] +name = "h2" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -297,6 +508,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -309,6 +529,105 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -366,6 +685,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itertools" version = "0.10.4" @@ -381,6 +706,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.2" @@ -408,6 +742,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "memchr" version = "2.5.0" @@ -420,12 +760,48 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys", +] + [[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -456,12 +832,67 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "pad" version = "0.1.6" @@ -497,6 +928,12 @@ dependencies = [ "syn", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "petgraph" version = "0.6.2" @@ -508,32 +945,70 @@ dependencies = [ ] [[package]] -name = "ppv-lite86" -version = "0.2.16" +name = "pin-project" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] [[package]] -name = "prettyplease" -version = "0.1.19" +name = "pin-project-internal" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", + "quote", "syn", ] [[package]] -name = "primitive-types" -version = "0.11.1" +name = "pin-project-lite" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" -dependencies = [ - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "uint", +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "prettyplease" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49e86d2c26a24059894a3afa13fd17d063419b05dfb83f06d9c3566060c3f5a" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", ] [[package]] @@ -744,6 +1219,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rlp" version = "0.5.1" @@ -766,6 +1278,39 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + +[[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.144" @@ -797,6 +1342,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha3" version = "0.10.5" @@ -807,6 +1364,25 @@ dependencies = [ "keccak", ] +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -946,6 +1522,21 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "substreams-integrations" +version = "0.1.0" +dependencies = [ + "prost 0.11.0", + "prost-types 0.11.1", + "reqwest", + "substreams", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + [[package]] name = "substreams-macro" version = "0.0.20" @@ -987,6 +1578,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tap" version = "1.0.1" @@ -1036,6 +1633,96 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.9" @@ -1045,6 +1732,151 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11cd56bdb54ef93935a6a79dbd1d91f1ebd4c64150fd61654031fd6b8b775c91" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.0", + "prost-derive 0.11.0", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbcd2800e34e743b9ae795867d5f77b535d3a3be69fd731e39145719752df8c" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.11.1", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1063,30 +1895,148 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + [[package]] name = "unicode-ident" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wee_alloc" version = "0.4.5" @@ -1132,6 +2082,58 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wyz" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 553b1fe4..10f4d804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "erc20-price", "erc20-market-cap", "erc721", + "integrations", "substreams-helper", "uniswap-v2" ] diff --git a/Makefile b/Makefile index ac9f5ea3..810c32d0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,16 @@ build-all: $(MAKE) -C erc20-market-cap build $(MAKE) -C erc721 build $(MAKE) -C compound-v2 build + $(MAKE) -C integrations build .PHONY: erc20-price-spkg erc20-price-spkg: - cd erc20-price && make pack && mv erc20-price-substreams-v0.1.0.spkg ../target \ No newline at end of file + cd erc20-price && make pack && mv erc20-price-substreams-v0.1.0.spkg ../target + +.PHONY: integrations-darwin +integrations-darwin: + cargo test --package substreams-integrations --target aarch64-apple-darwin -- --nocapture + +.PHONY: integrations-linux +integrations-linux: + cargo test --package substreams-integrations --target x86_64-unknown-linux-gnu -- --nocapture diff --git a/erc20-market-cap/Makefile b/erc20-market-cap/Makefile index 4c7fc2b2..e9ccc172 100644 --- a/erc20-market-cap/Makefile +++ b/erc20-market-cap/Makefile @@ -10,4 +10,4 @@ build: .PHONY: example example: - substreams run -e api-dev.streamingfast.io:443 substreams.yaml map_market_cap -s 13000700 -t +1 + substreams run -e api-dev.streamingfast.io:443 substreams.yaml map_market_cap -s 13000000 -t +1 diff --git a/erc20-market-cap/src/pb.rs b/erc20-market-cap/src/pb.rs index 86b08905..f142c82a 100644 --- a/erc20-market-cap/src/pb.rs +++ b/erc20-market-cap/src/pb.rs @@ -1,2 +1,2 @@ -pub mod erc20_market_cap; pub mod erc20_price; +pub mod erc20_market_cap; \ No newline at end of file diff --git a/erc20-market-cap/substreams.yaml b/erc20-market-cap/substreams.yaml index 98fee54e..71437c6a 100644 --- a/erc20-market-cap/substreams.yaml +++ b/erc20-market-cap/substreams.yaml @@ -21,7 +21,7 @@ binaries: modules: - name: map_market_cap kind: map - initialBlock: 13000700 + initialBlock: 13000000 inputs: - map: erc20Price:map_price output: diff --git a/erc20-price/Cargo.toml b/erc20-price/Cargo.toml index 520427a8..a0e0b63e 100644 --- a/erc20-price/Cargo.toml +++ b/erc20-price/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "erc20-price-substreams" -version = "0.0.1" +version = "0.1.0" description = "Price all ERC20 tokens." edition = "2021" repository = "https://github.com/messari/substreams/erc20-price" diff --git a/erc20-price/build.rs b/erc20-price/build.rs index 68bc205f..fb321d31 100644 --- a/erc20-price/build.rs +++ b/erc20-price/build.rs @@ -2,8 +2,8 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { - // println!("cargo:rerun-if-changed=proto"); - // println!("cargo:rerun-if-changed=abi"); + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); codegen::generate(None)?; Ok(()) diff --git a/erc721/build.rs b/erc721/build.rs index 68bc205f..fb321d31 100644 --- a/erc721/build.rs +++ b/erc721/build.rs @@ -2,8 +2,8 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { - // println!("cargo:rerun-if-changed=proto"); - // println!("cargo:rerun-if-changed=abi"); + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); codegen::generate(None)?; Ok(()) diff --git a/integrations/.cargo/config.toml b/integrations/.cargo/config.toml new file mode 100644 index 00000000..ab370428 --- /dev/null +++ b/integrations/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "aarch64-apple-darwin" \ No newline at end of file diff --git a/integrations/Cargo.toml b/integrations/Cargo.toml new file mode 100644 index 00000000..841cdc30 --- /dev/null +++ b/integrations/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substreams-integrations" +version = "0.1.0" +edition = "2021" + +[dependencies] +prost = "0.11.0" +prost-types = "0.11.1" +substreams = "0.0.20" +thiserror = "1.0.35" +tokio = { version = "1", features = ["macros", "rt"] } +tokio-stream = "0.1.10" +tonic = "0.8.1" + +[build-dependencies] +reqwest = { version = "0.11", features = ["json", "blocking"] } +tokio = { version = "1", features = ["macros", "rt"] } +tonic-build = "0.8.0" diff --git a/integrations/Makefile b/integrations/Makefile new file mode 100644 index 00000000..2388f607 --- /dev/null +++ b/integrations/Makefile @@ -0,0 +1,3 @@ +.PHONY: build +build: + cargo build \ No newline at end of file diff --git a/integrations/build.rs b/integrations/build.rs new file mode 100644 index 00000000..c7eeabca --- /dev/null +++ b/integrations/build.rs @@ -0,0 +1,58 @@ +use std::fs; +use std::path::Path; + +fn main() { + download_and_compile_substream_v1(); +} + +fn download_and_compile_substream_v1() { + const SUBSTREAMS_GITHUB_REV: &str = "https://raw.githubusercontent.com/streamingfast/substreams/73ec39f882638be99960b59feab8e23f2ea76c50"; + const SUBSTREAMS_PROTO_SRC: &str = "/proto/sf/substreams/v1/substreams.proto"; + const CLOCK_PROTO_SRC: &str = "/proto/sf/substreams/v1/clock.proto"; + const MODULE_PROTO_SRC: &str = "/proto/sf/substreams/v1/modules.proto"; + + const DEFAULT_TARGET_DIR: &str = "./target/sf/substreams/v1"; + + let out_dir = Path::new(DEFAULT_TARGET_DIR); + + fs::create_dir_all(out_dir).ok(); + + let substreams_proto = out_dir.join("substreams.proto"); + let clock_proto = out_dir.join("clock.proto"); + let module_proto = out_dir.join("modules.proto"); + + let substreams_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, SUBSTREAMS_PROTO_SRC); + let clock_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, CLOCK_PROTO_SRC); + let module_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, MODULE_PROTO_SRC); + + let mut substreams_proto_file = fs::File::create(&substreams_proto).unwrap(); + let mut clock_proto_file = fs::File::create(&clock_proto).unwrap(); + let mut module_proto_file = fs::File::create(&module_proto).unwrap(); + + reqwest::blocking::get(&substreams_proto_url) + .unwrap() + .copy_to(&mut substreams_proto_file) + .unwrap(); + reqwest::blocking::get(&clock_proto_url) + .unwrap() + .copy_to(&mut clock_proto_file) + .unwrap(); + reqwest::blocking::get(&module_proto_url) + .unwrap() + .copy_to(&mut module_proto_file) + .unwrap(); + + tonic_build::configure() + .build_client(true) + .build_server(true) + .out_dir(out_dir) + .compile( + &[ + substreams_proto.as_path(), + clock_proto.as_path(), + module_proto.as_path(), + ], + &["./target"], + ) + .unwrap(); +} diff --git a/integrations/src/errors.rs b/integrations/src/errors.rs new file mode 100644 index 00000000..03186a55 --- /dev/null +++ b/integrations/src/errors.rs @@ -0,0 +1,17 @@ +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error("Missing module info: {0}")] + MissingModule(String), + + #[error("Substreams output is empty")] + EmptyOutput, + + #[error("Substreams output does not match cached output")] + OutputMismatch, + + #[error("Substreams error: {0}")] + Substream(String), +} diff --git a/integrations/src/lib.rs b/integrations/src/lib.rs new file mode 100644 index 00000000..b1ff6667 --- /dev/null +++ b/integrations/src/lib.rs @@ -0,0 +1,9 @@ +pub mod errors; +pub mod module; + +#[rustfmt::skip] +#[path = "../target/sf/substreams/v1/sf.substreams.v1.rs"] +pub mod substreams_client; + +#[cfg(test)] +mod tests; diff --git a/integrations/src/module.rs b/integrations/src/module.rs new file mode 100644 index 00000000..defc2d8c --- /dev/null +++ b/integrations/src/module.rs @@ -0,0 +1,88 @@ +use crate::errors::Error; + +const DEFAULT_START_BLOCK: u64 = 13e6 as u64; +const DEFAULT_NUM_BLOCKS: usize = 10; + +pub struct Module { + pub method: String, + pub path: String, + pub cache_file: String, + pub start_block: Option, + pub num_blocks: Option, +} + +impl Module { + pub fn new(method: &str, path: &str, cache_file: &str) -> Self { + Self { + method: method.to_string(), + path: path.to_string(), + cache_file: cache_file.to_string(), + start_block: None, + num_blocks: None, + } + } + + pub fn cached_output(&self) -> Result, Error> { + let cache = std::fs::read_to_string(&self.cache_file).ok(); + Ok(cache) + } + + pub fn check_cached_output(&self, output: String) -> Result<(), Error> { + let cache = self.cached_output()?; + + if output.is_empty() { + return Err(Error::EmptyOutput); + } else if cache.is_none() || cache.clone().unwrap().is_empty() { + self.cache_output(output)?; + } else if cache.unwrap() != output { + return Err(Error::OutputMismatch); + } + + Ok(()) + } + + pub fn cache_output(&self, output: String) -> Result<(), Error> { + if output.is_empty() { + return Err(Error::EmptyOutput); + } + + std::fs::write(&self.cache_file, output)?; + Ok(()) + } + + // Runs substream module using the `substreams` cli. + // NOTE: `substreams` cli must be installed to run this command. + pub fn run(&self) -> Result { + use std::process::Command; + + // generate pb files under src/pb + let output = Command::new("substreams") + .args(&[ + "run", + "-e", + "api-dev.streamingfast.io:443", + &self.path, + &self.method, + "-s", + self.start_block + .unwrap_or(DEFAULT_START_BLOCK) + .to_string() + .as_str(), + "-t", + format!("+{}", self.num_blocks.unwrap_or(DEFAULT_NUM_BLOCKS)).as_str(), + ]) + .output()?; + + if !output.stderr.is_empty() { + return Err(Error::Substream(String::from_utf8(output.stderr).unwrap())); + } + + let content = String::from_utf8_lossy(&output.stdout); + + Ok(content.to_string()) + } + + pub fn test(&self) -> Result<(), Error> { + self.check_cached_output(self.run()?) + } +} diff --git a/integrations/src/pb/messari.erc20_price.v1.rs b/integrations/src/pb/messari.erc20_price.v1.rs new file mode 100644 index 00000000..bb4e4ca0 --- /dev/null +++ b/integrations/src/pb/messari.erc20_price.v1.rs @@ -0,0 +1,54 @@ +// @generated +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Erc20Price { + #[prost(uint64, tag="1")] + pub block_number: u64, + #[prost(string, tag="2")] + pub price_usd: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="3")] + pub token_address: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Erc20Prices { + #[prost(message, repeated, tag="1")] + pub items: ::prost::alloc::vec::Vec, +} +/// Encoded file descriptor set for the `messari.erc20_price.v1` package +pub const FILE_DESCRIPTOR_SET: &[u8] = &[ + 0x0a, 0xa8, 0x04, 0x0a, 0x11, 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, + 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x71, + 0x0a, 0x0a, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, + 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x69, 0x63, 0x65, 0x5f, 0x75, 0x73, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x69, 0x63, 0x65, 0x55, 0x73, 0x64, 0x12, 0x23, 0x0a, 0x0d, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x22, 0x47, 0x0a, 0x0b, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, 0x69, 0x63, 0x65, 0x73, + 0x12, 0x38, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, 0x65, 0x72, 0x63, 0x32, 0x30, 0x5f, + 0x70, 0x72, 0x69, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x52, 0x43, 0x32, 0x30, 0x50, 0x72, + 0x69, 0x63, 0x65, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xb6, 0x02, 0x0a, 0x06, 0x12, + 0x04, 0x00, 0x00, 0x0c, 0x01, 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, + 0x08, 0x0a, 0x01, 0x02, 0x12, 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, + 0x04, 0x04, 0x00, 0x08, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, + 0x12, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x1a, 0x0a, 0x0c, + 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x00, 0x01, 0x12, 0x03, 0x05, 0x09, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, + 0x02, 0x00, 0x03, 0x12, 0x03, 0x05, 0x18, 0x19, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, + 0x12, 0x03, 0x06, 0x02, 0x17, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, + 0x06, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, + 0x12, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x15, 0x16, 0x0a, + 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x02, 0x12, 0x03, 0x07, 0x02, 0x1a, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x02, 0x05, 0x12, 0x03, 0x07, 0x02, 0x07, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, + 0x02, 0x02, 0x01, 0x12, 0x03, 0x07, 0x08, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02, + 0x03, 0x12, 0x03, 0x07, 0x18, 0x19, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, 0x12, 0x04, 0x0a, 0x00, + 0x0c, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x0a, 0x08, 0x13, 0x0a, 0x0b, + 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0b, 0x02, 0x20, 0x0a, 0x0c, 0x0a, 0x05, 0x04, + 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0b, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, + 0x00, 0x06, 0x12, 0x03, 0x0b, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x01, + 0x12, 0x03, 0x0b, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x03, 0x12, 0x03, + 0x0b, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +]; +// @@protoc_insertion_point(module) \ No newline at end of file diff --git a/integrations/src/tests.rs b/integrations/src/tests.rs new file mode 100644 index 00000000..6478d455 --- /dev/null +++ b/integrations/src/tests.rs @@ -0,0 +1,81 @@ +use crate::module::Module; + +#[test] +fn test_erc20_price_module_map_price() -> Result<(), Box> { + let module = Module::new( + "map_price", + "../erc20-price/substreams.yaml", + "./target/erc20-price-cache.txt", + ); + module.test()?; + + Ok(()) +} + +#[test] +fn test_erc20_market_cap_module_map_market_cap() -> Result<(), Box> { + let mut module = Module::new( + "map_market_cap", + "../erc20-market-cap/substreams.yaml", + "./target/erc20-market-cap-cache.txt", + ); + module.num_blocks = Some(1); + module.test()?; + + Ok(()) +} + +#[test] +fn test_erc721_module_block_to_transfers() -> Result<(), Box> { + let module = Module::new( + "block_to_transfers", + "../erc721/substreams.yaml", + "./target/erc721-cache.txt", + ); + module.test()?; + + Ok(()) +} + +#[test] +fn test_uniswap_v2_module_store_pools() -> Result<(), Box> { + let mut module = Module::new( + "map_pair_created_event,map_pools,store_pools", + "../uniswap-v2/substreams.yaml", + "./target/uniswap-v2-cache.txt", + ); + module.start_block = Some(10008355); + module.num_blocks = Some(200); + module.test()?; + + Ok(()) +} + +// NOTE: Receiving GOAWAY error with grpc clientl; +// May be due to improper API key; +// TODO: Contact SF to understand how to use the gRPC `blocks` +// endpoint. +#[ignore] +#[tokio::test] +async fn test_substreams_client() -> Result<(), Box> { + use crate::substreams_client::{stream_client::StreamClient, Request}; + use tokio_stream::StreamExt; + + const DEFAULT_SUBSTREAMS_ENDPOINT: &str = "https://api-dev.streamingfast.io:443"; + + let mut client = StreamClient::connect(DEFAULT_SUBSTREAMS_ENDPOINT).await?; + let mut request = Request::default(); + request.start_block_num = 13e5 as i64; + request.stop_block_num = (request.start_block_num + 10) as u64; + request.output_modules = vec!["sf.ethereum.v1".to_string()]; + let mut stream = client + .blocks(tonic::Request::new(request)) + .await? + .into_inner(); + + while let Some(output) = stream.next().await { + println!("{:?}", output?); + } + + Ok(()) +} diff --git a/integrations/substreams.yaml b/integrations/substreams.yaml new file mode 100644 index 00000000..f17ff784 --- /dev/null +++ b/integrations/substreams.yaml @@ -0,0 +1,21 @@ +specVersion: v0.1.0 +package: + name: "substreams_integrations" + version: v0.1.0 + +imports: + erc20Price: ./erc20-price-substreams-v0.1.0.spkg + +binaries: + default: + type: wasm/rust-v1 + file: ../target/wasm32-unknown-unknown/release/substreams_integration.wasm + +modules: + - name: map_market_cap + kind: map + initialBlock: 13000700 + inputs: + - map: erc20Price:map_price + output: + type: proto:messari.erc20_market_cap.v1.Erc20MarketCaps diff --git a/substreams-helper/build.rs b/substreams-helper/build.rs index 0f1a2864..127e2376 100644 --- a/substreams-helper/build.rs +++ b/substreams-helper/build.rs @@ -2,6 +2,7 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { + println!("cargo:rerun-if-changed=abi"); codegen::generate_abi(None)?; Ok(()) } diff --git a/uniswap-v2/build.rs b/uniswap-v2/build.rs index 68bc205f..fb321d31 100644 --- a/uniswap-v2/build.rs +++ b/uniswap-v2/build.rs @@ -2,8 +2,8 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { - // println!("cargo:rerun-if-changed=proto"); - // println!("cargo:rerun-if-changed=abi"); + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); codegen::generate(None)?; Ok(()) From bf76de307e26288ce97fbe54c5db6b2ccb37c873 Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Fri, 23 Sep 2022 16:28:55 -0700 Subject: [PATCH 2/6] fix(ci): remove integrations build step from build-all Signed-off-by: Ryan Tate --- Makefile | 1 - uniswap-v2/src/pb.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 810c32d0..bee91d85 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ build-all: $(MAKE) -C erc20-market-cap build $(MAKE) -C erc721 build $(MAKE) -C compound-v2 build - $(MAKE) -C integrations build .PHONY: erc20-price-spkg erc20-price-spkg: diff --git a/uniswap-v2/src/pb.rs b/uniswap-v2/src/pb.rs index db229ca6..dce92cb8 100644 --- a/uniswap-v2/src/pb.rs +++ b/uniswap-v2/src/pb.rs @@ -1,3 +1,3 @@ pub mod uniswap; -pub mod dex_amm; -pub mod common; \ No newline at end of file +pub mod common; +pub mod dex_amm; \ No newline at end of file From 356c3863358f38cc32a88de9ac6fdb18ab1c2f20 Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Fri, 23 Sep 2022 16:52:38 -0700 Subject: [PATCH 3/6] fix(ci): install protoc Signed-off-by: Ryan Tate --- .github/workflows/ci.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 508358a0..2dcd3a5c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,8 @@ jobs: build: name: Build runs-on: ubuntu-latest + env: + PROTOC: /usr/local/bin/protoc steps: - name: checkout repo uses: actions/checkout@v2 @@ -20,6 +22,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: install buf uses: bufbuild/buf-setup-action@v1.7.0 + - name: install protoc + env: + PROTOC_ZIP: protoc-3.14.0-linux-x86_64.zip + run: | + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP + sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc + sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' + rm -f $PROTOC_ZIP + sudo chmod +x /usr/local/bin/protoc - name: rust-toolchain uses: actions-rs/toolchain@v1.0.6 with: From fc1f0d4b6a8709837cd21588640602e5e6ccf07f Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Fri, 23 Sep 2022 17:21:51 -0700 Subject: [PATCH 4/6] fix(integrations): comment out unused substreams client build step Signed-off-by: Ryan Tate --- integrations/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/build.rs b/integrations/build.rs index c7eeabca..e0c7b94d 100644 --- a/integrations/build.rs +++ b/integrations/build.rs @@ -2,7 +2,7 @@ use std::fs; use std::path::Path; fn main() { - download_and_compile_substream_v1(); + // download_and_compile_substream_v1(); } fn download_and_compile_substream_v1() { From 153d30bdf79abd40d23644910c9a77780604a94f Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Fri, 23 Sep 2022 17:34:05 -0700 Subject: [PATCH 5/6] fix(ci): comment out substreams grpc client codegen and mod Signed-off-by: Ryan Tate --- integrations/build.rs | 118 +++++++++++++++++++------------------- integrations/src/lib.rs | 6 +- integrations/src/tests.rs | 40 ++++++------- 3 files changed, 83 insertions(+), 81 deletions(-) diff --git a/integrations/build.rs b/integrations/build.rs index e0c7b94d..b98fa435 100644 --- a/integrations/build.rs +++ b/integrations/build.rs @@ -1,58 +1,60 @@ -use std::fs; -use std::path::Path; - -fn main() { - // download_and_compile_substream_v1(); -} - -fn download_and_compile_substream_v1() { - const SUBSTREAMS_GITHUB_REV: &str = "https://raw.githubusercontent.com/streamingfast/substreams/73ec39f882638be99960b59feab8e23f2ea76c50"; - const SUBSTREAMS_PROTO_SRC: &str = "/proto/sf/substreams/v1/substreams.proto"; - const CLOCK_PROTO_SRC: &str = "/proto/sf/substreams/v1/clock.proto"; - const MODULE_PROTO_SRC: &str = "/proto/sf/substreams/v1/modules.proto"; - - const DEFAULT_TARGET_DIR: &str = "./target/sf/substreams/v1"; - - let out_dir = Path::new(DEFAULT_TARGET_DIR); - - fs::create_dir_all(out_dir).ok(); - - let substreams_proto = out_dir.join("substreams.proto"); - let clock_proto = out_dir.join("clock.proto"); - let module_proto = out_dir.join("modules.proto"); - - let substreams_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, SUBSTREAMS_PROTO_SRC); - let clock_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, CLOCK_PROTO_SRC); - let module_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, MODULE_PROTO_SRC); - - let mut substreams_proto_file = fs::File::create(&substreams_proto).unwrap(); - let mut clock_proto_file = fs::File::create(&clock_proto).unwrap(); - let mut module_proto_file = fs::File::create(&module_proto).unwrap(); - - reqwest::blocking::get(&substreams_proto_url) - .unwrap() - .copy_to(&mut substreams_proto_file) - .unwrap(); - reqwest::blocking::get(&clock_proto_url) - .unwrap() - .copy_to(&mut clock_proto_file) - .unwrap(); - reqwest::blocking::get(&module_proto_url) - .unwrap() - .copy_to(&mut module_proto_file) - .unwrap(); - - tonic_build::configure() - .build_client(true) - .build_server(true) - .out_dir(out_dir) - .compile( - &[ - substreams_proto.as_path(), - clock_proto.as_path(), - module_proto.as_path(), - ], - &["./target"], - ) - .unwrap(); -} +fn main() {} + +// NOTE: The follow method is currently not used, but is kept here +// for future reference in case we want to use the substreams grpc +// blocks endpoint directly, instead of the substreams cli. +// +// +// fn download_and_compile_substream_v1() { +// use std::fs; +// use std::path::Path; +// const SUBSTREAMS_GITHUB_REV: &str = "https://raw.githubusercontent.com/streamingfast/substreams/73ec39f882638be99960b59feab8e23f2ea76c50"; +// const SUBSTREAMS_PROTO_SRC: &str = "/proto/sf/substreams/v1/substreams.proto"; +// const CLOCK_PROTO_SRC: &str = "/proto/sf/substreams/v1/clock.proto"; +// const MODULE_PROTO_SRC: &str = "/proto/sf/substreams/v1/modules.proto"; + +// const DEFAULT_TARGET_DIR: &str = "./target/sf/substreams/v1"; + +// let out_dir = Path::new(DEFAULT_TARGET_DIR); + +// fs::create_dir_all(out_dir).ok(); + +// let substreams_proto = out_dir.join("substreams.proto"); +// let clock_proto = out_dir.join("clock.proto"); +// let module_proto = out_dir.join("modules.proto"); + +// let substreams_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, SUBSTREAMS_PROTO_SRC); +// let clock_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, CLOCK_PROTO_SRC); +// let module_proto_url = format!("{}/{}", SUBSTREAMS_GITHUB_REV, MODULE_PROTO_SRC); + +// let mut substreams_proto_file = fs::File::create(&substreams_proto).unwrap(); +// let mut clock_proto_file = fs::File::create(&clock_proto).unwrap(); +// let mut module_proto_file = fs::File::create(&module_proto).unwrap(); + +// reqwest::blocking::get(&substreams_proto_url) +// .unwrap() +// .copy_to(&mut substreams_proto_file) +// .unwrap(); +// reqwest::blocking::get(&clock_proto_url) +// .unwrap() +// .copy_to(&mut clock_proto_file) +// .unwrap(); +// reqwest::blocking::get(&module_proto_url) +// .unwrap() +// .copy_to(&mut module_proto_file) +// .unwrap(); + +// tonic_build::configure() +// .build_client(true) +// .build_server(true) +// .out_dir(out_dir) +// .compile( +// &[ +// substreams_proto.as_path(), +// clock_proto.as_path(), +// module_proto.as_path(), +// ], +// &["./target"], +// ) +// .unwrap(); +// } diff --git a/integrations/src/lib.rs b/integrations/src/lib.rs index b1ff6667..5ed57529 100644 --- a/integrations/src/lib.rs +++ b/integrations/src/lib.rs @@ -1,9 +1,9 @@ pub mod errors; pub mod module; -#[rustfmt::skip] -#[path = "../target/sf/substreams/v1/sf.substreams.v1.rs"] -pub mod substreams_client; +// #[rustfmt::skip] +// #[path = "../target/sf/substreams/v1/sf.substreams.v1.rs"] +// pub mod substreams_client; #[cfg(test)] mod tests; diff --git a/integrations/src/tests.rs b/integrations/src/tests.rs index 6478d455..bdee30c3 100644 --- a/integrations/src/tests.rs +++ b/integrations/src/tests.rs @@ -55,27 +55,27 @@ fn test_uniswap_v2_module_store_pools() -> Result<(), Box // May be due to improper API key; // TODO: Contact SF to understand how to use the gRPC `blocks` // endpoint. -#[ignore] -#[tokio::test] -async fn test_substreams_client() -> Result<(), Box> { - use crate::substreams_client::{stream_client::StreamClient, Request}; - use tokio_stream::StreamExt; +// #[ignore] +// #[tokio::test] +// async fn test_substreams_client() -> Result<(), Box> { +// use crate::substreams_client::{stream_client::StreamClient, Request}; +// use tokio_stream::StreamExt; - const DEFAULT_SUBSTREAMS_ENDPOINT: &str = "https://api-dev.streamingfast.io:443"; +// const DEFAULT_SUBSTREAMS_ENDPOINT: &str = "https://api-dev.streamingfast.io:443"; - let mut client = StreamClient::connect(DEFAULT_SUBSTREAMS_ENDPOINT).await?; - let mut request = Request::default(); - request.start_block_num = 13e5 as i64; - request.stop_block_num = (request.start_block_num + 10) as u64; - request.output_modules = vec!["sf.ethereum.v1".to_string()]; - let mut stream = client - .blocks(tonic::Request::new(request)) - .await? - .into_inner(); +// let mut client = StreamClient::connect(DEFAULT_SUBSTREAMS_ENDPOINT).await?; +// let mut request = Request::default(); +// request.start_block_num = 13e5 as i64; +// request.stop_block_num = (request.start_block_num + 10) as u64; +// request.output_modules = vec!["sf.ethereum.v1".to_string()]; +// let mut stream = client +// .blocks(tonic::Request::new(request)) +// .await? +// .into_inner(); - while let Some(output) = stream.next().await { - println!("{:?}", output?); - } +// while let Some(output) = stream.next().await { +// println!("{:?}", output?); +// } - Ok(()) -} +// Ok(()) +// } From 54b331c5121afbbcdaa9f694502f252db1dbc0fc Mon Sep 17 00:00:00 2001 From: Ryan Tate Date: Mon, 26 Sep 2022 20:51:04 -0700 Subject: [PATCH 6/6] ci: set substreams auth token, use protoc gh action Signed-off-by: Ryan Tate --- .github/workflows/ci.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2dcd3a5c..9eaa008c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,17 +20,16 @@ jobs: cache: enable env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: request streaming fast authentication token + run: | + FIREHOSE_API_TOKEN=$(curl https://auth.dfuse.io/v1/auth/issue -s --data-binary '{"api_key":"${{secrets.STREAMING_FAST}}"}' | jq -r .token) + echo "SUBSTREAMS_API_TOKEN=$FIREHOSE_API_TOKEN" >> $GITHUB_ENV - name: install buf uses: bufbuild/buf-setup-action@v1.7.0 - name: install protoc - env: - PROTOC_ZIP: protoc-3.14.0-linux-x86_64.zip - run: | - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP - sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc - sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' - rm -f $PROTOC_ZIP - sudo chmod +x /usr/local/bin/protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.14.0' - name: rust-toolchain uses: actions-rs/toolchain@v1.0.6 with: