From f5aa1a8908f38c1bc75dee79ae6b3af130cfa3d7 Mon Sep 17 00:00:00 2001 From: Alexey Pashinov Date: Wed, 23 Aug 2023 15:24:56 +0200 Subject: [PATCH 1/3] Add generated proto files to avoid using protoc anywhere --- .github/workflows/docs.yml | 7 - .github/workflows/master.yml | 14 -- Cargo.toml | 1 + gen-protoc/Cargo.toml | 14 ++ gen-protoc/src/main.rs | 26 +++ nekoton-proto/Cargo.toml | 3 - nekoton-proto/build.rs | 11 - nekoton-proto/src/lib.rs | 5 +- nekoton-proto/src/models.rs | 6 +- nekoton-proto/src/protos/mod.rs | 3 + nekoton-proto/src/{ => protos}/rpc.proto | 0 nekoton-proto/src/protos/rpc.rs | 262 +++++++++++++++++++++++ src/transport/proto/mod.rs | 2 +- 13 files changed, 311 insertions(+), 43 deletions(-) create mode 100644 gen-protoc/Cargo.toml create mode 100644 gen-protoc/src/main.rs delete mode 100644 nekoton-proto/build.rs create mode 100644 nekoton-proto/src/protos/mod.rs rename nekoton-proto/src/{ => protos}/rpc.proto (100%) create mode 100644 nekoton-proto/src/protos/rpc.rs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 35e98f975..da38de0d4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,13 +20,6 @@ jobs: toolchain: stable override: true - - name: Install protobuf - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protoc-24.0-linux-x86_64.zip - unzip protoc-24.0-linux-x86_64.zip - sudo cp bin/protoc /usr/bin/protoc - sudo cp -r include/. /usr/include - - name: Build docs uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 43dd98471..a5182f53a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -36,13 +36,6 @@ jobs: toolchain: stable override: true - - name: Install protobuf - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protoc-24.0-linux-x86_64.zip - unzip protoc-24.0-linux-x86_64.zip - sudo cp bin/protoc /usr/bin/protoc - sudo cp -r include/. /usr/include - - name: Run cargo test for nekoton uses: actions-rs/cargo@v1 with: @@ -88,13 +81,6 @@ jobs: override: true components: rustfmt, clippy - - name: Install protobuf - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protoc-24.0-linux-x86_64.zip - unzip protoc-24.0-linux-x86_64.zip - sudo cp bin/protoc /usr/bin/protoc - sudo cp -r include/. /usr/include - - name: Run cargo fmt uses: actions-rs/cargo@v1 with: diff --git a/Cargo.toml b/Cargo.toml index 634c1c4ca..6cd970b9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [workspace] members = [ + "gen-protoc", "nekoton-abi", "nekoton-contracts", "nekoton-derive", diff --git a/gen-protoc/Cargo.toml b/gen-protoc/Cargo.toml new file mode 100644 index 000000000..a350856b7 --- /dev/null +++ b/gen-protoc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "gen-protos" +description = "Generate Protocol Buffers definitions for nekoton-proto crate" +version = "0.13.0" +authors = [ + "Alexey Pashinov ", + "Vladimir Petrzhikovskiy ", + "Ivan Kalinin " +] +rust-version = "1.62.0" +edition = "2021" + +[dependencies] +prost-build = "0.11" diff --git a/gen-protoc/src/main.rs b/gen-protoc/src/main.rs new file mode 100644 index 000000000..9342b3316 --- /dev/null +++ b/gen-protoc/src/main.rs @@ -0,0 +1,26 @@ +use std::io::Result; +use std::path::Path; + +fn main() -> Result<()> { + let input = ["rpc.proto"]; + + let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let protos_dir = root.join("nekoton-proto").join("src").join("protos"); + + prost_build::Config::new() + .out_dir(&protos_dir) + .include_file("mod.rs") + // For old protoc versions. 3.12.4 needs this, but 3.21.12 doesn't. + .protoc_arg("--experimental_allow_proto3_optional") + // Replace Vec to Bytes + .bytes(["."]) + // Add EQ macro + .type_attribute("rpc.Response.GetTimings", "#[derive(Eq)]") + .compile_protos( + &input + .into_iter() + .map(|x| protos_dir.join(x)) + .collect::>(), + &[protos_dir], + ) +} diff --git a/nekoton-proto/Cargo.toml b/nekoton-proto/Cargo.toml index 4358587b3..85248dfb5 100644 --- a/nekoton-proto/Cargo.toml +++ b/nekoton-proto/Cargo.toml @@ -11,6 +11,3 @@ ton_block = { git = "https://github.com/broxus/ton-labs-block.git" } ton_types = { git = "https://github.com/broxus/ton-labs-types" } nekoton-abi = { path = "../nekoton-abi"} - -[build-dependencies] -prost-build = "0.11" diff --git a/nekoton-proto/build.rs b/nekoton-proto/build.rs deleted file mode 100644 index de20bb12f..000000000 --- a/nekoton-proto/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -fn main() -> std::io::Result<()> { - let mut prost_build = prost_build::Config::new(); - - // Replace Vec to Bytes - prost_build.bytes(["."]); - - // Add macro - prost_build.type_attribute("rpc.Response.GetTimings", "#[derive(Eq)]"); - - prost_build.compile_protos(&["src/rpc.proto"], &["src/"]) -} diff --git a/nekoton-proto/src/lib.rs b/nekoton-proto/src/lib.rs index cf08f7d21..dfdd6820a 100644 --- a/nekoton-proto/src/lib.rs +++ b/nekoton-proto/src/lib.rs @@ -1,8 +1,5 @@ -pub mod rpc { - include!(concat!(env!("OUT_DIR"), "/rpc.rs")); -} - pub mod models; +pub mod protos; pub mod utils; pub use prost; diff --git a/nekoton-proto/src/models.rs b/nekoton-proto/src/models.rs index 7946c128f..562d72f31 100644 --- a/nekoton-proto/src/models.rs +++ b/nekoton-proto/src/models.rs @@ -1,9 +1,9 @@ use prost::bytes::Bytes; use ton_types::UInt256; -use crate::rpc::response::get_contract_state::exists::{Exact, Inexact, LastTransactionId}; -use crate::rpc::response::get_contract_state::not_exist::GenTimings; -use crate::rpc::response::get_contract_state::{NotExist, Timings}; +use crate::protos::rpc::response::get_contract_state::exists::{Exact, Inexact, LastTransactionId}; +use crate::protos::rpc::response::get_contract_state::not_exist::GenTimings; +use crate::protos::rpc::response::get_contract_state::{NotExist, Timings}; impl From for nekoton_abi::GenTimings { fn from(t: GenTimings) -> Self { diff --git a/nekoton-proto/src/protos/mod.rs b/nekoton-proto/src/protos/mod.rs new file mode 100644 index 000000000..51bbd2b4b --- /dev/null +++ b/nekoton-proto/src/protos/mod.rs @@ -0,0 +1,3 @@ +pub mod rpc { + include!("rpc.rs"); +} diff --git a/nekoton-proto/src/rpc.proto b/nekoton-proto/src/protos/rpc.proto similarity index 100% rename from nekoton-proto/src/rpc.proto rename to nekoton-proto/src/protos/rpc.proto diff --git a/nekoton-proto/src/protos/rpc.rs b/nekoton-proto/src/protos/rpc.rs new file mode 100644 index 000000000..1851b1e0d --- /dev/null +++ b/nekoton-proto/src/protos/rpc.rs @@ -0,0 +1,262 @@ +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Request { + #[prost(oneof = "request::Call", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11")] + pub call: ::core::option::Option, +} +/// Nested message and enum types in `Request`. +pub mod request { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetContractState { + #[prost(bytes = "bytes", tag = "1")] + pub address: ::prost::bytes::Bytes, + #[prost(uint64, optional, tag = "2")] + pub last_transaction_lt: ::core::option::Option, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetTransaction { + #[prost(bytes = "bytes", tag = "1")] + pub id: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetDstTransaction { + #[prost(bytes = "bytes", tag = "1")] + pub message_hash: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetTransactionsList { + #[prost(bytes = "bytes", tag = "1")] + pub account: ::prost::bytes::Bytes, + #[prost(uint64, optional, tag = "2")] + pub last_transaction_lt: ::core::option::Option, + #[prost(uint32, tag = "3")] + pub limit: u32, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetAccountsByCodeHash { + #[prost(bytes = "bytes", tag = "1")] + pub code_hash: ::prost::bytes::Bytes, + #[prost(bytes = "bytes", optional, tag = "2")] + pub continuation: ::core::option::Option<::prost::bytes::Bytes>, + #[prost(uint32, tag = "3")] + pub limit: u32, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct SendMessage { + #[prost(bytes = "bytes", tag = "1")] + pub message: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Call { + #[prost(message, tag = "1")] + GetCapabilities(()), + #[prost(message, tag = "2")] + GetLatestKeyBlock(()), + #[prost(message, tag = "3")] + GetBlockchainConfig(()), + #[prost(message, tag = "4")] + GetStatus(()), + #[prost(message, tag = "5")] + GetTimings(()), + #[prost(message, tag = "6")] + GetContractState(GetContractState), + #[prost(message, tag = "7")] + GetTransaction(GetTransaction), + #[prost(message, tag = "8")] + GetDstTransaction(GetDstTransaction), + #[prost(message, tag = "9")] + GetTransactionsList(GetTransactionsList), + #[prost(message, tag = "10")] + GetAccountsByCodeHash(GetAccountsByCodeHash), + #[prost(message, tag = "11")] + SendMessage(SendMessage), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Response { + #[prost(oneof = "response::Result", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] + pub result: ::core::option::Option, +} +/// Nested message and enum types in `Response`. +pub mod response { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetRawTransaction { + #[prost(bytes = "bytes", optional, tag = "1")] + pub transaction: ::core::option::Option<::prost::bytes::Bytes>, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetTransactionsList { + #[prost(bytes = "bytes", repeated, tag = "1")] + pub transactions: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, + } + #[derive(Eq)] + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetTimings { + #[prost(uint32, tag = "1")] + pub last_mc_block_seqno: u32, + #[prost(uint32, tag = "2")] + pub last_shard_client_mc_block_seqno: u32, + #[prost(uint32, tag = "3")] + pub last_mc_utime: u32, + #[prost(int64, tag = "4")] + pub mc_time_diff: i64, + #[prost(int64, tag = "5")] + pub shard_client_time_diff: i64, + #[prost(uint64, tag = "6")] + pub smallest_known_lt: u64, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetStatus { + #[prost(bool, tag = "1")] + pub ready: bool, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetCapabilities { + #[prost(string, repeated, tag = "1")] + pub capabilities: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetLatestKeyBlock { + #[prost(bytes = "bytes", tag = "1")] + pub block: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetBlockchainConfig { + #[prost(int32, tag = "1")] + pub global_id: i32, + #[prost(bytes = "bytes", tag = "2")] + pub config: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetAccountsByCodeHash { + #[prost(bytes = "bytes", repeated, tag = "1")] + pub account: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct GetContractState { + #[prost(oneof = "get_contract_state::State", tags = "1, 2, 3")] + pub state: ::core::option::Option, + } + /// Nested message and enum types in `GetContractState`. + pub mod get_contract_state { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Timings { + #[prost(uint64, tag = "1")] + pub gen_lt: u64, + #[prost(uint32, tag = "2")] + pub gen_utime: u32, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct NotExist { + #[prost(oneof = "not_exist::GenTimings", tags = "2, 3")] + pub gen_timings: ::core::option::Option, + } + /// Nested message and enum types in `NotExist`. + pub mod not_exist { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum GenTimings { + #[prost(message, tag = "2")] + Known(super::Timings), + #[prost(message, tag = "3")] + Unknown(()), + } + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Exists { + #[prost(bytes = "bytes", tag = "1")] + pub account: ::prost::bytes::Bytes, + #[prost(message, optional, tag = "2")] + pub gen_timings: ::core::option::Option, + #[prost(oneof = "exists::LastTransactionId", tags = "3, 4")] + pub last_transaction_id: ::core::option::Option, + } + /// Nested message and enum types in `Exists`. + pub mod exists { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Exact { + #[prost(uint64, tag = "1")] + pub lt: u64, + #[prost(bytes = "bytes", tag = "2")] + pub hash: ::prost::bytes::Bytes, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Inexact { + #[prost(uint64, tag = "1")] + pub latest_lt: u64, + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum LastTransactionId { + #[prost(message, tag = "3")] + Exact(Exact), + #[prost(message, tag = "4")] + Inexact(Inexact), + } + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum State { + #[prost(message, tag = "1")] + NotExists(NotExist), + #[prost(message, tag = "2")] + Exists(Exists), + #[prost(message, tag = "3")] + Unchanged(Timings), + } + } + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Result { + #[prost(message, tag = "1")] + GetRawTransaction(GetRawTransaction), + #[prost(message, tag = "2")] + GetTransactionsList(GetTransactionsList), + #[prost(message, tag = "3")] + GetTimings(GetTimings), + #[prost(message, tag = "4")] + GetStatus(GetStatus), + #[prost(message, tag = "5")] + GetCapabilities(GetCapabilities), + #[prost(message, tag = "6")] + GetLatestKeyBlock(GetLatestKeyBlock), + #[prost(message, tag = "7")] + GetBlockchainConfig(GetBlockchainConfig), + #[prost(message, tag = "8")] + GetAccounts(GetAccountsByCodeHash), + #[prost(message, tag = "9")] + GetContractState(GetContractState), + #[prost(message, tag = "10")] + SendMessage(()), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Error { + #[prost(int32, tag = "1")] + pub code: i32, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} diff --git a/src/transport/proto/mod.rs b/src/transport/proto/mod.rs index 9193dfc54..1bcc50e80 100644 --- a/src/transport/proto/mod.rs +++ b/src/transport/proto/mod.rs @@ -4,7 +4,7 @@ use anyhow::Result; use ton_block::{Block, Deserializable, MsgAddressInt, Serializable}; use nekoton_proto::prost::{bytes::Bytes, Message}; -use nekoton_proto::rpc; +use nekoton_proto::protos::rpc; use nekoton_proto::utils; use nekoton_utils::*; From efa7de36e4dce1125d71359bdda9e3ea3d38a2e4 Mon Sep 17 00:00:00 2001 From: Alexey Pashinov Date: Wed, 23 Aug 2023 15:49:38 +0200 Subject: [PATCH 2/3] Add checking protos to gh CI --- .github/workflows/master.yml | 23 +++++++++++++++++++++++ Cargo.toml | 2 +- README.md | 16 ++++++++++++++++ {gen-protoc => gen-protos}/Cargo.toml | 0 {gen-protoc => gen-protos}/src/main.rs | 0 5 files changed, 40 insertions(+), 1 deletion(-) rename {gen-protoc => gen-protos}/Cargo.toml (100%) rename {gen-protoc => gen-protos}/src/main.rs (100%) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a5182f53a..1e31771ff 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -22,6 +22,29 @@ jobs: with: command: check + check-protos: + name: Check protos + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install protobuf + run: sudo apt update && sudo apt-get -y install protobuf-compiler + + - name: Generate Rust code from .proto files + run: cargo run -p gen-protos + + - name: Check for uncommitted changes + run: git diff --exit-code + test: name: Test Suite runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index 6cd970b9b..63738f35e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ edition = "2021" [workspace] members = [ - "gen-protoc", + "gen-protos", "nekoton-abi", "nekoton-contracts", "nekoton-derive", diff --git a/README.md b/README.md index c6c1be01a..9e9074ad5 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,22 @@ cargo add nekoton - Rust 1.62+ - `wasm-pack` 0.9.1+ (to test build for wasm target) +### Modifying protobuffers + +Occasionally, you may need to change the `.proto` files that define request/response +data format. In this case, you will need to add a few steps to the above +workflow. + +- Install the `protoc` compiler. +- Run `cargo run -p gen-protos` regularly (or after every edit to a `.proto` + file). The `gen-protos` binary will use the `prost-build` library to compile the + `.proto` files into `.rs` files. +- If you are adding a new `.proto` file, you will need to edit the list of + these files in `gen-protos/src/main.rs`. + +The `.rs` files generated from `.proto` files are included in the repository, +and there is a Github CI check that will complain if they do not match. + ## Contributing We welcome contributions to the project! If you notice any issues or errors, feel free to open an issue or submit a pull request. diff --git a/gen-protoc/Cargo.toml b/gen-protos/Cargo.toml similarity index 100% rename from gen-protoc/Cargo.toml rename to gen-protos/Cargo.toml diff --git a/gen-protoc/src/main.rs b/gen-protos/src/main.rs similarity index 100% rename from gen-protoc/src/main.rs rename to gen-protos/src/main.rs From f7b75b6ca21aac3758b1c4492e61aa41cac5a47e Mon Sep 17 00:00:00 2001 From: Alexey Pashinov Date: Wed, 23 Aug 2023 15:56:14 +0200 Subject: [PATCH 3/3] Update for readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9e9074ad5..2e65619da 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ cargo add nekoton - Rust 1.62+ - `wasm-pack` 0.9.1+ (to test build for wasm target) +- protoc 3.12.4+ (to generate .rs files from .proto) ### Modifying protobuffers