From 3ee1010105c2604c47635558968d0c183882d068 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 16 Mar 2023 20:22:00 +0600 Subject: [PATCH 001/192] An empty DAC Validator pallet crate --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + frame/ddc-validator/Cargo.toml | 19 +++++++++++++++++++ frame/ddc-validator/README.md | 1 + frame/ddc-validator/src/lib.rs | 26 ++++++++++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 frame/ddc-validator/Cargo.toml create mode 100644 frame/ddc-validator/README.md create mode 100644 frame/ddc-validator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e485f29b0e835..58c2b20b679c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5844,6 +5844,16 @@ dependencies = [ "substrate-test-utils", ] +[[package]] +name = "pallet-ddc-validator" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", +] + [[package]] name = "pallet-democracy" version = "4.0.0-dev" diff --git a/Cargo.toml b/Cargo.toml index 1fb8f474d9a5f..5502576871a67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ members = [ "frame/ddc-metrics-offchain-worker", "frame/ddc-pallet", "frame/ddc-staking", + "frame/ddc-validator", "frame/executive", "frame/gilt", "frame/grandpa", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml new file mode 100644 index 0000000000000..d9525b4dfd9af --- /dev/null +++ b/frame/ddc-validator/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pallet-ddc-validator" +version = "0.1.0" +edition = "2021" + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", +] diff --git a/frame/ddc-validator/README.md b/frame/ddc-validator/README.md new file mode 100644 index 0000000000000..3a21d9d6cc2b5 --- /dev/null +++ b/frame/ddc-validator/README.md @@ -0,0 +1 @@ +# DDC Validator diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs new file mode 100644 index 0000000000000..0cecc036b50ce --- /dev/null +++ b/frame/ddc-validator/src/lib.rs @@ -0,0 +1,26 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +pub use frame_support::pallet_prelude::*; +pub use frame_system::pallet_prelude::*; +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + type Event: From> + IsType<::Event>; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event {} + + #[pallet::error] + pub enum Error {} +} From 9998f50ad9500df97dd64088ac2aab9b3013884f Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 15:31:57 +0600 Subject: [PATCH 002/192] Add DAC Validator pallet to the runtime --- Cargo.lock | 1 + bin/node/runtime/Cargo.toml | 2 ++ bin/node/runtime/src/lib.rs | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 58c2b20b679c2..bc9b451a20811 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5049,6 +5049,7 @@ dependencies = [ "pallet-contracts-rpc-runtime-api", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", + "pallet-ddc-validator", "pallet-democracy", "pallet-election-provider-multi-phase", "pallet-elections-phragmen", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 52830d110122e..ed563f445eb15 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -96,6 +96,7 @@ pallet-erc721 = { version = "2.0.0", default-features = false, path = "../../../ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../frame/erc20" } pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } +pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } @@ -170,6 +171,7 @@ std = [ "pallet-child-bounties/std", "pallet-ddc-metrics-offchain-worker/std", "pallet-ddc-staking/std", + "pallet-ddc-validator/std", ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 51a54091e7aee..d13d2595cf0bb 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -81,6 +81,7 @@ pub use pallet_cere_ddc; pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; +pub use pallet_ddc_validator; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1229,6 +1230,10 @@ impl pallet_ddc_staking::Config for Runtime { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +impl pallet_ddc_validator::Config for Runtime { + type Event = Event; +} + construct_runtime!( pub enum Runtime where Block = Block, @@ -1279,6 +1284,7 @@ construct_runtime!( Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, DdcStaking: pallet_ddc_staking, + DdcValidator: pallet_ddc_validator, } ); From d1ee63fd1024315b7bca9219b53f6942e65b8bc8 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 16:38:15 +0600 Subject: [PATCH 003/192] Era counter with privileged inc for DAC Validator --- frame/ddc-validator/src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0cecc036b50ce..a9b77496845ff 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -17,10 +17,29 @@ pub mod pallet { type Event: From> + IsType<::Event>; } + #[pallet::storage] + #[pallet::getter(fn global_era_counter)] + pub type GlobalEraCounter = StorageValue<_, u32>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event {} #[pallet::error] pub enum Error {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(100_000)] + pub fn inc_era(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + if let Some(era) = >::get() { + let new_era = era.checked_add(1).unwrap_or_default(); + >::put(new_era); + } else { + >::put(1); + } + Ok(()) + } + } } From 2a1b9468208f88b4aebe6c3a74c83fafd4d5d046 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 17:15:04 +0600 Subject: [PATCH 004/192] Hooks and storage items for DAC Validator --- Cargo.lock | 354 +++++++++++++++++---------------- frame/ddc-validator/Cargo.toml | 13 ++ frame/ddc-validator/src/lib.rs | 56 +++++- 3 files changed, 248 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc9b451a20811..cd4b107499615 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,7 +255,7 @@ dependencies = [ "fastrand", "futures-lite", "libc", - "log 0.4.14", + "log 0.4.17", "nb-connect", "once_cell", "parking", @@ -301,7 +301,7 @@ dependencies = [ "futures-lite", "gloo-timers", "kv-log-macro", - "log 0.4.14", + "log 0.4.17", "memchr", "num_cpus", "once_cell", @@ -515,7 +515,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "hex", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-chain-spec", @@ -559,7 +559,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-rpc", @@ -579,7 +579,7 @@ dependencies = [ "env_logger 0.9.0", "hex", "hex-literal", - "log 0.4.14", + "log 0.4.17", "tiny-keccak", ] @@ -1217,7 +1217,7 @@ dependencies = [ "cranelift-codegen-shared 0.76.0", "cranelift-entity 0.76.0", "gimli 0.25.0", - "log 0.4.14", + "log 0.4.17", "regalloc 0.0.31", "smallvec 1.8.0", "target-lexicon", @@ -1234,7 +1234,7 @@ dependencies = [ "cranelift-codegen-shared 0.80.0", "cranelift-entity 0.80.0", "gimli 0.26.1", - "log 0.4.14", + "log 0.4.17", "regalloc 0.0.33", "smallvec 1.8.0", "target-lexicon", @@ -1293,7 +1293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279afcc0d3e651b773f94837c3d581177b348c8d69e928104b2e9fccb226f921" dependencies = [ "cranelift-codegen 0.76.0", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "target-lexicon", ] @@ -1305,7 +1305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7" dependencies = [ "cranelift-codegen 0.80.0", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "target-lexicon", ] @@ -1331,7 +1331,7 @@ dependencies = [ "cranelift-entity 0.80.0", "cranelift-frontend 0.80.0", "itertools", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "wasmparser 0.81.0", "wasmtime-types", @@ -1971,7 +1971,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime 1.3.0", - "log 0.4.14", + "log 0.4.17", "regex", "termcolor", ] @@ -1982,7 +1982,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ - "log 0.4.14", + "log 0.4.17", "regex", ] @@ -1994,7 +1994,7 @@ checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime 2.1.0", - "log 0.4.14", + "log 0.4.17", "regex", "termcolor", ] @@ -2088,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" dependencies = [ "env_logger 0.7.1", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -2100,7 +2100,7 @@ dependencies = [ "either", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "num-traits", "parity-scale-codec", "parking_lot 0.11.2", @@ -2185,7 +2185,7 @@ dependencies = [ "frame-system", "hex-literal", "linregress", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "paste", "scale-info", @@ -2216,7 +2216,7 @@ dependencies = [ "itertools", "kvdb", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "memory-db", "parity-scale-codec", "rand 0.8.4", @@ -2337,7 +2337,7 @@ dependencies = [ "frame-support-procedural", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "once_cell", "parity-scale-codec", "parity-util-mem", @@ -2441,7 +2441,7 @@ version = "4.0.0-dev" dependencies = [ "criterion", "frame-support", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "serde", @@ -2776,7 +2776,7 @@ dependencies = [ "bitflags", "libc", "libgit2-sys", - "log 0.4.14", + "log 0.4.17", "url 2.2.1", ] @@ -2795,7 +2795,7 @@ dependencies = [ "aho-corasick", "bstr", "fnv", - "log 0.4.14", + "log 0.4.17", "regex", ] @@ -2854,7 +2854,7 @@ version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ - "log 0.4.14", + "log 0.4.17", "pest", "pest_derive", "quick-error 2.0.0", @@ -3097,7 +3097,7 @@ dependencies = [ "ct-logs", "futures-util", "hyper 0.14.16", - "log 0.4.14", + "log 0.4.17", "rustls 0.19.1", "rustls-native-certs 0.5.0", "tokio", @@ -3179,7 +3179,7 @@ dependencies = [ "if-addrs", "ipnet", "libc", - "log 0.4.14", + "log 0.4.17", "winapi 0.3.9", ] @@ -3334,7 +3334,7 @@ dependencies = [ "hyper-tls", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "serde", "serde_json 1.0.79", "tokio", @@ -3351,7 +3351,7 @@ dependencies = [ "futures 0.3.21", "futures-executor", "futures-util", - "log 0.4.14", + "log 0.4.17", "serde", "serde_derive", "serde_json 1.0.79", @@ -3389,7 +3389,7 @@ dependencies = [ "hyper 0.14.16", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "net2", "parking_lot 0.11.2", "unicase 2.6.0", @@ -3404,7 +3404,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "parity-tokio-ipc", "parking_lot 0.11.2", "tower-service", @@ -3419,7 +3419,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.11.2", "rand 0.7.3", "serde", @@ -3436,7 +3436,7 @@ dependencies = [ "globset", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.17", "tokio", "tokio-stream", "tokio-util 0.6.7", @@ -3452,7 +3452,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "parity-ws", "parking_lot 0.11.2", "slab", @@ -3596,7 +3596,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -3628,7 +3628,7 @@ checksum = "ca7fbdfd71cd663dceb0faf3367a99f8cf724514933e9867cec4995b6027cbc1" dependencies = [ "fs-swap", "kvdb", - "log 0.4.14", + "log 0.4.17", "num_cpus", "owning_ref", "parity-util-mem", @@ -3762,7 +3762,7 @@ dependencies = [ "futures-timer", "lazy_static", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "multiaddr", "multihash 0.14.0", "multistream-select", @@ -3801,7 +3801,7 @@ dependencies = [ "async-std-resolver", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "trust-dns-resolver", ] @@ -3817,7 +3817,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3839,7 +3839,7 @@ dependencies = [ "hex_fmt", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3859,7 +3859,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "lru 0.6.6", "prost", "prost-build", @@ -3881,7 +3881,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3907,7 +3907,7 @@ dependencies = [ "lazy_static", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "smallvec 1.8.0", "socket2 0.4.4", @@ -3938,7 +3938,7 @@ dependencies = [ "bytes 1.1.0", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "nohash-hasher", "parking_lot 0.11.2", "rand 0.7.3", @@ -3957,7 +3957,7 @@ dependencies = [ "futures 0.3.21", "lazy_static", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.8.4", @@ -3977,7 +3977,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "void", "wasm-timer", @@ -3993,7 +3993,7 @@ dependencies = [ "bytes 1.1.0", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "unsigned-varint 0.7.0", @@ -4007,7 +4007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1a458bbda880107b5b36fcb9b5a1ef0c329685da0e203ed692a8ebe64cc92c" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "rand 0.7.3", "salsa20", @@ -4026,7 +4026,7 @@ dependencies = [ "futures-timer", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "prost", "prost-build", @@ -4048,7 +4048,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.8.4", @@ -4070,7 +4070,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "lru 0.7.3", "rand 0.7.3", "smallvec 1.8.0", @@ -4087,7 +4087,7 @@ dependencies = [ "either", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "smallvec 1.8.0", "void", @@ -4117,7 +4117,7 @@ dependencies = [ "ipnet", "libc", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "socket2 0.4.4", ] @@ -4130,7 +4130,7 @@ dependencies = [ "async-std", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -4157,7 +4157,7 @@ dependencies = [ "futures 0.3.21", "futures-rustls", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "quicksink", "rw-stream-sink", "soketto 0.7.1", @@ -4326,14 +4326,14 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.14", + "log 0.4.17", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", "value-bag", @@ -4567,7 +4567,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.14", + "log 0.4.17", "miow 0.2.2", "net2", "slab", @@ -4581,7 +4581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" dependencies = [ "libc", - "log 0.4.14", + "log 0.4.17", "miow 0.3.6", "ntapi", "winapi 0.3.9", @@ -4594,7 +4594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "slab", ] @@ -4714,7 +4714,7 @@ checksum = "7d91ec0a2440aaff5f78ec35631a7027d50386c6163aa975f7caa0d5da4b6ff8" dependencies = [ "bytes 1.1.0", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "smallvec 1.8.0", "unsigned-varint 0.7.0", @@ -4766,7 +4766,7 @@ checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "openssl", "openssl-probe", "openssl-sys", @@ -4823,7 +4823,7 @@ dependencies = [ "kvdb", "kvdb-rocksdb", "lazy_static", - "log 0.4.14", + "log 0.4.17", "node-primitives", "node-runtime", "node-testing", @@ -4861,7 +4861,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.21", "hex-literal", - "log 0.4.14", + "log 0.4.17", "nix", "node-executor", "node-inspect", @@ -5032,7 +5032,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "frame-try-runtime", "hex-literal", - "log 0.4.14", + "log 0.4.17", "node-primitives", "pallet-authority-discovery", "pallet-authorship", @@ -5118,7 +5118,7 @@ dependencies = [ "frame-system", "fs_extra", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "node-executor", "node-primitives", "node-runtime", @@ -5478,7 +5478,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -5507,7 +5507,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -5535,7 +5535,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-bags-list", "pallet-staking", "remote-externalities", @@ -5554,7 +5554,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-transaction-payment", "parity-scale-codec", "scale-info", @@ -5593,7 +5593,7 @@ dependencies = [ "hex", "hex-literal", "k256", - "log 0.4.14", + "log 0.4.17", "pallet-beefy", "pallet-mmr", "pallet-mmr-primitives", @@ -5615,7 +5615,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-treasury", "parity-scale-codec", @@ -5666,7 +5666,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-bounties", "pallet-treasury", @@ -5685,7 +5685,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -5705,7 +5705,7 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-contracts-primitives", "pallet-contracts-proc-macro", @@ -5851,8 +5851,15 @@ version = "0.1.0" dependencies = [ "frame-support", "frame-system", + "log 0.4.17", + "pallet-ddc-staking", + "pallet-staking", "parity-scale-codec", "scale-info", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", ] [[package]] @@ -5881,7 +5888,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "parking_lot 0.12.0", @@ -5905,7 +5912,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -5981,7 +5988,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -6026,7 +6033,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-session", "parity-scale-codec", @@ -6080,7 +6087,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -6115,7 +6122,7 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "serde", "sp-api", @@ -6178,7 +6185,7 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -6193,7 +6200,7 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6321,7 +6328,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-preimage", "parity-scale-codec", "scale-info", @@ -6354,7 +6361,7 @@ dependencies = [ "frame-support", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "pallet-timestamp", "parity-scale-codec", "scale-info", @@ -6415,7 +6422,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-bags-list", "pallet-balances", @@ -6452,7 +6459,7 @@ dependencies = [ name = "pallet-staking-reward-fn" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "sp-arithmetic", ] @@ -6477,7 +6484,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -6495,7 +6502,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-treasury", "parity-scale-codec", @@ -6597,7 +6604,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6630,7 +6637,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6669,7 +6676,7 @@ dependencies = [ "fs2", "hex", "libc", - "log 0.4.14", + "log 0.4.17", "lz4", "memmap2 0.2.1", "parking_lot 0.11.2", @@ -6717,7 +6724,7 @@ checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" dependencies = [ "futures 0.3.21", "libc", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "tokio", "winapi 0.3.9", @@ -6774,7 +6781,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "httparse", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "mio-extras", "rand 0.7.3", @@ -7075,7 +7082,7 @@ checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" dependencies = [ "cfg-if 0.1.10", "libc", - "log 0.4.14", + "log 0.4.17", "wepoll-sys", "winapi 0.3.9", ] @@ -7259,7 +7266,7 @@ dependencies = [ "heck 0.3.2", "itertools", "lazy_static", - "log 0.4.14", + "log 0.4.17", "multimap", "petgraph", "prost", @@ -7340,7 +7347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger 0.8.4", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", ] @@ -7677,7 +7684,7 @@ version = "0.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" dependencies = [ - "log 0.4.14", + "log 0.4.17", "rustc-hash", "smallvec 1.8.0", ] @@ -7688,7 +7695,7 @@ version = "0.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" dependencies = [ - "log 0.4.14", + "log 0.4.17", "rustc-hash", "smallvec 1.8.0", ] @@ -7751,7 +7758,7 @@ dependencies = [ "env_logger 0.9.0", "frame-support", "jsonrpsee", - "log 0.4.14", + "log 0.4.17", "pallet-elections-phragmen", "parity-scale-codec", "serde", @@ -7914,7 +7921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", - "log 0.4.14", + "log 0.4.17", "ring", "sct 0.6.0", "webpki 0.21.4", @@ -7926,7 +7933,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" dependencies = [ - "log 0.4.14", + "log 0.4.17", "ring", "sct 0.7.0", "webpki 0.22.0", @@ -8025,7 +8032,7 @@ dependencies = [ name = "sc-allocator" version = "4.1.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "sp-core", "sp-wasm-interface", "thiserror", @@ -8040,7 +8047,7 @@ dependencies = [ "futures-timer", "ip_network", "libp2p", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "prost", "prost-build", @@ -8066,7 +8073,7 @@ version = "0.10.0-dev" dependencies = [ "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8137,7 +8144,7 @@ dependencies = [ "futures 0.3.21", "hex", "libp2p", - "log 0.4.14", + "log 0.4.17", "names", "parity-scale-codec", "rand 0.7.3", @@ -8172,7 +8179,7 @@ dependencies = [ "fnv", "futures 0.3.21", "hash-db", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-executor", @@ -8204,7 +8211,7 @@ dependencies = [ "kvdb-memorydb", "kvdb-rocksdb", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "parity-db", "parity-scale-codec", "parking_lot 0.12.0", @@ -8231,7 +8238,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "sc-client-api", "sc-utils", @@ -8253,7 +8260,7 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8291,7 +8298,7 @@ dependencies = [ "async-trait", "fork-tree", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "merlin", "num-bigint", "num-rational 0.2.4", @@ -8385,7 +8392,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-basic-authorship", "sc-client-api", @@ -8421,7 +8428,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-client-api", @@ -8445,7 +8452,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-consensus", @@ -8533,7 +8540,7 @@ dependencies = [ name = "sc-executor-wasmi" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-allocator", "sc-executor-common", @@ -8550,7 +8557,7 @@ version = "0.10.0-dev" dependencies = [ "cfg-if 1.0.0", "libc", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-wasm 0.42.2", "sc-allocator", @@ -8577,7 +8584,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "hex", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "rand 0.8.4", @@ -8621,7 +8628,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-block-builder", "sc-client-api", @@ -8645,7 +8652,7 @@ dependencies = [ "ansi_term 0.12.1", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-util-mem", "sc-client-api", "sc-network", @@ -8690,7 +8697,7 @@ dependencies = [ "libp2p", "linked-hash-map", "linked_hash_set", - "log 0.4.14", + "log 0.4.17", "lru 0.7.3", "parity-scale-codec", "parking_lot 0.12.0", @@ -8734,7 +8741,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", + "log 0.4.17", "lru 0.7.3", "quickcheck", "sc-network", @@ -8753,7 +8760,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "rand 0.7.3", "sc-block-builder", @@ -8813,7 +8820,7 @@ version = "4.0.0-dev" dependencies = [ "futures 0.3.21", "libp2p", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "sc-utils", "serde_json 1.0.79", @@ -8824,7 +8831,7 @@ dependencies = [ name = "sc-proposer-metrics" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "substrate-prometheus-endpoint", ] @@ -8838,7 +8845,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-pubsub", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8874,7 +8881,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-chain-spec", @@ -8900,7 +8907,7 @@ dependencies = [ "jsonrpc-ipc-server", "jsonrpc-pubsub", "jsonrpc-ws-server", - "log 0.4.14", + "log 0.4.17", "serde_json 1.0.79", "substrate-prometheus-endpoint", "tokio", @@ -8933,7 +8940,7 @@ dependencies = [ "hash-db", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parking_lot 0.12.0", @@ -8994,7 +9001,7 @@ dependencies = [ "futures 0.3.21", "hex", "hex-literal", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -9026,7 +9033,7 @@ dependencies = [ name = "sc-state-db" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parity-util-mem-derive", @@ -9062,7 +9069,7 @@ dependencies = [ "chrono", "futures 0.3.21", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "pin-project 1.0.10", "rand 0.7.3", @@ -9082,7 +9089,7 @@ dependencies = [ "criterion", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "once_cell", "parking_lot 0.12.0", "regex", @@ -9123,7 +9130,7 @@ dependencies = [ "futures-timer", "hex", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parking_lot 0.12.0", @@ -9152,7 +9159,7 @@ name = "sc-transaction-pool-api" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "serde", "sp-blockchain", "sp-runtime", @@ -9166,7 +9173,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "prometheus", "tokio-test", @@ -9658,7 +9665,7 @@ dependencies = [ "bytes 0.5.6", "futures 0.3.21", "httparse", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "sha-1 0.9.4", ] @@ -9674,7 +9681,7 @@ dependencies = [ "flate2", "futures 0.3.21", "httparse", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "sha-1 0.9.4", ] @@ -9684,7 +9691,7 @@ name = "sp-api" version = "4.0.0-dev" dependencies = [ "hash-db", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-api-proc-macro", "sp-core", @@ -9713,7 +9720,7 @@ version = "2.0.1" dependencies = [ "criterion", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "rustversion", "sc-block-builder", @@ -9818,7 +9825,7 @@ name = "sp-blockchain" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "lru 0.7.3", "parity-scale-codec", "parking_lot 0.12.0", @@ -9837,7 +9844,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-inherents", @@ -9942,7 +9949,7 @@ dependencies = [ "impl-serde", "lazy_static", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "merlin", "num-traits", "parity-scale-codec", @@ -10028,7 +10035,7 @@ name = "sp-finality-grandpa" version = "4.0.0-dev" dependencies = [ "finality-grandpa", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "serde", @@ -10061,7 +10068,7 @@ dependencies = [ "futures 0.3.21", "hash-db", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "secp256k1", @@ -10177,7 +10184,7 @@ dependencies = [ "either", "hash256-std-hasher", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "paste", @@ -10273,7 +10280,7 @@ name = "sp-sandbox" version = "0.10.0-dev" dependencies = [ "assert_matches", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-io", @@ -10320,7 +10327,7 @@ version = "0.12.0" dependencies = [ "hash-db", "hex-literal", - "log 0.4.14", + "log 0.4.17", "num-traits", "parity-scale-codec", "parking_lot 0.12.0", @@ -10358,7 +10365,7 @@ dependencies = [ name = "sp-tasks" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-externalities", @@ -10385,7 +10392,7 @@ version = "4.0.0-dev" dependencies = [ "async-trait", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-api", "sp-inherents", @@ -10418,7 +10425,7 @@ name = "sp-transaction-storage-proof" version = "4.0.0-dev" dependencies = [ "async-trait", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -10480,7 +10487,7 @@ name = "sp-wasm-interface" version = "6.0.0" dependencies = [ "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-std", "wasmi", @@ -10636,7 +10643,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-rpc-api", @@ -10657,7 +10664,7 @@ version = "0.10.0-dev" dependencies = [ "futures-util", "hyper 0.14.16", - "log 0.4.14", + "log 0.4.17", "prometheus", "thiserror", "tokio", @@ -10670,7 +10677,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-rpc-api", @@ -10721,7 +10728,7 @@ dependencies = [ "frame-system", "frame-system-rpc-runtime-api", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "memory-db", "pallet-babe", "pallet-timestamp", @@ -11083,7 +11090,7 @@ checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", "futures 0.1.31", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -11116,7 +11123,7 @@ dependencies = [ "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "num_cpus", "parking_lot 0.9.0", @@ -11216,7 +11223,7 @@ dependencies = [ "bytes 1.1.0", "futures-core", "futures-sink", - "log 0.4.14", + "log 0.4.17", "pin-project-lite 0.2.6", "tokio", ] @@ -11257,7 +11264,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", - "log 0.4.14", + "log 0.4.17", "pin-project-lite 0.2.6", "tracing-attributes", "tracing-core", @@ -11300,7 +11307,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" dependencies = [ "lazy_static", - "log 0.4.14", + "log 0.4.17", "tracing-core", ] @@ -11373,7 +11380,7 @@ checksum = "d32d034c0d3db64b43c31de38e945f15b40cd4ca6d2dcfc26d4798ce8de4ab83" dependencies = [ "hash-db", "hashbrown 0.12.0", - "log 0.4.14", + "log 0.4.17", "rustc-hex", "smallvec 1.8.0", ] @@ -11413,7 +11420,7 @@ dependencies = [ "idna 0.2.2", "ipnet", "lazy_static", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "smallvec 1.8.0", "thiserror", @@ -11431,7 +11438,7 @@ dependencies = [ "futures-util", "ipconfig", "lazy_static", - "log 0.4.14", + "log 0.4.17", "lru-cache", "parking_lot 0.11.2", "resolv-conf", @@ -11452,7 +11459,7 @@ version = "0.10.0-dev" dependencies = [ "clap 3.1.6", "jsonrpsee", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "remote-externalities", "sc-chain-spec", @@ -11664,11 +11671,12 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.0.0-alpha.6" +version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", + "version_check 0.9.2", ] [[package]] @@ -11733,7 +11741,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.14", + "log 0.4.17", "try-lock", ] @@ -11767,7 +11775,7 @@ checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.14", + "log 0.4.17", "proc-macro2", "quote", "syn", @@ -11821,7 +11829,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c32691b6c7e6c14e7f8fd55361a9088b507aa49620fcd06c09b3a1082186b9" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-wasm 0.32.0", "rustc-demangle", ] @@ -12115,7 +12123,7 @@ dependencies = [ "indexmap", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "object 0.27.1", "paste", "psm", @@ -12144,7 +12152,7 @@ dependencies = [ "bincode", "directories-next", "file-per-thread-logger", - "log 0.4.14", + "log 0.4.17", "rustix", "serde", "sha2 0.9.8", @@ -12166,7 +12174,7 @@ dependencies = [ "cranelift-native", "cranelift-wasm", "gimli 0.26.1", - "log 0.4.14", + "log 0.4.17", "more-asserts", "object 0.27.1", "target-lexicon", @@ -12185,7 +12193,7 @@ dependencies = [ "cranelift-entity 0.80.0", "gimli 0.26.1", "indexmap", - "log 0.4.14", + "log 0.4.17", "more-asserts", "object 0.27.1", "serde", @@ -12230,7 +12238,7 @@ dependencies = [ "indexmap", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "mach", "memoffset", "more-asserts", @@ -12518,7 +12526,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "nohash-hasher", "parking_lot 0.11.2", "rand 0.8.4", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index d9525b4dfd9af..057a274a2ec1b 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -7,7 +7,14 @@ edition = "2021" codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +log = { version = "0.4.17", default-features = false } +pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } [features] default = ["std"] @@ -15,5 +22,11 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", + "pallet-ddc-staking/std", + "pallet-staking/std", "scale-info/std", + "sp-io/std", + "sp-runtime/std", + "sp-staking/std", + "sp-std/std", ] diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a9b77496845ff..df766e3695547 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,8 +1,27 @@ #![cfg_attr(not(feature = "std"), no_std)] -pub use frame_support::pallet_prelude::*; +pub use frame_support::{pallet_prelude::*, parameter_types, weights::Weight, BoundedVec}; pub use frame_system::pallet_prelude::*; +pub use pallet_ddc_staking::{self as ddc_staking}; +pub use pallet_staking::{self as staking}; pub use pallet::*; +pub use sp_std::prelude::*; + +parameter_types! { + pub DdcValidatorsQuorumSize: u32 = 3; +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum ValidationMethodKind { + ProofOfDelivery, +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct Decision { + pub decision: Option, + pub method: ValidationMethodKind, + pub validator: AccountId, +} #[frame_support::pallet] pub mod pallet { @@ -13,14 +32,27 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config { + pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { type Event: From> + IsType<::Event>; } + #[pallet::storage] + #[pallet::getter(fn tasks)] + pub type Tasks = StorageMap< + _, + Twox64Concat, + T::AccountId, + BoundedVec, DdcValidatorsQuorumSize>, + >; + #[pallet::storage] #[pallet::getter(fn global_era_counter)] pub type GlobalEraCounter = StorageValue<_, u32>; + #[pallet::storage] + #[pallet::getter(fn last_managed_era)] + pub type LastManagedEra = StorageValue<_, u32>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event {} @@ -42,4 +74,24 @@ pub mod pallet { Ok(()) } } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(block_number: T::BlockNumber) -> Weight { + let validators: Vec = >::iter_keys().collect(); + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators: {:?}, edges: {:?}", + block_number, + >::get(), + >::get(), + validators, + edges, + ); + 0 + } + fn offchain_worker(block_number: T::BlockNumber) { + log::info!("Off-chain worker at block {:?}", block_number); + } + } } From 643a6c84cb68e54abe47c09719161af1ca228a5c Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 17 Mar 2023 14:25:39 +0100 Subject: [PATCH 005/192] add function fetch_tasks --- frame/ddc-validator/src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index df766e3695547..77d1d894b2257 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -94,4 +94,20 @@ pub mod pallet { log::info!("Off-chain worker at block {:?}", block_number); } } + + impl Pallet { + /// Fetch the tasks related to current validator + fn fetch_tasks(validator: T::AccountId) -> Vec { + let mut cdn_nodes: Vec = vec![]; + for (cdn_id, cdn_tasks) in >::iter() { + for decision in cdn_tasks.iter() { + if decision.validator == validator { + cdn_nodes.push(cdn_id); + break; + } + } + } + cdn_nodes + } + } } From 15b8341ac1929dfc067bc8fdb6378fb77d3f0dc3 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 1 Mar 2023 09:33:18 +0100 Subject: [PATCH 006/192] Create OCW, read from redis, compare bytes sent and bytes recieved, send tx onchain with the comparison Make http request from ocw Fix arbitary http request Try to connect to redis Clean up Try use redis via webdis Introduce structs Small improvements Code quality improvements Call offchain validator once per era Set shorted duration Update EPOCH_DURATION_IN_BLOCKS Fix short durations Integrade OCW to staking Specify ValidatorId type in OCW config trait bound Add logs Try fix issue with storage Increase points via on_initialize hook Try to send tx from offchain worker Read and store data from OCW Configure keys for OCW Use more advanced data storage Make save_validated_data with updated storage Revert "Fix short durations" This reverts commit d36efd54594d2c698e7c713d85120c6c5b86e9ca. Revert "Update EPOCH_DURATION_IN_BLOCKS" This reverts commit 87c67b138f3f593fffd5b745a4261c682dd0a24a. Revert "Set shorted duration" This reverts commit e5283c8202308364d32418ff4c483ef0b9996f0f. Fix logs for --chain=local Add signer to validation results and fix bug with KeyTypeId Validated fetched data and send tx Migrate to the latest pallet macro Remove ocw with old pallet macro --- Cargo.lock | 115 +++--- bin/node/runtime/Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 8 + .../dac-validator-offchain-worker/Cargo.toml | 58 +++ .../dac-validator-offchain-worker/src/lib.rs | 360 ++++++++++++++++++ frame/staking/src/lib.rs | 14 + frame/staking/src/pallet/impls.rs | 39 +- frame/staking/src/pallet/mod.rs | 13 +- 8 files changed, 559 insertions(+), 49 deletions(-) create mode 100644 frame/dac-validator-offchain-worker/Cargo.toml create mode 100644 frame/dac-validator-offchain-worker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cd4b107499615..3d241900ddcb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,7 @@ checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ "getrandom 0.2.3", "once_cell", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -369,7 +369,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4401f0a3622dad2e0763fa79e0eb328bc70fb7dccfdd645341f00d671247d6" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-sink", "futures-util", "memchr", @@ -382,7 +382,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0de5164e5edbf51c45fb8c2d9664ae1c095cce1b265ecf7569093c0d66ef690" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-sink", "futures-util", "memchr", @@ -869,9 +869,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bzip2-sys" @@ -2708,7 +2708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" dependencies = [ "typenum", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -2829,7 +2829,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -3002,7 +3002,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "itoa 0.4.8", ] @@ -3013,7 +3013,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "http", "pin-project-lite 0.2.6", ] @@ -3070,7 +3070,7 @@ version = "0.14.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -3111,7 +3111,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "hyper 0.14.16", "native-tls", "tokio", @@ -3431,7 +3431,7 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "globset", "jsonrpc-core", @@ -3702,9 +3702,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.1" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libp2p" @@ -3713,7 +3713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bec54343492ba5940a6c555e512c6721139835d28c59bc22febece72dfd0d9d" dependencies = [ "atomic", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "lazy_static", "libp2p-core", @@ -3833,7 +3833,7 @@ dependencies = [ "asynchronous-codec 0.6.0", "base64 0.13.0", "byteorder", - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures 0.3.21", "hex_fmt", @@ -3875,7 +3875,7 @@ checksum = "a2297dc0ca285f3a09d1368bde02449e539b46f94d32d53233f53f6625bcd3ba" dependencies = [ "arrayvec 0.5.2", "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "either", "fnv", "futures 0.3.21", @@ -3935,7 +3935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f2cd64ef597f40e14bfce0497f50ecb63dd6d201c61796daeb4227078834fbf" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", "log 0.4.17", @@ -3952,7 +3952,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8772c7a99088221bb7ca9c5c0574bf55046a7ab4c319f3619b275f28c8fb87a" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "curve25519-dalek 3.0.2", "futures 0.3.21", "lazy_static", @@ -3990,7 +3990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fba1a6ff33e4a274c89a3b1d78b9f34f32af13265cc5c46c16938262d4e945a" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", "log 0.4.17", @@ -4021,7 +4021,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2852b61c90fa8ce3c8fcc2aba76e6cefc20d648f9df29157d6b3a916278ef3e3" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "futures-timer", "libp2p-core", @@ -4066,7 +4066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a877a4ced6d46bf84677e1974e8cf61fb434af73b2e96fb48d6cb6223a4634d8" dependencies = [ "async-trait", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", "libp2p-swarm", @@ -4712,7 +4712,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d91ec0a2440aaff5f78ec35631a7027d50386c6163aa975f7caa0d5da4b6ff8" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "log 0.4.17", "pin-project 1.0.10", @@ -5047,6 +5047,7 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", + "pallet-dac-validator-offchain-worker", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", "pallet-ddc-validator", @@ -5164,7 +5165,7 @@ checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" dependencies = [ "memchr", "minimal-lexical", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -5284,9 +5285,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.8.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "oorandom" @@ -5803,6 +5804,34 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-dac-validator-offchain-worker" +version = "0.0.1" +dependencies = [ + "alt_serde", + "frame-support", + "frame-system", + "hex", + "hex-literal", + "lite-json", + "pallet-balances", + "pallet-contracts", + "pallet-randomness-collective-flip", + "pallet-session", + "pallet-staking", + "pallet-timestamp", + "parity-scale-codec", + "pretty_assertions 0.6.1", + "scale-info", + "serde", + "serde_json 1.0.44", + "sp-core", + "sp-io", + "sp-keystore", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-ddc-metrics-offchain-worker" version = "2.0.1" @@ -7209,7 +7238,7 @@ dependencies = [ "proc-macro2", "quote", "syn", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -7220,7 +7249,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -7252,7 +7281,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "prost-derive", ] @@ -7262,7 +7291,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "heck 0.3.2", "itertools", "lazy_static", @@ -7295,7 +7324,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "prost", ] @@ -8685,7 +8714,7 @@ dependencies = [ "async-trait", "asynchronous-codec 0.5.0", "bitflags", - "bytes 1.1.0", + "bytes 1.4.0", "cid", "either", "fnv", @@ -8782,7 +8811,7 @@ dependencies = [ name = "sc-offchain" version = "4.0.0-dev" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures 0.3.21", "futures-timer", @@ -9677,7 +9706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.0", - "bytes 1.1.0", + "bytes 1.4.0", "flate2", "futures 0.3.21", "httparse", @@ -11047,7 +11076,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "libc", "memchr", "mio 0.8.0", @@ -11197,7 +11226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" dependencies = [ "async-stream", - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "tokio", "tokio-stream", @@ -11220,7 +11249,7 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "futures-sink", "log 0.4.17", @@ -11234,7 +11263,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "futures-io", "futures-sink", @@ -11555,7 +11584,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -11623,7 +11652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35581ff83d4101e58b582e607120c7f5ffb17e632a980b1f38334d76b36908b2" dependencies = [ "asynchronous-codec 0.5.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures-io", "futures-util", ] @@ -11635,7 +11664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f8d425fafb8cd76bc3f22aace4af471d3156301d7508f2107e98fbeae10bc7f" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures-io", "futures-util", ] @@ -11699,9 +11728,9 @@ checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "void" diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index ed563f445eb15..fd8b311a7a709 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -97,6 +97,7 @@ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../f pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } +pallet-dac-validator-offchain-worker = { version = "0.0.1", default-features = false, path = "../../../frame/dac-validator-offchain-worker" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index d13d2595cf0bb..a0ec718430091 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -82,6 +82,7 @@ pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; pub use pallet_ddc_validator; +pub use pallet_dac_validator_offchain_worker; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1214,6 +1215,12 @@ parameter_types! { pub const OcwBlockInterval: u32 = pallet_ddc_metrics_offchain_worker::BLOCK_INTERVAL; } +impl pallet_dac_validator_offchain_worker::Config for Runtime { + type Call = Call; + + type AuthorityId = pallet_dac_validator_offchain_worker::crypto::TestAuthId; +} + impl pallet_ddc_metrics_offchain_worker::Config for Runtime { type BlockInterval = OcwBlockInterval; @@ -1283,6 +1290,7 @@ construct_runtime!( Erc721: pallet_erc721::{Pallet, Call, Storage, Event}, Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, + DdcValidatorOffchainWorker: pallet_dac_validator_offchain_worker::{Pallet, Call, Storage}, DdcStaking: pallet_ddc_staking, DdcValidator: pallet_ddc_validator, } diff --git a/frame/dac-validator-offchain-worker/Cargo.toml b/frame/dac-validator-offchain-worker/Cargo.toml new file mode 100644 index 0000000000000..7f978de7fe981 --- /dev/null +++ b/frame/dac-validator-offchain-worker/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "pallet-dac-validator-offchain-worker" +version = "0.0.1" +authors = ["Parity Technologies "] +edition = "2021" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet for offchain worker" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["full"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +serde = { version = "1.0.101", optional = true } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } +lite-json = { version = "0.2.0", default-features = false } +alt_serde = { version = "1", default-features = false, features = ["derive"] } +serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } +hex-literal = "^0.3.1" +hex = { version = "0.4", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } +pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-keystore", + "frame-support/std", + "frame-system/std", + "serde", + "lite-json/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", + "pallet-contracts/std", + "pallet-staking/std", + "pallet-session/std", + # "pallet-contracts-rpc-runtime-api/std", +] + +[dev-dependencies] +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } +pretty_assertions = "0.6.1" \ No newline at end of file diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs new file mode 100644 index 0000000000000..5670f9689dcb7 --- /dev/null +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -0,0 +1,360 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use alloc::{format, string::String}; +use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +use frame_support::{ + decl_event, decl_module, decl_storage, + log::{error, info, warn}, + traits::Currency, + weights::{Weight}, + dispatch::DispatchResult, + RuntimeDebug, + BoundedVec, + parameter_types, +}; +use frame_system::{ensure_signed, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; +use pallet_staking::{self as staking}; +use sp_core::crypto::{KeyTypeId, UncheckedFrom};use sp_io::offchain::timestamp; +use sp_runtime::offchain::{http, Duration}; +use sp_std::{vec, vec::Vec}; +extern crate alloc; +use pallet_session as session; + +parameter_types! { + pub DdcValidatorsQuorumSize: u32 = 3; +} + +use core::fmt::Debug; +use scale_info::TypeInfo; + +type BalanceOf = <::Currency as Currency< + ::AccountId, +>>::Balance; + +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); + +pub const HTTP_TIMEOUT_MS: u64 = 30_000; + +const TIME_START_MS: u64 = 1_672_531_200_000; +const ERA_DURATION_MS: u64 = 120_000; +const ERA_IN_BLOCKS: u8 = 20; + +const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; + +type ResultStr = Result; + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct RedisFtAggregate { + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: (u32, Vec), +} + +#[derive(Clone)] +struct BytesSent { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesSent { + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let (_, values) = aggregate.ft_aggregate; + + BytesSent { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } +} + +#[derive(Clone)] +struct BytesReceived { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesReceived { + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let (_, values) = aggregate.ft_aggregate; + + BytesReceived { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } +} + +#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] +pub struct ValidationResult { + block_number: BlockNumber, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, +} + +// use sp_std::fmt; +// impl fmt::Display for ValidationResult +// { +// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +// write!(f, "ValidationResult signer {}", self.signer) +// } +// } + +pub mod crypto { + use super::KEY_TYPE; + use frame_system::offchain::AppCrypto; + use sp_core::sr25519::Signature as Sr25519Signature; + use sp_runtime::{ + app_crypto::{app_crypto, sr25519}, + traits::Verify, + }; + app_crypto!(sr25519, KEY_TYPE); + + use sp_runtime::{MultiSignature, MultiSigner}; + + pub struct TestAuthId; + + impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } + + impl AppCrypto for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::without_storage_info] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: + frame_system::Config + + pallet_contracts::Config + + pallet_session::Config::AccountId> + + pallet_staking::Config + + CreateSignedTransaction> + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + /// The overarching dispatch call type. + type Call: From>; + + type AuthorityId: AppCrypto; + } + + #[pallet::hooks] + impl Hooks> for Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + fn offchain_worker(block_number: T::BlockNumber) { + let res = Self::offchain_worker_main(block_number); + + match res { + Ok(()) => info!("[DDC Validator] DDC Validator is suspended."), + Err(err) => error!("[DDC Validator] Error in Offchain Worker: {}", err), + }; + } + } + + #[pallet::call] + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + #[pallet::weight(1000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, block_number: T::BlockNumber) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + info!("author: {:?}", signer); + let mut v_results = ValidationResults::::get(); + + let cur_validation = ValidationResult:: { + block_number, + val_res, + cdn_node_pub_key, + signer, + }; + + v_results.push(cur_validation); + + ValidationResults::::set(v_results); + + Ok(()) + } + } + + #[pallet::storage] + #[pallet::getter(fn validation_results)] + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; +} + +impl Pallet +where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, +{ + fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { + return Ok(()) + } + + let signer = match Self::get_signer() { + Err(e) => { + warn!("{:?}", e); + return Ok(()); + } + Ok(signer) => signer, + }; + + info!("ValidationResults: {:?}", ValidationResults::::get()); + + // Read data from DataModel and do dumb validation + let current_era = Self::get_current_era(); + let (bytes_sent, bytes_received) = Self::fetch_data(current_era); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let cdn_node_pub_key = bytes_sent.node_public_key.clone(); + let tx_res = signer.send_signed_transaction(|_acct| { + info!("Sending save_validated_data tx"); + + // This is the on-chain function + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), block_number } + }); + + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting save_validated_data TX") + } + Some((_, Ok(()))) => {} + } + + info!("save_validated_data: {:?}", ValidationResults::::get()); + + + Ok(()) + } + + fn get_signer() -> ResultStr> { + let signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + return Err("[DDC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + } + Ok(signer) + } + + fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { + return if bytes_sent.sum == bytes_received.sum { + true + } else { + false + } + } + + fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + info!("[DDC Validator] DDC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::new(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::new(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_current_era() -> u64 { + // info!("timestamp: {}", timestamp().unix_millis()); + // info!("TIME_START_MS: {}", TIME_START_MS); + // info!("ERA_DURATION_MS: {}", ERA_DURATION_MS); + // info!("1: {}", timestamp().unix_millis() - TIME_START_MS); + // info!("2: {}", (timestamp().unix_millis() - TIME_START_MS) % ERA_DURATION_MS); + + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + } + + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + } + + fn http_get_json(url: &str) -> ResultStr { + let body = Self::http_get_request(url).map_err(|err| { + error!("[DDC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" + })?; + + let parsed = serde_json::from_slice(&body).map_err(|err| { + warn!("[DDC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" + }); + + parsed + } + + fn http_get_request(http_url: &str) -> Result, http::Error> { + info!("[DDC Validator] Sending request to: {:?}", http_url); + + // Initiate an external HTTP GET request. This is using high-level wrappers from + // `sp_runtime`. + let request = http::Request::get(http_url); + + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + warn!("[DDC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + // Next we fully read the response body and collect it to a vector of bytes. + Ok(response.body().collect::>()) + } + + fn validators() -> Vec<::ValidatorId> { + >::validators() + } + + /// Reward a validator. + pub fn reward_by_ids( + validators_points: impl IntoIterator, + ) -> Result<(), ()> { + >::reward_by_ids(validators_points); + Ok(()) + } +} \ No newline at end of file diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 872ed2e1af404..70387a8ed537f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -381,6 +381,20 @@ impl Default for EraRewardPoints { } } +#[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] +pub struct EraDacRewardPoints { + /// Total number of points. Equals the sum of reward points for each validator. + total: RewardPoint, + /// The reward points earned by a given validator. + individual: BTreeMap, +} + +impl Default for EraDacRewardPoints { + fn default() -> Self { + EraDacRewardPoints { total: Default::default(), individual: BTreeMap::new() } + } +} + /// Indicates the initial status of the staker. #[derive(RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, Clone))] diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 90f19c6badd8f..4e7c3460da2fb 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -16,12 +16,13 @@ // limitations under the License. //! Implementations for the Staking FRAME Pallet. - use frame_election_provider_support::{ data_provider, ElectionDataProvider, ElectionProvider, ScoreProvider, SortedListProvider, Supports, VoteWeight, VoterOf, }; use frame_support::{ + decl_event, decl_module, decl_storage, + log::{error, info, warn}, pallet_prelude::*, traits::{ Currency, CurrencyToVote, Defensive, EstimateNextNewSession, Get, Imbalance, @@ -44,7 +45,7 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, Exposure, ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, - SessionInterface, StakingLedger, ValidatorPrefs, + SessionInterface, StakingLedger, ValidatorPrefs, EraDacRewardPoints, }; use super::{pallet::*, STAKING_ID}; @@ -630,6 +631,40 @@ impl Pallet { era_rewards.total += points; } }); + let erasRewardPoints = >::get(active_era.index); + } + } + + pub fn reward_by_ids2(validators_points: impl IntoIterator) { + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + let mut p = EraDacRewardPoints { + total: era_rewards.total.clone(), + individual: era_rewards.individual.clone(), + }; + for (validator, points) in validators_points.into_iter() { + *p.individual.entry(validator).or_default() += points; + p.total += points; + } + return p; + }); + let dacRewardPoints = >::get(active_era.index); + } + } + + pub fn get_reward_points(validators_points: impl IntoIterator) { + info!("Hello from get_reward_points"); + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + for (validator, points) in validators_points.into_iter() { + // info!("validator: {:?}, points: {:?}", validator, points); + // *era_rewards.individual.entry(validator).or_default() += points; + info!("reading era_rewards.total: {:?}", era_rewards.total); + // era_rewards.total += points; + } + }); + let dacRewardPoints = >::get(active_era.index); + info!("reading dacRewardPoints: {:?}", dacRewardPoints); } } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index fa8c453c2b0fe..85579e59d426a 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -40,10 +40,10 @@ mod impls; pub use impls::*; use crate::{ - slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, EraRewardPoints, Exposure, - Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, Releases, - RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, - ValidatorPrefs, + slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraDacRewardPoints, EraPayout, + EraRewardPoints, Exposure, Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, + PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakingLedger, + UnappliedSlash, UnlockChunk, ValidatorPrefs, }; const STAKING_ID: LockIdentifier = *b"staking "; @@ -382,6 +382,11 @@ pub mod pallet { pub type ErasRewardPoints = StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; + #[pallet::storage] + #[pallet::getter(fn eras_dac_reward_points)] + pub type ErasDacRewardPoints = + StorageMap<_, Twox64Concat, EraIndex, EraDacRewardPoints, ValueQuery>; + /// The total amount staked for the last `HISTORY_DEPTH` eras. /// If total hasn't been set or has been removed then 0 stake is returned. #[pallet::storage] From 065e5338dcbcaab67ae0e6c06550eedaa1bb7345 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 11:54:31 +0600 Subject: [PATCH 007/192] Assign a random DAC validator for each edge --- bin/node/runtime/src/lib.rs | 1 + frame/ddc-validator/src/lib.rs | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a0ec718430091..f54221ba66a95 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1239,6 +1239,7 @@ impl pallet_ddc_staking::Config for Runtime { impl pallet_ddc_validator::Config for Runtime { type Event = Event; + type Randomness = RandomnessCollectiveFlip; } construct_runtime!( diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 77d1d894b2257..bb2dd76411c31 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,6 +1,8 @@ #![cfg_attr(not(feature = "std"), no_std)] -pub use frame_support::{pallet_prelude::*, parameter_types, weights::Weight, BoundedVec}; +pub use frame_support::{ + pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, +}; pub use frame_system::pallet_prelude::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; @@ -34,6 +36,7 @@ pub mod pallet { #[pallet::config] pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { type Event: From> + IsType<::Event>; + type Randomness: Randomness; } #[pallet::storage] @@ -79,15 +82,34 @@ pub mod pallet { impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; let edges: Vec = >::iter_keys().collect(); log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators: {:?}, edges: {:?}", + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, >::get(), >::get(), + validators_count, validators, edges, ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); + } + Tasks::::insert(edge, decisions); + } 0 } fn offchain_worker(block_number: T::BlockNumber) { @@ -109,5 +131,31 @@ pub mod pallet { } cdn_nodes } + + fn choose(total: u32) -> Option { + if total == 0 { + return None + } + let mut random_number = Self::generate_random_number(0); + + // Best effort attempt to remove bias from modulus operator. + for i in 1..128 { + if random_number < u32::MAX - u32::MAX % total { + break + } + + random_number = Self::generate_random_number(i); + } + + Some(random_number % total) + } + + fn generate_random_number(seed: u32) -> u32 { + let (random_seed, _) = T::Randomness::random(&(b"ddc-validator", seed).encode()); + let random_number = ::decode(&mut random_seed.as_ref()) + .expect("secure hashes should always be bigger than u32; qed"); + + random_number + } } } From f9c39b5d00e442cc8f34a1c1e2112ca520d5221c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 12:29:00 +0600 Subject: [PATCH 008/192] Update Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3d241900ddcb1..dc15ef872f506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11705,7 +11705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] From 9834b14fb480d10f5b5b46fafca37067629675a3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 14:27:15 +0600 Subject: [PATCH 009/192] Assign DAC validation tasks on era increment --- frame/ddc-validator/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index bb2dd76411c31..a88d229d435e7 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -81,6 +81,19 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { + match (>::get(), >::get()) { + (Some(global_era_counter), Some(last_managed_era)) => { + if last_managed_era >= global_era_counter { + return 0 + } + >::put(global_era_counter); + }, + (Some(global_era_counter), None) => { + >::put(global_era_counter); + }, + _ => { return 0 }, + }; + let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; let edges: Vec = >::iter_keys().collect(); From 7c0d2f82e08260e620cc02c043ce0251bcc25dd1 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 14:54:08 +0600 Subject: [PATCH 010/192] Format DAC Validator files --- frame/ddc-validator/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a88d229d435e7..344048de95113 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -4,9 +4,9 @@ pub use frame_support::{ pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, }; pub use frame_system::pallet_prelude::*; +pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; -pub use pallet::*; pub use sp_std::prelude::*; parameter_types! { @@ -91,7 +91,7 @@ pub mod pallet { (Some(global_era_counter), None) => { >::put(global_era_counter); }, - _ => { return 0 }, + _ => return 0, }; let validators: Vec = >::iter_keys().collect(); @@ -125,6 +125,7 @@ pub mod pallet { } 0 } + fn offchain_worker(block_number: T::BlockNumber) { log::info!("Off-chain worker at block {:?}", block_number); } @@ -138,7 +139,7 @@ pub mod pallet { for decision in cdn_tasks.iter() { if decision.validator == validator { cdn_nodes.push(cdn_id); - break; + break } } } From 25bb5efdd0f5ed61866a7c22507382c4787b04a6 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Tue, 21 Mar 2023 15:15:39 +0100 Subject: [PATCH 011/192] adjust era --- frame/ddc-validator/src/lib.rs | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 344048de95113..10f72a1ac84ff 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -8,11 +8,16 @@ pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; pub use sp_std::prelude::*; +pub use sp_io::offchain::timestamp; parameter_types! { pub DdcValidatorsQuorumSize: u32 = 3; } +const TIME_START_MS: u64 = 1_672_531_200_000; +const ERA_DURATION_MS: u64 = 120_000; +const ERA_IN_BLOCKS: u8 = 20; + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { ProofOfDelivery, @@ -48,10 +53,6 @@ pub mod pallet { BoundedVec, DdcValidatorsQuorumSize>, >; - #[pallet::storage] - #[pallet::getter(fn global_era_counter)] - pub type GlobalEraCounter = StorageValue<_, u32>; - #[pallet::storage] #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, u32>; @@ -63,21 +64,6 @@ pub mod pallet { #[pallet::error] pub enum Error {} - #[pallet::call] - impl Pallet { - #[pallet::weight(100_000)] - pub fn inc_era(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - if let Some(era) = >::get() { - let new_era = era.checked_add(1).unwrap_or_default(); - >::put(new_era); - } else { - >::put(1); - } - Ok(()) - } - } - #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { @@ -100,7 +86,7 @@ pub mod pallet { log::info!( "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, - >::get(), + Self::get_current_era(), >::get(), validators_count, validators, @@ -170,6 +156,9 @@ pub mod pallet { .expect("secure hashes should always be bigger than u32; qed"); random_number + // Get the current era; Shall we start era count from 0 or from 1? + fn get_current_era() -> u64 { + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS } } } From b1456761dc34e989c530ce9e7557f229b46c2c15 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Tue, 21 Mar 2023 15:40:37 +0100 Subject: [PATCH 012/192] fix era --- frame/ddc-validator/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 10f72a1ac84ff..3d708a0a132c2 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -156,6 +156,8 @@ pub mod pallet { .expect("secure hashes should always be bigger than u32; qed"); random_number + } + // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS From 406b434589d15151137d988ef3f27133e8c7983e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 21 Mar 2023 10:45:34 +0100 Subject: [PATCH 013/192] Prepare for demo --- .../dac-validator-offchain-worker/src/lib.rs | 110 +++++++++++++----- 1 file changed, 83 insertions(+), 27 deletions(-) diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs index 5670f9689dcb7..161a646514e13 100644 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -49,9 +49,20 @@ type ResultStr = Result; #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec), + pub ft_aggregate: (u32, Vec, Vec), + + // This struct should be correct for any length but there is error while parsing JSON + // #[serde(rename = "FT.AGGREGATE")] + // pub ft_aggregate: Vec, } +// #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +// #[serde(crate = "alt_serde")] +// pub enum FtAggregate { +// Length(u32), +// Node(Vec), +// } + #[derive(Clone)] struct BytesSent { node_public_key: String, @@ -61,7 +72,7 @@ struct BytesSent { impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values) = aggregate.ft_aggregate; + let (_, values, values2) = aggregate.ft_aggregate; BytesSent { node_public_key: values[1].clone(), @@ -71,6 +82,24 @@ impl BytesSent { } } +// impl BytesSent { +// pub fn new(aggregate: RedisFtAggregate) -> BytesSent { +// // let (_, values, values2) = aggregate.ft_aggregate; +// let data = aggregate.ft_aggregate[1].clone(); +// +// match data { +// FtAggregate::Node(node) => { +// return BytesSent { +// node_public_key: node[1].clone(), +// era: node[3].clone(), +// sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), +// } +// } +// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), +// } +// } +// } + #[derive(Clone)] struct BytesReceived { node_public_key: String, @@ -80,7 +109,7 @@ struct BytesReceived { impl BytesReceived { pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values) = aggregate.ft_aggregate; + let (_, values, values2) = aggregate.ft_aggregate; BytesReceived { node_public_key: values[1].clone(), @@ -90,9 +119,28 @@ impl BytesReceived { } } +// impl BytesReceived { +// pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { +// // let (_, values, values2) = aggregate.ft_aggregate; +// +// let data = aggregate.ft_aggregate[1].clone(); +// +// match data { +// FtAggregate::Node(node) => { +// return BytesReceived { +// node_public_key: node[1].clone(), +// era: node[3].clone(), +// sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), +// } +// } +// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), +// } +// } +// } + #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - block_number: BlockNumber, +pub struct ValidationResult { + era: String, signer: AccountId, val_res: bool, cdn_node_pub_key: String, @@ -173,8 +221,8 @@ pub mod pallet { let res = Self::offchain_worker_main(block_number); match res { - Ok(()) => info!("[DDC Validator] DDC Validator is suspended."), - Err(err) => error!("[DDC Validator] Error in Offchain Worker: {}", err), + Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), }; } } @@ -185,20 +233,26 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { - #[pallet::weight(1000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, block_number: T::BlockNumber) -> DispatchResult { + #[pallet::weight(10000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; - info!("author: {:?}", signer); + info!("[DAC Validator] author: {:?}", signer); let mut v_results = ValidationResults::::get(); - let cur_validation = ValidationResult:: { - block_number, + let cur_validation = ValidationResult:: { + era, val_res, cdn_node_pub_key, signer, }; + // ValidationResults::::append(cur_validation); + // ValidationResults::::mutate(|results| { + // results.push(cur_validation); + // + // results.clone() + // }); v_results.push(cur_validation); ValidationResults::::set(v_results); @@ -209,7 +263,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; } impl Pallet @@ -218,6 +272,8 @@ where as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { return Ok(()) } @@ -230,19 +286,19 @@ where Ok(signer) => signer, }; - info!("ValidationResults: {:?}", ValidationResults::::get()); + info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era(); + let current_era = Self::get_current_era() - 1u64; let (bytes_sent, bytes_received) = Self::fetch_data(current_era); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let cdn_node_pub_key = bytes_sent.node_public_key.clone(); let tx_res = signer.send_signed_transaction(|_acct| { - info!("Sending save_validated_data tx"); + info!("[DAC Validator] Sending save_validated_data tx"); // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), block_number } + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } }); match &tx_res { @@ -252,8 +308,7 @@ where Some((_, Ok(()))) => {} } - info!("save_validated_data: {:?}", ValidationResults::::get()); - + // info!("[DAC Validator] save_validated_data: {:?}", ValidationResults::::get()); Ok(()) } @@ -261,8 +316,9 @@ where fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { - return Err("[DDC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); } + Ok(signer) } @@ -275,18 +331,18 @@ where } fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ - info!("[DDC Validator] DDC Validator is running. Current era is {}", era); + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("Bytes sent sum is fetched: {:?}", bytes_sent_res); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error let bytes_received_query = Self::get_bytes_received_query_url(era); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); - info!("Bytes received sum is fetched:: {:?}", bytes_received_res); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); (bytes_sent, bytes_received) @@ -312,12 +368,12 @@ where fn http_get_json(url: &str) -> ResultStr { let body = Self::http_get_request(url).map_err(|err| { - error!("[DDC Validator] Error while getting {}: {:?}", url, err); + error!("[DAC Validator] Error while getting {}: {:?}", url, err); "HTTP GET error" })?; let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DDC Validator] Error while parsing JSON from {}: {:?}", url, err); + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); "HTTP JSON parse error" }); @@ -325,7 +381,7 @@ where } fn http_get_request(http_url: &str) -> Result, http::Error> { - info!("[DDC Validator] Sending request to: {:?}", http_url); + info!("[DAC Validator] Sending request to: {:?}", http_url); // Initiate an external HTTP GET request. This is using high-level wrappers from // `sp_runtime`. @@ -338,7 +394,7 @@ where let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; if response.code != 200 { - warn!("[DDC Validator] http_get_request unexpected status code: {}", response.code); + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); return Err(http::Error::Unknown) } From af805441738b45ebbdc5dce440dab5f9e8a596e4 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 11:32:21 +0100 Subject: [PATCH 014/192] merge offchain worker with ddc validator --- Cargo.lock | 7 + bin/node/runtime/src/lib.rs | 2 + frame/ddc-validator/Cargo.toml | 12 ++ frame/ddc-validator/src/lib.rs | 322 +++++++++++++++++++++++++++++++-- 4 files changed, 326 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc15ef872f506..6359b04d3e23c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5878,14 +5878,21 @@ dependencies = [ name = "pallet-ddc-validator" version = "0.1.0" dependencies = [ + "alt_serde", "frame-support", "frame-system", "log 0.4.17", + "pallet-contracts", "pallet-ddc-staking", + "pallet-session", "pallet-staking", "parity-scale-codec", "scale-info", + "serde", + "serde_json 1.0.44", + "sp-core", "sp-io", + "sp-keystore", "sp-runtime", "sp-staking", "sp-std", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index f54221ba66a95..3758337a84c1e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1240,6 +1240,8 @@ impl pallet_ddc_staking::Config for Runtime { impl pallet_ddc_validator::Config for Runtime { type Event = Event; type Randomness = RandomnessCollectiveFlip; + type Call = Call; + type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; } construct_runtime!( diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 057a274a2ec1b..55172f6848949 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -4,13 +4,20 @@ version = "0.1.0" edition = "2021" [dependencies] +alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } log = { version = "0.4.17", default-features = false } +pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } +pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.101", optional = true } +serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } @@ -22,9 +29,14 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", + "pallet-contracts/std", "pallet-ddc-staking/std", "pallet-staking/std", + "pallet-session/std", "scale-info/std", + "serde", + "sp-keystore", + "sp-core/std", "sp-io/std", "sp-runtime/std", "sp-staking/std", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 3d708a0a132c2..9cab6656ca750 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,23 +1,53 @@ #![cfg_attr(not(feature = "std"), no_std)] +pub use alloc::{format, string::String}; +pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +pub use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +pub use core::fmt::Debug; pub use frame_support::{ - pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, + decl_event, decl_module, decl_storage, + log::{error, info, warn}, + pallet_prelude::*, + traits::{Randomness, Currency}, + weights::Weight, + dispatch::DispatchResult, + RuntimeDebug, + BoundedVec, + parameter_types, }; -pub use frame_system::pallet_prelude::*; +pub use frame_system::{ensure_signed, pallet_prelude::*, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; +pub use pallet_session as session; +pub use scale_info::TypeInfo; +pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; +pub use sp_runtime::offchain::{http, Duration}; pub use sp_std::prelude::*; pub use sp_io::offchain::timestamp; +extern crate alloc; parameter_types! { pub DdcValidatorsQuorumSize: u32 = 3; } +type BalanceOf = <::Currency as Currency< + ::AccountId, +>>::Balance; + +type ResultStr = Result; + + +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); + +pub const HTTP_TIMEOUT_MS: u64 = 30_000; + const TIME_START_MS: u64 = 1_672_531_200_000; const ERA_DURATION_MS: u64 = 120_000; const ERA_IN_BLOCKS: u8 = 20; +const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { ProofOfDelivery, @@ -30,18 +60,112 @@ pub struct Decision { pub validator: AccountId, } +#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] +pub struct ValidationResult { + era: String, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct RedisFtAggregate { + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: (u32, Vec, Vec), +} + +#[derive(Clone)] +struct BytesSent { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesSent { + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let (_, values, values2) = aggregate.ft_aggregate; + + BytesSent { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } +} + +#[derive(Clone)] +struct BytesReceived { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesReceived { + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let (_, values, values2) = aggregate.ft_aggregate; + + BytesReceived { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } +} + +pub mod crypto { + use super::KEY_TYPE; + use frame_system::offchain::AppCrypto; + use sp_core::sr25519::Signature as Sr25519Signature; + use sp_runtime::{ + app_crypto::{app_crypto, sr25519}, + traits::Verify, + }; + app_crypto!(sr25519, KEY_TYPE); + + use sp_runtime::{MultiSignature, MultiSigner}; + + pub struct TestAuthId; + + impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } + + impl AppCrypto for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } +} + #[frame_support::pallet] pub mod pallet { use super::*; #[pallet::pallet] + #[pallet::without_storage_info] #[pallet::generate_store(pub(super) trait Store)] pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { + pub trait Config: + frame_system::Config + + pallet_contracts::Config + + pallet_session::Config::AccountId> + + pallet_staking::Config + + ddc_staking::Config + + CreateSignedTransaction> + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { type Event: From> + IsType<::Event>; type Randomness: Randomness; + type Call: From>; + type AuthorityId: AppCrypto; } #[pallet::storage] @@ -55,26 +179,34 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn last_managed_era)] - pub type LastManagedEra = StorageValue<_, u32>; + pub type LastManagedEra = StorageValue<_, u64>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event {} + pub enum Event + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + {} #[pallet::error] pub enum Error {} #[pallet::hooks] - impl Hooks> for Pallet { + impl Hooks> for Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { fn on_initialize(block_number: T::BlockNumber) -> Weight { - match (>::get(), >::get()) { - (Some(global_era_counter), Some(last_managed_era)) => { + match (Self::get_current_era(), >::get()) { + (global_era_counter, Some(last_managed_era)) => { if last_managed_era >= global_era_counter { return 0 } >::put(global_era_counter); }, - (Some(global_era_counter), None) => { + (global_era_counter, None) => { >::put(global_era_counter); }, _ => return 0, @@ -113,11 +245,172 @@ pub mod pallet { } fn offchain_worker(block_number: T::BlockNumber) { - log::info!("Off-chain worker at block {:?}", block_number); + let res = Self::offchain_worker_main(block_number); + + match res { + Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), + }; } } - impl Pallet { + #[pallet::call] + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + #[pallet::weight(10000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + info!("[DAC Validator] author: {:?}", signer); + let mut v_results = ValidationResults::::get(); + + let cur_validation = ValidationResult:: { + era, + val_res, + cdn_node_pub_key, + signer, + }; + + v_results.push(cur_validation); + + ValidationResults::::set(v_results); + + Ok(()) + } + } + + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { + return Ok(()) + } + + let signer = match Self::get_signer() { + Err(e) => { + warn!("{:?}", e); + return Ok(()); + } + Ok(signer) => signer, + }; + + info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); + + // Read data from DataModel and do dumb validation + let current_era = Self::get_current_era() - 1u64; + let (bytes_sent, bytes_received) = Self::fetch_data(current_era); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let cdn_node_pub_key = bytes_sent.node_public_key.clone(); + let tx_res = signer.send_signed_transaction(|_acct| { + info!("[DAC Validator] Sending save_validated_data tx"); + + // This is the on-chain function + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } + }); + + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting save_validated_data TX") + } + Some((_, Ok(()))) => {} + } + + Ok(()) + } + + fn get_signer() -> ResultStr> { + let signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + } + + Ok(signer) + } + + // Get the current era; Shall we start era count from 0 or from 1? + fn get_current_era() -> u64 { + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + } + + fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::new(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::new(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + } + + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + } + + fn http_get_json(url: &str) -> ResultStr { + let body = Self::http_get_request(url).map_err(|err| { + error!("[DAC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" + })?; + + let parsed = serde_json::from_slice(&body).map_err(|err| { + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" + }); + + parsed + } + + fn http_get_request(http_url: &str) -> Result, http::Error> { + info!("[DAC Validator] Sending request to: {:?}", http_url); + + // Initiate an external HTTP GET request. This is using high-level wrappers from + // `sp_runtime`. + let request = http::Request::get(http_url); + + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + // Next we fully read the response body and collect it to a vector of bytes. + Ok(response.body().collect::>()) + } + + fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { + return if bytes_sent.sum == bytes_received.sum { + true + } else { + false + } + } + /// Fetch the tasks related to current validator fn fetch_tasks(validator: T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; @@ -151,16 +444,11 @@ pub mod pallet { } fn generate_random_number(seed: u32) -> u32 { - let (random_seed, _) = T::Randomness::random(&(b"ddc-validator", seed).encode()); + let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); let random_number = ::decode(&mut random_seed.as_ref()) .expect("secure hashes should always be bigger than u32; qed"); random_number } - - // Get the current era; Shall we start era count from 0 or from 1? - fn get_current_era() -> u64 { - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS - } } } From f640a5b3375348daea2b81e828869f3139963010 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 14:43:26 +0100 Subject: [PATCH 015/192] add proof of delivery --- frame/ddc-validator/src/lib.rs | 75 +++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 9cab6656ca750..52219d6dc0481 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -181,6 +181,10 @@ pub mod pallet { #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, u64>; + #[pallet::storage] + #[pallet::getter(fn validation_results)] + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event @@ -209,7 +213,6 @@ pub mod pallet { (global_era_counter, None) => { >::put(global_era_counter); }, - _ => return 0, }; let validators: Vec = >::iter_keys().collect(); @@ -280,6 +283,27 @@ pub mod pallet { Ok(()) } + + #[pallet::weight(10000)] + pub fn proof_of_delivery(origin: OriginFor, era: u64) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + for cdn_node_id in cdn_nodes_to_validate { + let (bytes_sent, bytes_received) = Self::fetch_data(era, &cdn_node_id); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let decisions_for_cdn = >::get(cdn_node_id); + for decision in decisions_for_cdn.unwrap().iter_mut() { + if decision.validator == signer { + decision.decision = Some(val_res); + decision.method = ValidationMethodKind::ProofOfDelivery; + } + } + } + + Ok(()) + } } impl Pallet @@ -306,23 +330,26 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - let (bytes_sent, bytes_received) = Self::fetch_data(current_era); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Sending save_validated_data tx"); + // for decision in &mut cdn_nodes_to_validate { - // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - }); + // } + // let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + // let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting save_validated_data TX") - } - Some((_, Ok(()))) => {} - } + // let tx_res = signer.send_signed_transaction(|_acct| { + // info!("[DAC Validator] Sending save_validated_data tx"); + + // // This is the on-chain function + // Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } + // }); + + // match &tx_res { + // None | Some((_, Err(()))) => { + // return Err("Error while submitting save_validated_data TX") + // } + // Some((_, Ok(()))) => {} + // } Ok(()) } @@ -341,16 +368,16 @@ pub mod pallet { (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS } - fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_query = Self::get_bytes_sent_query_url(era, cdn_node); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); @@ -359,12 +386,12 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + fn get_bytes_sent_query_url(era: u64, cdn_node: &T::AccountId) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era, *cdn_node) } - fn get_bytes_received_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + fn get_bytes_received_query_url(era: u64, cdn_node: &T::AccountId) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era, *cdn_node) } fn http_get_json(url: &str) -> ResultStr { @@ -412,11 +439,11 @@ pub mod pallet { } /// Fetch the tasks related to current validator - fn fetch_tasks(validator: T::AccountId) -> Vec { + fn fetch_tasks(validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; for (cdn_id, cdn_tasks) in >::iter() { for decision in cdn_tasks.iter() { - if decision.validator == validator { + if decision.validator == *validator { cdn_nodes.push(cdn_id); break } From 041fbe971ddf67b27ac7a47bb1546143ab02c066 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 14:45:48 +0100 Subject: [PATCH 016/192] add PoD trigger in offchain worker --- frame/ddc-validator/src/lib.rs | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 52219d6dc0481..d599567bef4aa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -331,25 +331,20 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - // for decision in &mut cdn_nodes_to_validate { + + let tx_res = signer.send_signed_transaction(|_acct| { + info!("[DAC Validator] Trigger proof of delivery"); - // } - // let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - // let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - - // let tx_res = signer.send_signed_transaction(|_acct| { - // info!("[DAC Validator] Sending save_validated_data tx"); - - // // This is the on-chain function - // Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - // }); + // This is the on-chain function + Call::proof_of_delivery { era: current_era } + }); - // match &tx_res { - // None | Some((_, Err(()))) => { - // return Err("Error while submitting save_validated_data TX") - // } - // Some((_, Ok(()))) => {} - // } + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting proof of delivery TX") + } + Some((_, Ok(()))) => {} + } Ok(()) } From 9a797bd9a4db33dbe998a3513617d3fbeca65791 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 22 Mar 2023 19:15:46 +0100 Subject: [PATCH 017/192] Fix json parse error for cases when in the response not 2 cdn nodes --- .../dac-validator-offchain-worker/src/lib.rs | 90 +++++++------------ 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs index 161a646514e13..6be8983dc768b 100644 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -49,19 +49,16 @@ type ResultStr = Result; #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec, Vec), - - // This struct should be correct for any length but there is error while parsing JSON - // #[serde(rename = "FT.AGGREGATE")] - // pub ft_aggregate: Vec, + pub ft_aggregate: Vec, } -// #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -// #[serde(crate = "alt_serde")] -// pub enum FtAggregate { -// Length(u32), -// Node(Vec), -// } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(untagged)] +pub enum FtAggregate { + Length(u32), + Node(Vec), +} #[derive(Clone)] struct BytesSent { @@ -72,34 +69,21 @@ struct BytesSent { impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesSent { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } } -// impl BytesSent { -// pub fn new(aggregate: RedisFtAggregate) -> BytesSent { -// // let (_, values, values2) = aggregate.ft_aggregate; -// let data = aggregate.ft_aggregate[1].clone(); -// -// match data { -// FtAggregate::Node(node) => { -// return BytesSent { -// node_public_key: node[1].clone(), -// era: node[3].clone(), -// sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), -// } -// } -// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), -// } -// } -// } - #[derive(Clone)] struct BytesReceived { node_public_key: String, @@ -109,35 +93,21 @@ struct BytesReceived { impl BytesReceived { pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesReceived { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } } -// impl BytesReceived { -// pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { -// // let (_, values, values2) = aggregate.ft_aggregate; -// -// let data = aggregate.ft_aggregate[1].clone(); -// -// match data { -// FtAggregate::Node(node) => { -// return BytesReceived { -// node_public_key: node[1].clone(), -// era: node[3].clone(), -// sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), -// } -// } -// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), -// } -// } -// } - #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { era: String, From aafe664fcbf03b7640660370b5be2693a4dbfdf6 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 23 Mar 2023 11:47:35 +0100 Subject: [PATCH 018/192] Move JSON parsing fix to ddc-validator --- frame/ddc-validator/src/lib.rs | 58 ++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index d599567bef4aa..2b1d5245eb286 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -72,8 +72,16 @@ pub struct ValidationResult { #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { - #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec, Vec), + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(untagged)] +pub enum FtAggregate { + Length(u32), + Node(Vec), } #[derive(Clone)] @@ -84,15 +92,20 @@ struct BytesSent { } impl BytesSent { - pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesSent { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } } #[derive(Clone)] @@ -103,15 +116,20 @@ struct BytesReceived { } impl BytesReceived { - pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesReceived { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } } pub mod crypto { From 332fd3053862c66c4e9f8e4afda071aacc4920c9 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Thu, 23 Mar 2023 15:23:09 +0100 Subject: [PATCH 019/192] fix timestamp bug --- bin/node/runtime/src/lib.rs | 1 + frame/ddc-validator/src/lib.rs | 96 ++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3758337a84c1e..6b84ccba26b1e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1242,6 +1242,7 @@ impl pallet_ddc_validator::Config for Runtime { type Randomness = RandomnessCollectiveFlip; type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; + type TimeProvider = pallet_timestamp::Pallet; } construct_runtime!( diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 2b1d5245eb286..e7481fe6ed9bc 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -8,7 +8,7 @@ pub use frame_support::{ decl_event, decl_module, decl_storage, log::{error, info, warn}, pallet_prelude::*, - traits::{Randomness, Currency}, + traits::{Randomness, Currency, UnixTime}, weights::Weight, dispatch::DispatchResult, RuntimeDebug, @@ -22,9 +22,9 @@ pub use pallet_staking::{self as staking}; pub use pallet_session as session; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; -pub use sp_runtime::offchain::{http, Duration}; +pub use sp_runtime::offchain::{http, Duration, Timestamp}; pub use sp_std::prelude::*; -pub use sp_io::offchain::timestamp; + extern crate alloc; parameter_types! { @@ -42,8 +42,8 @@ pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); pub const HTTP_TIMEOUT_MS: u64 = 30_000; -const TIME_START_MS: u64 = 1_672_531_200_000; -const ERA_DURATION_MS: u64 = 120_000; +const TIME_START_MS: u128 = 1_672_531_200_000; +const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; @@ -184,6 +184,7 @@ pub mod pallet { type Randomness: Randomness; type Call: From>; type AuthorityId: AppCrypto; + type TimeProvider: UnixTime; } #[pallet::storage] @@ -221,48 +222,53 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - match (Self::get_current_era(), >::get()) { - (global_era_counter, Some(last_managed_era)) => { - if last_managed_era >= global_era_counter { - return 0 + if block_number != 0u32.into() && block_number != 1u32.into() { + let era = Self::get_current_era(); + match (era, >::get()) { + (global_era_counter, Some(last_managed_era)) => { + if last_managed_era >= global_era_counter { + return 0 + } + >::put(global_era_counter); + }, + (global_era_counter, None) => { + >::put(global_era_counter); + }, + }; + + let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", + block_number, + era, + >::get(), + validators_count, + validators, + edges, + ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); } - >::put(global_era_counter); - }, - (global_era_counter, None) => { - >::put(global_era_counter); - }, - }; - - let validators: Vec = >::iter_keys().collect(); - let validators_count = validators.len() as u32; - let edges: Vec = >::iter_keys().collect(); - log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", - block_number, - Self::get_current_era(), - >::get(), - validators_count, - validators, - edges, - ); - - // A naive approach assigns random validators for each edge. - for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = - Default::default(); - while !decisions.is_full() { - let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - let validator: T::AccountId = validators[validator_idx].clone(); - let assignment = Decision { - validator, - method: ValidationMethodKind::ProofOfDelivery, - decision: None, - }; - decisions.try_push(assignment).unwrap(); + Tasks::::insert(edge, decisions); } - Tasks::::insert(edge, decisions); + 0 + } else { + 0 } - 0 } fn offchain_worker(block_number: T::BlockNumber) { @@ -378,7 +384,7 @@ pub mod pallet { // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS).try_into().unwrap() } fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { From 66a17f4b4cd452f3110e3276795be31a88347e45 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 24 Mar 2023 11:17:31 +0600 Subject: [PATCH 020/192] Autoformat DAC Validator files --- frame/ddc-validator/Cargo.toml | 8 +- frame/ddc-validator/src/lib.rs | 205 +++++++++++++++++---------------- 2 files changed, 109 insertions(+), 104 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 55172f6848949..af7a0108ea9e0 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -11,14 +11,14 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../sys log = { version = "0.4.17", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } -sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } @@ -31,13 +31,13 @@ std = [ "frame-system/std", "pallet-contracts/std", "pallet-ddc-staking/std", - "pallet-staking/std", "pallet-session/std", + "pallet-staking/std", "scale-info/std", "serde", - "sp-keystore", "sp-core/std", "sp-io/std", + "sp-keystore", "sp-runtime/std", "sp-staking/std", "sp-std/std", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index e7481fe6ed9bc..a2e6586a88120 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -2,24 +2,27 @@ pub use alloc::{format, string::String}; pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; -pub use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +pub use codec::{Decode, Encode, HasCompact, MaxEncodedLen}; pub use core::fmt::Debug; pub use frame_support::{ decl_event, decl_module, decl_storage, - log::{error, info, warn}, - pallet_prelude::*, - traits::{Randomness, Currency, UnixTime}, - weights::Weight, dispatch::DispatchResult, - RuntimeDebug, - BoundedVec, + log::{error, info, warn}, + pallet_prelude::*, parameter_types, + traits::{Currency, Randomness, UnixTime}, + weights::Weight, + BoundedVec, RuntimeDebug, +}; +pub use frame_system::{ + ensure_signed, + offchain::{AppCrypto, CreateSignedTransaction, SendSignedTransaction, Signer, SigningTypes}, + pallet_prelude::*, }; -pub use frame_system::{ensure_signed, pallet_prelude::*, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; -pub use pallet_staking::{self as staking}; pub use pallet_session as session; +pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_runtime::offchain::{http, Duration, Timestamp}; @@ -32,12 +35,11 @@ parameter_types! { } type BalanceOf = <::Currency as Currency< - ::AccountId, + ::AccountId, >>::Balance; type ResultStr = Result; - pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); pub const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -62,10 +64,10 @@ pub struct Decision { #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { - era: String, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, + era: String, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -86,9 +88,9 @@ pub enum FtAggregate { #[derive(Clone)] struct BytesSent { - node_public_key: String, - era: String, - sum: u32, + node_public_key: String, + era: String, + sum: u32, } impl BytesSent { @@ -96,13 +98,12 @@ impl BytesSent { let data = aggregate.ft_aggregate[1].clone(); match data { - FtAggregate::Node(node) => { + FtAggregate::Node(node) => return BytesSent { node_public_key: node[1].clone(), era: node[3].clone(), sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } @@ -110,9 +111,9 @@ impl BytesSent { #[derive(Clone)] struct BytesReceived { - node_public_key: String, - era: String, - sum: u32, + node_public_key: String, + era: String, + sum: u32, } impl BytesReceived { @@ -120,13 +121,14 @@ impl BytesReceived { let data = aggregate.ft_aggregate[1].clone(); match data { - FtAggregate::Node(node) => { + FtAggregate::Node(node) => return BytesReceived { node_public_key: node[1].clone(), era: node[3].clone(), - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertable to u32"), + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } @@ -137,8 +139,8 @@ pub mod crypto { use frame_system::offchain::AppCrypto; use sp_core::sr25519::Signature as Sr25519Signature; use sp_runtime::{ - app_crypto::{app_crypto, sr25519}, - traits::Verify, + app_crypto::{app_crypto, sr25519}, + traits::Verify, }; app_crypto!(sr25519, KEY_TYPE); @@ -147,15 +149,15 @@ pub mod crypto { pub struct TestAuthId; impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; } impl AppCrypto for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; } } @@ -169,21 +171,21 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: + pub trait Config: frame_system::Config + pallet_contracts::Config - + pallet_session::Config::AccountId> + + pallet_session::Config::AccountId> + pallet_staking::Config + ddc_staking::Config + CreateSignedTransaction> - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { type Event: From> + IsType<::Event>; type Randomness: Randomness; type Call: From>; - type AuthorityId: AppCrypto; + type AuthorityId: AppCrypto; type TimeProvider: UnixTime; } @@ -202,28 +204,28 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + pub(super) type ValidationResults = + StorageValue<_, Vec>, ValueQuery>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event + pub enum Event where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - {} + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} #[pallet::error] pub enum Error {} #[pallet::hooks] - impl Hooks> for Pallet + impl Hooks> for Pallet where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { if block_number != 0u32.into() && block_number != 1u32.into() { - let era = Self::get_current_era(); + let era = Self::get_current_era(); match (era, >::get()) { (global_era_counter, Some(last_managed_era)) => { if last_managed_era >= global_era_counter { @@ -235,10 +237,11 @@ pub mod pallet { >::put(global_era_counter); }, }; - + let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; - let edges: Vec = >::iter_keys().collect(); + let edges: Vec = + >::iter_keys().collect(); log::info!( "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, @@ -248,7 +251,7 @@ pub mod pallet { validators, edges, ); - + // A naive approach assigns random validators for each edge. for edge in edges { let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = @@ -282,24 +285,25 @@ pub mod pallet { } #[pallet::call] - impl Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { #[pallet::weight(10000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { + pub fn save_validated_data( + origin: OriginFor, + val_res: bool, + cdn_node_pub_key: String, + era: String, + ) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; info!("[DAC Validator] author: {:?}", signer); let mut v_results = ValidationResults::::get(); - let cur_validation = ValidationResult:: { - era, - val_res, - cdn_node_pub_key, - signer, - }; + let cur_validation = + ValidationResult:: { era, val_res, cdn_node_pub_key, signer }; v_results.push(cur_validation); @@ -330,24 +334,27 @@ pub mod pallet { } } - impl Pallet + impl Pallet where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + info!( + "[DAC Validator] Validation data stored onchain: {:?}", + ValidationResults::::get() + ); if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) + return Ok(()) } let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()); - } - Ok(signer) => signer, + Err(e) => { + warn!("{:?}", e); + return Ok(()) + }, + Ok(signer) => signer, }; info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); @@ -355,19 +362,17 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Trigger proof of delivery"); + info!("[DAC Validator] Trigger proof of delivery"); - // This is the on-chain function - Call::proof_of_delivery { era: current_era } + // This is the on-chain function + Call::proof_of_delivery { era: current_era } }); match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting proof of delivery TX") - } - Some((_, Ok(()))) => {} + None | Some((_, Err(()))) => + return Err("Error while submitting proof of delivery TX"), + Some((_, Ok(()))) => {}, } Ok(()) @@ -376,7 +381,7 @@ pub mod pallet { fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { - return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); } Ok(signer) @@ -384,7 +389,9 @@ pub mod pallet { // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { - ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS).try_into().unwrap() + ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) + .try_into() + .unwrap() } fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { @@ -398,7 +405,7 @@ pub mod pallet { // Todo: handle the error let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); + Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); @@ -415,13 +422,13 @@ pub mod pallet { fn http_get_json(url: &str) -> ResultStr { let body = Self::http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); - "HTTP GET error" + error!("[DAC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" })?; let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); - "HTTP JSON parse error" + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" }); parsed @@ -438,23 +445,20 @@ pub mod pallet { let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; - let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + let response = + pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); - return Err(http::Error::Unknown) + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) } // Next we fully read the response body and collect it to a vector of bytes. Ok(response.body().collect::>()) - } + } fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { - true - } else { - false - } + return if bytes_sent.sum == bytes_received.sum { true } else { false } } /// Fetch the tasks related to current validator @@ -490,7 +494,8 @@ pub mod pallet { } fn generate_random_number(seed: u32) -> u32 { - let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); + let (random_seed, _) = + ::Randomness::random(&(b"ddc-validator", seed).encode()); let random_number = ::decode(&mut random_seed.as_ref()) .expect("secure hashes should always be bigger than u32; qed"); From 9dd6178ee87c9cca2215bf4233248b8cd4c68eff Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 24 Mar 2023 12:02:07 +0100 Subject: [PATCH 021/192] add filtering of requests --- frame/ddc-validator/Cargo.toml | 1 + frame/ddc-validator/src/lib.rs | 91 +++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index af7a0108ea9e0..0e0fe0ea95000 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +array-bytes = "6.0.0" alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a2e6586a88120..5a553ca1dde7a 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -107,6 +107,27 @@ impl BytesSent { FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec!(); + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + }; + + res.push(node); + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res; + } } #[derive(Clone)] @@ -132,6 +153,27 @@ impl BytesReceived { FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec!(); + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + }; + + res.push(node); + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res; + } } pub mod crypto { @@ -317,8 +359,9 @@ pub mod pallet { let signer: T::AccountId = ensure_signed(origin)?; let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + let (s, r) = Self::fetch_data1(era); for cdn_node_id in cdn_nodes_to_validate { - let (bytes_sent, bytes_received) = Self::fetch_data(era, &cdn_node_id); + let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let decisions_for_cdn = >::get(cdn_node_id); @@ -397,13 +440,13 @@ pub mod pallet { fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era, cdn_node); + let bytes_sent_query = Self::get_bytes_sent_query_url(era); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); + let bytes_received_query = Self::get_bytes_received_query_url(era); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); @@ -412,12 +455,46 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64, cdn_node: &T::AccountId) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era, *cdn_node) + fn account_to_string(account: T::AccountId) -> String { + let to32 = T::AccountId::encode(&account); + let pub_key_str = array_bytes::bytes2hex("", to32); + + pub_key_str + } + + fn filter_data(s: &Vec, r: &Vec, a: &T::AccountId) -> (BytesSent, BytesReceived){ + let ac = Self::account_to_string(a.clone()); + + let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); + let filtered_r = &*r.into_iter().find(|br| br.node_public_key == ac).unwrap(); + + (filtered_s.clone(), filtered_r.clone()) + } + + fn fetch_data1(era: u64 ) -> (Vec, Vec){ + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::get_all(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::get_all(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) } - fn get_bytes_received_query_url(era: u64, cdn_node: &T::AccountId) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era, *cdn_node) + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) } fn http_get_json(url: &str) -> ResultStr { From 507b6b9ac8ffdc7dc08f85d7d008ad20afe9e024 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 14:38:27 +0600 Subject: [PATCH 022/192] Refactor guard condition --- frame/ddc-validator/src/lib.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 5a553ca1dde7a..dc5b4688cdb6d 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -268,18 +268,12 @@ pub mod pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { if block_number != 0u32.into() && block_number != 1u32.into() { let era = Self::get_current_era(); - match (era, >::get()) { - (global_era_counter, Some(last_managed_era)) => { - if last_managed_era >= global_era_counter { - return 0 - } - >::put(global_era_counter); - }, - (global_era_counter, None) => { - >::put(global_era_counter); - }, - }; - + if let Some(last_managed_era) = >::get() { + if last_managed_era >= era { + return 0 + } + } + >::put(era); let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; let edges: Vec = From 1a77df7982e7b8119bc53430416cff0aad284440 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 15:30:57 +0600 Subject: [PATCH 023/192] Refactor a guard condition to decrease indentation --- frame/ddc-validator/src/lib.rs | 79 +++++++++++++++++----------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index dc5b4688cdb6d..cf4423173bc88 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -266,48 +266,49 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - if block_number != 0u32.into() && block_number != 1u32.into() { - let era = Self::get_current_era(); - if let Some(last_managed_era) = >::get() { - if last_managed_era >= era { - return 0 - } + if block_number < 1u32.into() { + return 0 + } + + let era = Self::get_current_era(); + if let Some(last_managed_era) = >::get() { + if last_managed_era >= era { + return 0 } - >::put(era); - let validators: Vec = >::iter_keys().collect(); - let validators_count = validators.len() as u32; - let edges: Vec = - >::iter_keys().collect(); - log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", - block_number, - era, - >::get(), - validators_count, - validators, - edges, - ); - - // A naive approach assigns random validators for each edge. - for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = - Default::default(); - while !decisions.is_full() { - let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - let validator: T::AccountId = validators[validator_idx].clone(); - let assignment = Decision { - validator, - method: ValidationMethodKind::ProofOfDelivery, - decision: None, - }; - decisions.try_push(assignment).unwrap(); - } - Tasks::::insert(edge, decisions); + } + >::put(era); + + let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", + block_number, + era, + >::get(), + validators_count, + validators, + edges, + ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); } - 0 - } else { - 0 + Tasks::::insert(edge, decisions); } + + 0 } fn offchain_worker(block_number: T::BlockNumber) { From 1b1c13debcaa806fce09c23ae7653b3121335ce7 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 27 Mar 2023 16:05:56 +0200 Subject: [PATCH 024/192] Removed poc pallet --- Cargo.lock | 36 +- bin/node/runtime/Cargo.toml | 1 - bin/node/runtime/src/lib.rs | 8 - .../dac-validator-offchain-worker/Cargo.toml | 58 --- .../dac-validator-offchain-worker/src/lib.rs | 386 ------------------ 5 files changed, 7 insertions(+), 482 deletions(-) delete mode 100644 frame/dac-validator-offchain-worker/Cargo.toml delete mode 100644 frame/dac-validator-offchain-worker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6359b04d3e23c..b2869c996bec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,6 +141,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "698b65a961a9d730fb45b6b0327e20207810c9f61ee421b082b27ba003f49e2b" +[[package]] +name = "array-bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f72e9d6fac4bc80778ea470b20197b88d28c292bb7d60c3fb099280003cd19" + [[package]] name = "arrayref" version = "0.3.6" @@ -5047,7 +5053,6 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", - "pallet-dac-validator-offchain-worker", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", "pallet-ddc-validator", @@ -5804,34 +5809,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "pallet-dac-validator-offchain-worker" -version = "0.0.1" -dependencies = [ - "alt_serde", - "frame-support", - "frame-system", - "hex", - "hex-literal", - "lite-json", - "pallet-balances", - "pallet-contracts", - "pallet-randomness-collective-flip", - "pallet-session", - "pallet-staking", - "pallet-timestamp", - "parity-scale-codec", - "pretty_assertions 0.6.1", - "scale-info", - "serde", - "serde_json 1.0.44", - "sp-core", - "sp-io", - "sp-keystore", - "sp-runtime", - "sp-std", -] - [[package]] name = "pallet-ddc-metrics-offchain-worker" version = "2.0.1" @@ -5879,6 +5856,7 @@ name = "pallet-ddc-validator" version = "0.1.0" dependencies = [ "alt_serde", + "array-bytes", "frame-support", "frame-system", "log 0.4.17", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index fd8b311a7a709..ed563f445eb15 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -97,7 +97,6 @@ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../f pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } -pallet-dac-validator-offchain-worker = { version = "0.0.1", default-features = false, path = "../../../frame/dac-validator-offchain-worker" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 6b84ccba26b1e..d167d2ee904c8 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -82,7 +82,6 @@ pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; pub use pallet_ddc_validator; -pub use pallet_dac_validator_offchain_worker; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1215,12 +1214,6 @@ parameter_types! { pub const OcwBlockInterval: u32 = pallet_ddc_metrics_offchain_worker::BLOCK_INTERVAL; } -impl pallet_dac_validator_offchain_worker::Config for Runtime { - type Call = Call; - - type AuthorityId = pallet_dac_validator_offchain_worker::crypto::TestAuthId; -} - impl pallet_ddc_metrics_offchain_worker::Config for Runtime { type BlockInterval = OcwBlockInterval; @@ -1294,7 +1287,6 @@ construct_runtime!( Erc721: pallet_erc721::{Pallet, Call, Storage, Event}, Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, - DdcValidatorOffchainWorker: pallet_dac_validator_offchain_worker::{Pallet, Call, Storage}, DdcStaking: pallet_ddc_staking, DdcValidator: pallet_ddc_validator, } diff --git a/frame/dac-validator-offchain-worker/Cargo.toml b/frame/dac-validator-offchain-worker/Cargo.toml deleted file mode 100644 index 7f978de7fe981..0000000000000 --- a/frame/dac-validator-offchain-worker/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -[package] -name = "pallet-dac-validator-offchain-worker" -version = "0.0.1" -authors = ["Parity Technologies "] -edition = "2021" -license = "Unlicense" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "FRAME example pallet for offchain worker" -readme = "README.md" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["full"] } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -serde = { version = "1.0.101", optional = true } -sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } -sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } -sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } -lite-json = { version = "0.2.0", default-features = false } -alt_serde = { version = "1", default-features = false, features = ["derive"] } -serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } -hex-literal = "^0.3.1" -hex = { version = "0.4", default-features = false } -scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } -pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } - -[features] -default = ["std"] -std = [ - "codec/std", - "sp-keystore", - "frame-support/std", - "frame-system/std", - "serde", - "lite-json/std", - "sp-core/std", - "sp-io/std", - "sp-runtime/std", - "sp-std/std", - "pallet-contracts/std", - "pallet-staking/std", - "pallet-session/std", - # "pallet-contracts-rpc-runtime-api/std", -] - -[dev-dependencies] -pallet-balances = { version = "4.0.0-dev", path = "../balances" } -pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } -pretty_assertions = "0.6.1" \ No newline at end of file diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs deleted file mode 100644 index 6be8983dc768b..0000000000000 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ /dev/null @@ -1,386 +0,0 @@ -#![cfg_attr(not(feature = "std"), no_std)] - -use alloc::{format, string::String}; -use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; -use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; -use frame_support::{ - decl_event, decl_module, decl_storage, - log::{error, info, warn}, - traits::Currency, - weights::{Weight}, - dispatch::DispatchResult, - RuntimeDebug, - BoundedVec, - parameter_types, -}; -use frame_system::{ensure_signed, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; -use pallet_staking::{self as staking}; -use sp_core::crypto::{KeyTypeId, UncheckedFrom};use sp_io::offchain::timestamp; -use sp_runtime::offchain::{http, Duration}; -use sp_std::{vec, vec::Vec}; -extern crate alloc; -use pallet_session as session; - -parameter_types! { - pub DdcValidatorsQuorumSize: u32 = 3; -} - -use core::fmt::Debug; -use scale_info::TypeInfo; - -type BalanceOf = <::Currency as Currency< - ::AccountId, ->>::Balance; - -pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); - -pub const HTTP_TIMEOUT_MS: u64 = 30_000; - -const TIME_START_MS: u64 = 1_672_531_200_000; -const ERA_DURATION_MS: u64 = 120_000; -const ERA_IN_BLOCKS: u8 = 20; - -const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; - -type ResultStr = Result; - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct RedisFtAggregate { - #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(untagged)] -pub enum FtAggregate { - Length(u32), - Node(Vec), -} - -#[derive(Clone)] -struct BytesSent { - node_public_key: String, - era: String, - sum: u32, -} - -impl BytesSent { - pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => { - return BytesSent { - node_public_key: node[1].clone(), - era: node[3].clone(), - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } -} - -#[derive(Clone)] -struct BytesReceived { - node_public_key: String, - era: String, - sum: u32, -} - -impl BytesReceived { - pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => { - return BytesReceived { - node_public_key: node[1].clone(), - era: node[3].clone(), - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } -} - -#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - era: String, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, -} - -// use sp_std::fmt; -// impl fmt::Display for ValidationResult -// { -// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { -// write!(f, "ValidationResult signer {}", self.signer) -// } -// } - -pub mod crypto { - use super::KEY_TYPE; - use frame_system::offchain::AppCrypto; - use sp_core::sr25519::Signature as Sr25519Signature; - use sp_runtime::{ - app_crypto::{app_crypto, sr25519}, - traits::Verify, - }; - app_crypto!(sr25519, KEY_TYPE); - - use sp_runtime::{MultiSignature, MultiSigner}; - - pub struct TestAuthId; - - impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; - } - - impl AppCrypto for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; - } -} - -pub use pallet::*; - -#[frame_support::pallet] -pub mod pallet { - use super::*; - use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; - - #[pallet::pallet] - #[pallet::without_storage_info] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: - frame_system::Config - + pallet_contracts::Config - + pallet_session::Config::AccountId> - + pallet_staking::Config - + CreateSignedTransaction> - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - /// The overarching dispatch call type. - type Call: From>; - - type AuthorityId: AppCrypto; - } - - #[pallet::hooks] - impl Hooks> for Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - fn offchain_worker(block_number: T::BlockNumber) { - let res = Self::offchain_worker_main(block_number); - - match res { - Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), - Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), - }; - } - } - - #[pallet::call] - impl Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - #[pallet::weight(10000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { - let signer: T::AccountId = ensure_signed(origin)?; - - info!("[DAC Validator] author: {:?}", signer); - let mut v_results = ValidationResults::::get(); - - let cur_validation = ValidationResult:: { - era, - val_res, - cdn_node_pub_key, - signer, - }; - - // ValidationResults::::append(cur_validation); - // ValidationResults::::mutate(|results| { - // results.push(cur_validation); - // - // results.clone() - // }); - v_results.push(cur_validation); - - ValidationResults::::set(v_results); - - Ok(()) - } - } - - #[pallet::storage] - #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; -} - -impl Pallet -where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, -{ - fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); - - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) - } - - let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()); - } - Ok(signer) => signer, - }; - - info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); - - // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; - let (bytes_sent, bytes_received) = Self::fetch_data(current_era); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - - let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Sending save_validated_data tx"); - - // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - }); - - match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting save_validated_data TX") - } - Some((_, Ok(()))) => {} - } - - // info!("[DAC Validator] save_validated_data: {:?}", ValidationResults::::get()); - - Ok(()) - } - - fn get_signer() -> ResultStr> { - let signer = Signer::<_, _>::any_account(); - if !signer.can_sign() { - return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); - } - - Ok(signer) - } - - fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { - true - } else { - false - } - } - - fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); - // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); - let bytes_sent = BytesSent::new(bytes_sent_res); - - // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); - let bytes_received = BytesReceived::new(bytes_received_res); - - (bytes_sent, bytes_received) - } - - fn get_current_era() -> u64 { - // info!("timestamp: {}", timestamp().unix_millis()); - // info!("TIME_START_MS: {}", TIME_START_MS); - // info!("ERA_DURATION_MS: {}", ERA_DURATION_MS); - // info!("1: {}", timestamp().unix_millis() - TIME_START_MS); - // info!("2: {}", (timestamp().unix_millis() - TIME_START_MS) % ERA_DURATION_MS); - - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS - } - - fn get_bytes_sent_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) - } - - fn get_bytes_received_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) - } - - fn http_get_json(url: &str) -> ResultStr { - let body = Self::http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); - "HTTP GET error" - })?; - - let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); - "HTTP JSON parse error" - }); - - parsed - } - - fn http_get_request(http_url: &str) -> Result, http::Error> { - info!("[DAC Validator] Sending request to: {:?}", http_url); - - // Initiate an external HTTP GET request. This is using high-level wrappers from - // `sp_runtime`. - let request = http::Request::get(http_url); - - let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); - - let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; - - let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; - - if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); - return Err(http::Error::Unknown) - } - - // Next we fully read the response body and collect it to a vector of bytes. - Ok(response.body().collect::>()) - } - - fn validators() -> Vec<::ValidatorId> { - >::validators() - } - - /// Reward a validator. - pub fn reward_by_ids( - validators_points: impl IntoIterator, - ) -> Result<(), ()> { - >::reward_by_ids(validators_points); - Ok(()) - } -} \ No newline at end of file From fbf4cdfc572fa83b68b4165167dab3d7b0124e04 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 20:04:43 +0600 Subject: [PATCH 025/192] Use era type from staking primitives --- frame/ddc-validator/src/lib.rs | 37 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index cf4423173bc88..1de5200c696ec 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -26,6 +26,7 @@ pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_runtime::offchain::{http, Duration, Timestamp}; +pub use sp_staking::EraIndex; pub use sp_std::prelude::*; extern crate alloc; @@ -64,7 +65,7 @@ pub struct Decision { #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { - era: String, + era: EraIndex, signer: AccountId, val_res: bool, cdn_node_pub_key: String, @@ -89,7 +90,7 @@ pub enum FtAggregate { #[derive(Clone)] struct BytesSent { node_public_key: String, - era: String, + era: EraIndex, sum: u32, } @@ -101,7 +102,8 @@ impl BytesSent { FtAggregate::Node(node) => return BytesSent { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), @@ -116,7 +118,8 @@ impl BytesSent { FtAggregate::Node(node) => { let node = BytesSent { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), }; @@ -133,7 +136,7 @@ impl BytesSent { #[derive(Clone)] struct BytesReceived { node_public_key: String, - era: String, + era: EraIndex, sum: u32, } @@ -145,7 +148,8 @@ impl BytesReceived { FtAggregate::Node(node) => return BytesReceived { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5] .parse::() .expect("bytesReceivedSum must be convertable to u32"), @@ -162,7 +166,8 @@ impl BytesReceived { FtAggregate::Node(node) => { let node = BytesReceived { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), }; @@ -242,7 +247,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn last_managed_era)] - pub type LastManagedEra = StorageValue<_, u64>; + pub type LastManagedEra = StorageValue<_, EraIndex>; #[pallet::storage] #[pallet::getter(fn validation_results)] @@ -332,7 +337,7 @@ pub mod pallet { origin: OriginFor, val_res: bool, cdn_node_pub_key: String, - era: String, + era: EraIndex, ) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; @@ -350,7 +355,7 @@ pub mod pallet { } #[pallet::weight(10000)] - pub fn proof_of_delivery(origin: OriginFor, era: u64) -> DispatchResult { + pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; let cdn_nodes_to_validate = Self::fetch_tasks(&signer); @@ -398,7 +403,7 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; + let current_era = Self::get_current_era() - 1; let tx_res = signer.send_signed_transaction(|_acct| { info!("[DAC Validator] Trigger proof of delivery"); @@ -426,13 +431,13 @@ pub mod pallet { } // Get the current era; Shall we start era count from 0 or from 1? - fn get_current_era() -> u64 { + fn get_current_era() -> EraIndex { ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) .try_into() .unwrap() } - fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { + fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); @@ -466,7 +471,7 @@ pub mod pallet { (filtered_s.clone(), filtered_r.clone()) } - fn fetch_data1(era: u64 ) -> (Vec, Vec){ + fn fetch_data1(era: EraIndex) -> (Vec, Vec){ info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); @@ -484,11 +489,11 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64) -> String { + fn get_bytes_sent_query_url(era: EraIndex) -> String { format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) } - fn get_bytes_received_query_url(era: u64) -> String { + fn get_bytes_received_query_url(era: EraIndex) -> String { format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) } From 27121ecce51a76283e0fa336d2da05c72e82b349 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:46:36 +0600 Subject: [PATCH 026/192] Fix faulty hook on block initialization --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 1de5200c696ec..edbaba17d690b 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -271,7 +271,7 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - if block_number < 1u32.into() { + if block_number <= 1u32.into() { return 0 } From 6771d11058cf40051219f2f3355d526d91468336 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:52:12 +0600 Subject: [PATCH 027/192] Autoformat DAC Validator files --- frame/ddc-validator/src/lib.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index edbaba17d690b..c0fe9def0cc93 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -111,7 +111,7 @@ impl BytesSent { } pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec!(); + let mut res: Vec = vec![]; for i in 1..aggregation.ft_aggregate.len() { let data = aggregation.ft_aggregate[i].clone(); match data { @@ -120,16 +120,18 @@ impl BytesSent { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + sum: node[5] + .parse::() + .expect("bytesSentSum must be convertable to u32"), }; res.push(node); - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } - return res; + return res } } @@ -159,7 +161,7 @@ impl BytesReceived { } pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec!(); + let mut res: Vec = vec![]; for i in 1..aggregation.ft_aggregate.len() { let data = aggregation.ft_aggregate[i].clone(); match data { @@ -168,16 +170,18 @@ impl BytesReceived { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertable to u32"), }; res.push(node); - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } - return res; + return res } } @@ -462,7 +466,11 @@ pub mod pallet { pub_key_str } - fn filter_data(s: &Vec, r: &Vec, a: &T::AccountId) -> (BytesSent, BytesReceived){ + fn filter_data( + s: &Vec, + r: &Vec, + a: &T::AccountId, + ) -> (BytesSent, BytesReceived) { let ac = Self::account_to_string(a.clone()); let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); @@ -471,7 +479,7 @@ pub mod pallet { (filtered_s.clone(), filtered_r.clone()) } - fn fetch_data1(era: EraIndex) -> (Vec, Vec){ + fn fetch_data1(era: EraIndex) -> (Vec, Vec) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); From 36ff86e7084961e1ec183127e504407cf2b2e2b1 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:53:59 +0600 Subject: [PATCH 028/192] Typo fix --- frame/ddc-validator/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index c0fe9def0cc93..a30027c7d1779 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -104,7 +104,7 @@ impl BytesSent { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + sum: node[5].parse::().expect("bytesSentSum must be convertible to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } @@ -122,7 +122,7 @@ impl BytesSent { as EraIndex, sum: node[5] .parse::() - .expect("bytesSentSum must be convertable to u32"), + .expect("bytesSentSum must be convertible to u32"), }; res.push(node); @@ -154,7 +154,7 @@ impl BytesReceived { as EraIndex, sum: node[5] .parse::() - .expect("bytesReceivedSum must be convertable to u32"), + .expect("bytesReceivedSum must be convertible to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } @@ -172,7 +172,7 @@ impl BytesReceived { as EraIndex, sum: node[5] .parse::() - .expect("bytesReceivedSum must be convertable to u32"), + .expect("bytesReceivedSum must be convertible to u32"), }; res.push(node); From 1a926240454792def5167c4e6dd697aab6b00b96 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 13:45:04 +0600 Subject: [PATCH 029/192] Introduce era key for DAC validator tasks --- frame/ddc-validator/src/lib.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a30027c7d1779..32354fa1933ee 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -242,9 +242,11 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn tasks)] - pub type Tasks = StorageMap< + pub type Tasks = StorageDoubleMap< _, Twox64Concat, + EraIndex, + Twox64Concat, T::AccountId, BoundedVec, DdcValidatorsQuorumSize>, >; @@ -314,7 +316,7 @@ pub mod pallet { }; decisions.try_push(assignment).unwrap(); } - Tasks::::insert(edge, decisions); + Tasks::::insert(era, edge, decisions); } 0 @@ -361,14 +363,14 @@ pub mod pallet { #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; - - let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + let era = Self::get_current_era(); + let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); let (s, r) = Self::fetch_data1(era); for cdn_node_id in cdn_nodes_to_validate { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let decisions_for_cdn = >::get(cdn_node_id); + let decisions_for_cdn = >::get(era, cdn_node_id); for decision in decisions_for_cdn.unwrap().iter_mut() { if decision.validator == signer { decision.decision = Some(val_res); @@ -547,9 +549,9 @@ pub mod pallet { } /// Fetch the tasks related to current validator - fn fetch_tasks(validator: &T::AccountId) -> Vec { + fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; - for (cdn_id, cdn_tasks) in >::iter() { + for (cdn_id, cdn_tasks) in >::iter_prefix(era) { for decision in cdn_tasks.iter() { if decision.validator == *validator { cdn_nodes.push(cdn_id); From e8dc17dc1c3cbb1548ead52297c8d128d83f5182 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 13:02:18 +0600 Subject: [PATCH 030/192] Set DAC Validators quorum size in runtime crate --- bin/node/runtime/src/lib.rs | 5 +++++ frame/ddc-validator/src/lib.rs | 11 +++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index d167d2ee904c8..a6d54b1471ce5 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1230,7 +1230,12 @@ impl pallet_ddc_staking::Config for Runtime { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +parameter_types! { + pub const DdcValidatorsQuorumSize: u32 = 3; +} + impl pallet_ddc_validator::Config for Runtime { + type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; type Event = Event; type Randomness = RandomnessCollectiveFlip; type Call = Call; diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 32354fa1933ee..eb284d97a7595 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -31,10 +31,6 @@ pub use sp_std::prelude::*; extern crate alloc; -parameter_types! { - pub DdcValidatorsQuorumSize: u32 = 3; -} - type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -238,6 +234,9 @@ pub mod pallet { type Call: From>; type AuthorityId: AppCrypto; type TimeProvider: UnixTime; + + #[pallet::constant] + type DdcValidatorsQuorumSize: Get; } #[pallet::storage] @@ -248,7 +247,7 @@ pub mod pallet { EraIndex, Twox64Concat, T::AccountId, - BoundedVec, DdcValidatorsQuorumSize>, + BoundedVec, T::DdcValidatorsQuorumSize>, >; #[pallet::storage] @@ -304,7 +303,7 @@ pub mod pallet { // A naive approach assigns random validators for each edge. for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = Default::default(); while !decisions.is_full() { let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; From 3c23004453e22d349ddc3a0b3f0281ca83f87cf8 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 15:26:38 +0600 Subject: [PATCH 031/192] Print pubkeys available to offchain worker --- frame/ddc-validator/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index eb284d97a7595..6c06457a1b4ae 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -25,6 +25,7 @@ pub use pallet_session as session; pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; +pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, Duration, Timestamp}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; @@ -322,6 +323,16 @@ pub mod pallet { } fn offchain_worker(block_number: T::BlockNumber) { + let pubkeys = sr25519_public_keys(KEY_TYPE); + if pubkeys.is_empty() { + log::info!("No local sr25519 accounts available to offchain worker."); + return + } + log::info!( + "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", + pubkeys, pubkeys.first().unwrap() + ); + let res = Self::offchain_worker_main(block_number); match res { From 2e8d0f63d74b8d5c8ae848442a27e71d41867ac4 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:46:42 +0200 Subject: [PATCH 032/192] merge with latest commit --- frame/ddc-validator/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6c06457a1b4ae..229e01f05a2fa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -32,6 +32,10 @@ pub use sp_std::prelude::*; extern crate alloc; +parameter_types! { + pub const ValidationThreshold: f32 = 5.0; +} + type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -555,7 +559,9 @@ pub mod pallet { } fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { true } else { false } + let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); + + return if percentage_difference > 0.0 && (ValidationThreshold::get() - percentage_difference) > 0.0 { true } else { false } } /// Fetch the tasks related to current validator From a415b9b0612915dfc8b9fd75c3d0b101c7be7bd6 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:49:55 +0200 Subject: [PATCH 033/192] fix merge --- frame/ddc-validator/src/lib.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 229e01f05a2fa..b39e005488fce 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -88,8 +88,8 @@ pub enum FtAggregate { Node(Vec), } -#[derive(Clone)] -struct BytesSent { +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesSent { node_public_key: String, era: EraIndex, sum: u32, @@ -136,8 +136,8 @@ impl BytesSent { } } -#[derive(Clone)] -struct BytesReceived { +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesReceived { node_public_key: String, era: EraIndex, sum: u32, @@ -375,8 +375,9 @@ pub mod pallet { } #[pallet::weight(10000)] - pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { + pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; + let era = Self::get_current_era(); let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); let (s, r) = Self::fetch_data1(era); @@ -423,15 +424,12 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1; + let current_era = Self::get_current_era() - 1u64; + let (s, r) = Self::fetch_data1(current_era); let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Trigger proof of delivery"); - - // This is the on-chain function - Call::proof_of_delivery { era: current_era } + Call::proof_of_delivery { s: s.clone(), r: r.clone() } }); - match &tx_res { None | Some((_, Err(()))) => return Err("Error while submitting proof of delivery TX"), From 816dff77f3c59bc2d74bdbd1bba0775da05abb5d Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:52:33 +0200 Subject: [PATCH 034/192] fix typo --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b39e005488fce..af78c5be65e82 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -424,7 +424,7 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; + let current_era = Self::get_current_era() - 1; let (s, r) = Self::fetch_data1(current_era); let tx_res = signer.send_signed_transaction(|_acct| { From bd2f5b7e27c0abcedbef67b5dd1b75f875b13fe4 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 16:35:22 +0600 Subject: [PATCH 035/192] Add doc comments --- frame/ddc-validator/src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index af78c5be65e82..eb28a1d59c77e 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -36,6 +36,7 @@ parameter_types! { pub const ValidationThreshold: f32 = 5.0; } +/// The balance type of this pallet. type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -52,15 +53,22 @@ const ERA_IN_BLOCKS: u8 = 20; const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; +/// DAC Validation methods. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { + /// Compare amount of served content with amount of content consumed. ProofOfDelivery, } +/// Associates validation decision with the validator and the method used to produce it. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct Decision { + /// Individual validator's decision. Can be `None` if the validator did not produce a decision + /// (yet). pub decision: Option, + /// The method used to produce the decision. pub method: ValidationMethodKind, + /// The validator who produced the decision. pub validator: AccountId, } @@ -234,16 +242,27 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { + /// The overarching event type. type Event: From> + IsType<::Event>; + + /// Something that provides randomness in the runtime. Required by the tasks assignment + /// procedure. type Randomness: Randomness; + + /// A dispatchable call. type Call: From>; + type AuthorityId: AppCrypto; type TimeProvider: UnixTime; + /// Number of validators expected to produce an individual validation decision to form a + /// consensus. Tasks assignment procedure use this value to determine the number of + /// validators are getting the same task. Must be an odd number. #[pallet::constant] type DdcValidatorsQuorumSize: Get; } + /// The map from the era and CDN participant stash key to the validation decisions related. #[pallet::storage] #[pallet::getter(fn tasks)] pub type Tasks = StorageDoubleMap< @@ -255,6 +274,7 @@ pub mod pallet { BoundedVec, T::DdcValidatorsQuorumSize>, >; + /// The last era for which the tasks assignment produced. #[pallet::storage] #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, EraIndex>; @@ -576,6 +596,9 @@ pub mod pallet { cdn_nodes } + /// Randomly choose a number in range `[0, total)`. + /// Returns `None` for zero input. + /// Modification of `choose_ticket` from `pallet-lottery` version `4.0.0-dev`. fn choose(total: u32) -> Option { if total == 0 { return None @@ -594,6 +617,11 @@ pub mod pallet { Some(random_number % total) } + /// Generate a random number from a given seed. + /// Note that there is potential bias introduced by using modulus operator. + /// You should call this function with different seed values until the random + /// number lies within `u32::MAX - u32::MAX % n`. + /// Modification of `generate_random_number` from `pallet-lottery` version `4.0.0-dev`. fn generate_random_number(seed: u32) -> u32 { let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); From 5b330b5ea6cbe422eef893567c0c186b2dc5f5e0 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 18:00:40 +0600 Subject: [PATCH 036/192] Module level docs --- frame/ddc-validator/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index eb28a1d59c77e..6402d9f24b772 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,3 +1,37 @@ +//! # DDC Validator pallet +//! +//! The DDC Validator pallet is responsible for producing validation decisions based on activity +//! data from DAC DataModel. It is expected to work on validators nodes only. +//! +//! - [`Config`] +//! - [`Call`] +//! - [`Pallet`] +//! - [`Hooks`] +//! +//! ## Responsibility +//! +//! 1. Assign validation tasks on DAC Validators in the beginning of each era, +//! 2. Spin the offchain worker which tries to execute the validation tasks each era, +//! 3. Fetch the data required for validation from DAC DataModel, +//! 4. Execute validation method on this data, +//! 5. Produce validation decision and submit it to the chain. +//! +//! ## Usage +//! +//! 1. Run the node with `--validator` flag, +//! 2. Setup validator key with `author_insertKey` RPC call. Use `dacv` validator key type and the +//! same private key as the one used to generate the validator's session keys, +//! 3. Proceed a regular validator setup, +//! 4. Tasks assignment will assign you a task in the beginning of the era which has your account in +//! validators set. +//! +//! ## Notes +//! +//! - Era definition in this pallet is different than in the `pallet-staking`. In this pallet era is +//! a period of time during which the validator is expected to produce a validation decision. +//! Means staking era and DAC era are different and are not related to each other, +//! - You can set DAC Validators quorum size by specifying `DdcValidatorsQuorumSize` parameter, + #![cfg_attr(not(feature = "std"), no_std)] pub use alloc::{format, string::String}; From 12a1e612024e9998b84c2d248d79fa55c072029e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 29 Mar 2023 22:51:19 +0200 Subject: [PATCH 037/192] Remove http request from tx --- frame/ddc-validator/src/lib.rs | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6402d9f24b772..0be9e9454a443 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -430,22 +430,32 @@ pub mod pallet { #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { + info!("[DAC Validator] processing proof_of_delivery"); let signer: T::AccountId = ensure_signed(origin)?; + info!("signer: {:?}", Self::account_to_string(signer.clone())); + let era = Self::get_current_era(); let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); - let (s, r) = Self::fetch_data1(era); + + info!("[DAC Validator] cdn_nodes_to_validate: {:?}", cdn_nodes_to_validate); + for cdn_node_id in cdn_nodes_to_validate { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let decisions_for_cdn = >::get(era, cdn_node_id); - for decision in decisions_for_cdn.unwrap().iter_mut() { + for decision in decisions_for_cdn.clone().unwrap().iter_mut() { if decision.validator == signer { decision.decision = Some(val_res); decision.method = ValidationMethodKind::ProofOfDelivery; } } + + info!( + "[DAC Validator] decisions_for_cdn: {:?}", + decisions_for_cdn + ); } Ok(()) @@ -458,11 +468,6 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!( - "[DAC Validator] Validation data stored onchain: {:?}", - ValidationResults::::get() - ); - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { return Ok(()) } @@ -475,8 +480,6 @@ pub mod pallet { Ok(signer) => signer, }; - info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); - // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1; let (s, r) = Self::fetch_data1(current_era); @@ -484,9 +487,13 @@ pub mod pallet { let tx_res = signer.send_signed_transaction(|_acct| { Call::proof_of_delivery { s: s.clone(), r: r.clone() } }); + match &tx_res { - None | Some((_, Err(()))) => - return Err("Error while submitting proof of delivery TX"), + None => return Err("Error while submitting proof of delivery TX"), + Some((_, Err(e))) => { + info!("Error while submitting proof of delivery TX: {:?}", e); + return Err("Error while submitting proof of delivery TX"); + }, Some((_, Ok(()))) => {}, } @@ -620,6 +627,8 @@ pub mod pallet { fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; for (cdn_id, cdn_tasks) in >::iter_prefix(era) { + info!("[DAC Validator] tasks assigned to {:?}: {:?}", cdn_id, cdn_tasks); + for decision in cdn_tasks.iter() { if decision.validator == *validator { cdn_nodes.push(cdn_id); From 311a27036beaa9db1190140276f7638a528d5fd4 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 30 Mar 2023 14:51:59 +0600 Subject: [PATCH 038/192] Submit DAC Validation decision call --- frame/ddc-validator/src/lib.rs | 50 ++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0be9e9454a443..b456baf621728 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -323,10 +323,22 @@ pub mod pallet { pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + /// DAC Validator successfully published the validation decision. + ValidationDecisionSubmitted, + } #[pallet::error] - pub enum Error {} + pub enum Error { + /// Validation decision attempts to submit the result for the wrong era (not the current + /// one). + BadEra, + /// Can't submit the validation decision twice. + DecisionAlreadySubmitted, + /// Task does not exist for a given era, CDN participant, and DAC validator. + TaskNotFound, + } #[pallet::hooks] impl Hooks> for Pallet @@ -428,6 +440,40 @@ pub mod pallet { Ok(()) } + /// Set validation decision in tasks assignment. + /// + /// `origin` must be a DAC Validator assigned to the task. + /// `era` must be a current era, otherwise the decision will be rejected. + /// `subject` is a CDN participant stash. + /// + /// Emits `ValidationDecisionSubmitted` event. + #[pallet::weight(100_000)] + pub fn submit_validation_decision( + origin: OriginFor, + era: EraIndex, + subject: T::AccountId, + method: ValidationMethodKind, + decision: bool, + ) -> DispatchResult { + let account = ensure_signed(origin)?; + + ensure!(Self::get_current_era() == era, Error::::BadEra); + + Tasks::::try_mutate_exists(era, &subject, |maybe_tasks| { + let mut tasks = maybe_tasks.take().ok_or(Error::::TaskNotFound)?; + let mut task = tasks + .iter_mut() + .find(|task| task.validator == account && task.method == method) + .ok_or(Error::::TaskNotFound)?; + ensure!(task.decision.is_none(), Error::::DecisionAlreadySubmitted); + task.decision = Some(decision); + + Self::deposit_event(Event::ValidationDecisionSubmitted); + + Ok(()) + }) + } + #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { info!("[DAC Validator] processing proof_of_delivery"); From 0239ff17acffbff56ce24a6492cdd03dc6906c49 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Fri, 14 Jan 2022 12:27:35 +0300 Subject: [PATCH 039/192] Set shorted duration --- bin/node/runtime/src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a6d54b1471ce5..b591b576392a9 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -681,12 +681,12 @@ impl pallet_bags_list::Config for Runtime { } parameter_types! { - pub const LaunchPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const VotingPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const FastTrackVotingPeriod: BlockNumber = 3 * 60 * MINUTES; - pub const MinimumDeposit: Balance = 5000 * DOLLARS; - pub const EnactmentPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const CooloffPeriod: BlockNumber = 7 * 24 * 60 * MINUTES; + pub const LaunchPeriod: BlockNumber = 5 * MINUTES; + pub const VotingPeriod: BlockNumber = 5 * MINUTES; + pub const FastTrackVotingPeriod: BlockNumber = 5 * MINUTES; + pub const MinimumDeposit: Balance = 10 * DOLLARS; + pub const EnactmentPeriod: BlockNumber = 5 * MINUTES; + pub const CooloffPeriod: BlockNumber = 5 * MINUTES; pub const MaxProposals: u32 = 100; } @@ -742,7 +742,7 @@ impl pallet_democracy::Config for Runtime { } parameter_types! { - pub const CouncilMotionDuration: BlockNumber = 7 * DAYS; + pub const CouncilMotionDuration: BlockNumber = 5 * MINUTES; pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; } @@ -764,7 +764,7 @@ parameter_types! { // 1 storage item created, key size is 32 bytes, value size is 16+16. pub const VotingBondBase: Balance = deposit(1, 64); pub const VotingBondFactor: Balance = 1 * DOLLARS; - pub const TermDuration: BlockNumber = 7 * DAYS; + pub const TermDuration: BlockNumber = 5 * MINUTES; pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 20; pub const ElectionsPhragmenPalletId: LockIdentifier = *b"phrelect"; @@ -794,7 +794,7 @@ impl pallet_elections_phragmen::Config for Runtime { } parameter_types! { - pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS; + pub const TechnicalMotionDuration: BlockNumber = 5 * MINUTES; pub const TechnicalMaxProposals: u32 = 100; pub const TechnicalMaxMembers: u32 = 100; } @@ -831,9 +831,9 @@ impl pallet_membership::Config for Runtime { parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const SpendPeriod: BlockNumber = 1 * DAYS; + pub const SpendPeriod: BlockNumber = 5 * MINUTES; pub const Burn: Permill = Permill::from_percent(0); - pub const TipCountdown: BlockNumber = 1 * DAYS; + pub const TipCountdown: BlockNumber = 5 * MINUTES; pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const DataDepositPerByte: Balance = 1 * CENTS; @@ -1117,10 +1117,10 @@ parameter_types! { pub const CandidateDeposit: Balance = 10 * DOLLARS; pub const WrongSideDeduction: Balance = 2 * DOLLARS; pub const MaxStrikes: u32 = 10; - pub const RotationPeriod: BlockNumber = 80 * HOURS; + pub const RotationPeriod: BlockNumber = 5 * MINUTES; pub const PeriodSpend: Balance = 500 * DOLLARS; pub const MaxLockDuration: BlockNumber = 36 * 30 * DAYS; - pub const ChallengePeriod: BlockNumber = 7 * DAYS; + pub const ChallengePeriod: BlockNumber = 15 * MINUTES; pub const MaxCandidateIntake: u32 = 10; pub const SocietyPalletId: PalletId = PalletId(*b"py/socie"); } From 3d4ea108ee378ae6ee089d85d8a335eb34ae53fb Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 1 Nov 2022 09:37:49 +0300 Subject: [PATCH 040/192] Update EPOCH_DURATION_IN_BLOCKS --- bin/node/runtime/src/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index aa4dcd79be23d..f854b194e4e11 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -61,7 +61,7 @@ pub mod time { // 1 in 4 blocks (on average, not counting collisions) will be primary BABE blocks. pub const PRIMARY_PROBABILITY: (u64, u64) = (1, 4); - pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 4 * HOURS; + pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 5 * MINUTES; pub const EPOCH_DURATION_IN_SLOTS: u64 = { const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64; From a7c549820773e397963ea64fed5cb5d581154028 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 31 Mar 2023 10:27:35 +0200 Subject: [PATCH 041/192] Revert "Update EPOCH_DURATION_IN_BLOCKS" This reverts commit 3d4ea108ee378ae6ee089d85d8a335eb34ae53fb. --- bin/node/runtime/src/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index f854b194e4e11..aa4dcd79be23d 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -61,7 +61,7 @@ pub mod time { // 1 in 4 blocks (on average, not counting collisions) will be primary BABE blocks. pub const PRIMARY_PROBABILITY: (u64, u64) = (1, 4); - pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 5 * MINUTES; + pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 4 * HOURS; pub const EPOCH_DURATION_IN_SLOTS: u64 = { const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64; From 6934eb1d01e86c32d17318a8c2e72d56eb05712a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 31 Mar 2023 10:28:14 +0200 Subject: [PATCH 042/192] Revert "Set shorted duration" This reverts commit 0239ff17acffbff56ce24a6492cdd03dc6906c49. --- bin/node/runtime/src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index b591b576392a9..a6d54b1471ce5 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -681,12 +681,12 @@ impl pallet_bags_list::Config for Runtime { } parameter_types! { - pub const LaunchPeriod: BlockNumber = 5 * MINUTES; - pub const VotingPeriod: BlockNumber = 5 * MINUTES; - pub const FastTrackVotingPeriod: BlockNumber = 5 * MINUTES; - pub const MinimumDeposit: Balance = 10 * DOLLARS; - pub const EnactmentPeriod: BlockNumber = 5 * MINUTES; - pub const CooloffPeriod: BlockNumber = 5 * MINUTES; + pub const LaunchPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const VotingPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const FastTrackVotingPeriod: BlockNumber = 3 * 60 * MINUTES; + pub const MinimumDeposit: Balance = 5000 * DOLLARS; + pub const EnactmentPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const CooloffPeriod: BlockNumber = 7 * 24 * 60 * MINUTES; pub const MaxProposals: u32 = 100; } @@ -742,7 +742,7 @@ impl pallet_democracy::Config for Runtime { } parameter_types! { - pub const CouncilMotionDuration: BlockNumber = 5 * MINUTES; + pub const CouncilMotionDuration: BlockNumber = 7 * DAYS; pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; } @@ -764,7 +764,7 @@ parameter_types! { // 1 storage item created, key size is 32 bytes, value size is 16+16. pub const VotingBondBase: Balance = deposit(1, 64); pub const VotingBondFactor: Balance = 1 * DOLLARS; - pub const TermDuration: BlockNumber = 5 * MINUTES; + pub const TermDuration: BlockNumber = 7 * DAYS; pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 20; pub const ElectionsPhragmenPalletId: LockIdentifier = *b"phrelect"; @@ -794,7 +794,7 @@ impl pallet_elections_phragmen::Config for Runtime { } parameter_types! { - pub const TechnicalMotionDuration: BlockNumber = 5 * MINUTES; + pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS; pub const TechnicalMaxProposals: u32 = 100; pub const TechnicalMaxMembers: u32 = 100; } @@ -831,9 +831,9 @@ impl pallet_membership::Config for Runtime { parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const SpendPeriod: BlockNumber = 5 * MINUTES; + pub const SpendPeriod: BlockNumber = 1 * DAYS; pub const Burn: Permill = Permill::from_percent(0); - pub const TipCountdown: BlockNumber = 5 * MINUTES; + pub const TipCountdown: BlockNumber = 1 * DAYS; pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const DataDepositPerByte: Balance = 1 * CENTS; @@ -1117,10 +1117,10 @@ parameter_types! { pub const CandidateDeposit: Balance = 10 * DOLLARS; pub const WrongSideDeduction: Balance = 2 * DOLLARS; pub const MaxStrikes: u32 = 10; - pub const RotationPeriod: BlockNumber = 5 * MINUTES; + pub const RotationPeriod: BlockNumber = 80 * HOURS; pub const PeriodSpend: Balance = 500 * DOLLARS; pub const MaxLockDuration: BlockNumber = 36 * 30 * DAYS; - pub const ChallengePeriod: BlockNumber = 15 * MINUTES; + pub const ChallengePeriod: BlockNumber = 7 * DAYS; pub const MaxCandidateIntake: u32 = 10; pub const SocietyPalletId: PalletId = PalletId(*b"py/socie"); } From 25cb8a813637e4a5d5788d3786343b518eddbdb5 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 31 Mar 2023 17:16:21 +0600 Subject: [PATCH 043/192] Fix validation decision update --- frame/ddc-validator/src/lib.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b456baf621728..1f10608c9e245 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -490,18 +490,17 @@ pub mod pallet { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let decisions_for_cdn = >::get(era, cdn_node_id); - for decision in decisions_for_cdn.clone().unwrap().iter_mut() { - if decision.validator == signer { - decision.decision = Some(val_res); - decision.method = ValidationMethodKind::ProofOfDelivery; - } - } - - info!( - "[DAC Validator] decisions_for_cdn: {:?}", - decisions_for_cdn - ); + >::mutate(era, cdn_node_id, |decisions_for_cdn| { + let decisions = + decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); + let mut decision = decisions + .iter_mut() + .find(|decision| decision.validator == signer) + .expect("unexpected validators set in tasks assignment"); + decision.decision = Some(val_res); + }); + + info!("[DAC Validator] decisions_for_cdn: {:?}", >::get(era, cdn_node_id)); } Ok(()) From 2f192211c3ccf12ba62d9fd4b9ed37cc1c073b3e Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 31 Mar 2023 17:33:09 +0600 Subject: [PATCH 044/192] Typo fix --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 1f10608c9e245..71c81b97f8499 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -490,7 +490,7 @@ pub mod pallet { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - >::mutate(era, cdn_node_id, |decisions_for_cdn| { + >::mutate(era, &cdn_node_id, |decisions_for_cdn| { let decisions = decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); let mut decision = decisions From ee6ba67a83ba73d8dbeabea36af3b2346b01ba8a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 11 Apr 2023 17:09:53 +0200 Subject: [PATCH 045/192] Try integrate unit tests --- Cargo.lock | 4 + frame/ddc-validator/Cargo.toml | 6 + frame/ddc-validator/src/lib.rs | 6 + frame/ddc-validator/src/mock.rs | 276 +++++++++++++++++++++++++++++++ frame/ddc-validator/src/tests.rs | 0 5 files changed, 292 insertions(+) create mode 100644 frame/ddc-validator/src/mock.rs create mode 100644 frame/ddc-validator/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index b2869c996bec3..07fb5329e9203 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5860,10 +5860,14 @@ dependencies = [ "frame-support", "frame-system", "log 0.4.17", + "pallet-balances", "pallet-contracts", "pallet-ddc-staking", + "pallet-randomness-collective-flip", "pallet-session", "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", "parity-scale-codec", "scale-info", "serde", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 0e0fe0ea95000..203e032e1e4d4 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -43,3 +43,9 @@ std = [ "sp-staking/std", "sp-std/std", ] + +[dev-dependencies] +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } \ No newline at end of file diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 71c81b97f8499..f1f64ad5e41c2 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -34,6 +34,12 @@ #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(test)] +mod mock; + +#[cfg(test)] +mod tests; + pub use alloc::{format, string::String}; pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; pub use codec::{Decode, Encode, HasCompact, MaxEncodedLen}; diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs new file mode 100644 index 0000000000000..b40c3a1c05ee7 --- /dev/null +++ b/frame/ddc-validator/src/mock.rs @@ -0,0 +1,276 @@ +use crate::{*, self as pallet_ddc_validator}; +use frame_support::{ + parameter_types, + weights::Weight, + traits::{ConstU16, ConstU64, Currency, Everything, Nothing} +}; +use frame_system::EnsureRoot; +use frame_system::offchain::SendTransactionTypes; +use sp_core::H256; +use sp_runtime::{ curve::PiecewiseLinear, generic, testing::{Header, TestXt}, traits::{BlakeTwo256, IdentityLookup, Verify, Extrinsic as ExtrinsicT, IdentifyAccount}, Perbill, MultiSignature, curve}; +use pallet_contracts as contracts; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; +type Balance = u128; +// type AccountId = u64; + +pub type Signature = MultiSignature; +pub type AccountId = <::Signer as IdentifyAccount>::AccountId; +// pub type Balance = u128; +pub type BlockNumber = u32; +pub type Moment = u64; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + Balances: pallet_balances, + Contracts: contracts, + Session: pallet_session, + Timestamp: pallet_timestamp, + RandomnessCollectiveFlip: pallet_randomness_collective_flip, + DdcStaking: pallet_ddc_staking, + DdcValidator: pallet_ddc_validator, + } +); + +parameter_types! { + pub const BlockHashCount: BlockNumber = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} + +impl frame_system::Config for Test { + type BaseCallFilter = Everything; + type BlockWeights = (); + type BlockLength = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = BlockNumber; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = AccountId; + // u64; // sp_core::sr25519::Public; + type Lookup = IdentityLookup; + type Header = generic::Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + +parameter_types! { + pub const SignedClaimHandicap: BlockNumber = 2; + pub const TombstoneDeposit: Balance = 16; + pub const StorageSizeOffset: u32 = 8; + pub const RentByteFee: Balance = 4; + pub const RentDepositOffset: Balance = 10_000; + pub const SurchargeReward: Balance = 150; + pub const MaxDepth: u32 = 100; + pub const MaxValueSize: u32 = 16_384; + pub Schedule: pallet_contracts::Schedule = Default::default(); +} + +use contracts::{Config as contractsConfig}; + +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +impl contracts::Config for Test { + type Time = Timestamp; + type Randomness = RandomnessCollectiveFlip; + type Currency = Balances; + type Event = Event; + type CallStack = [pallet_contracts::Frame; 31]; + type WeightPrice = Self; //pallet_transaction_payment::Module; + type WeightInfo = (); + type ChainExtension = (); + type DeletionQueueDepth = (); + type DeletionWeightLimit = (); + type Schedule = Schedule; + type Call = Call; + type CallFilter = Nothing; + type DepositPerByte = DepositPerByte; + type DepositPerItem = DepositPerItem; + type AddressGenerator = pallet_contracts::DefaultAddressGenerator; +} + +parameter_types! { + pub const TransactionByteFee: u64 = 0; + pub const DepositPerItem: Balance = 0; + pub const DepositPerByte: Balance = 0; +} + +parameter_types! { + pub const MinimumPeriod: u64 = 1; +} + +impl pallet_timestamp::Config for Test { + type Moment = Moment; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; + type WeightInfo = (); +} + +impl pallet_randomness_collective_flip::Config for Test {} + +parameter_types! { + pub const DdcValidatorsQuorumSize: u32 = 3; +} + +impl pallet_session::Config for Test { + type Event = (); + type ValidatorId = AccountId; + type ValidatorIdOf = (); + type ShouldEndSession = (); + type NextSessionRotation = (); + type SessionManager = (); + type SessionHandler = (); + type Keys = (); + type WeightInfo = (); +} + +pallet_staking_reward_curve::build! { + const REWARD_CURVE: PiecewiseLinear<'static> = curve!( + min_inflation: 0_000_100, + max_inflation: 0_050_000, + ideal_stake: 0_200_000, + falloff: 0_050_000, + max_piece_count: 100, + test_precision: 0_050_000, + ); +} + +parameter_types! { + pub const SessionsPerEra: sp_staking::SessionIndex = 6; + pub const BondingDuration: sp_staking::EraIndex = 3; + pub const SlashDeferDuration: sp_staking::EraIndex = 2; + pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; + pub const MaxNominatorRewardedPerValidator: u32 = 256; + pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); + pub OffchainRepeat: BlockNumber = 5; +} + +impl pallet_staking::Config for Test { + type MaxNominations = (); + type Currency = Balances; + type UnixTime = Timestamp; + type CurrencyToVote = (); + type RewardRemainder = (); + type Event = Event; + type Slash = (); // send the slashed funds to the treasury. + type Reward = (); // rewards are minted from the void + type SessionsPerEra = SessionsPerEra; + type BondingDuration = BondingDuration; + type SlashDeferDuration = SlashDeferDuration; + /// A super-majority of the council can cancel the slash. + type SlashCancelOrigin = (); + type SessionInterface = Self; + type EraPayout = pallet_staking::ConvertCurve; + type NextNewSession = Session; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; + type ElectionProvider = (); + type GenesisElectionProvider = (); + type VoterList = (); + type MaxUnlockingChunks = ConstU32<32>; + type WeightInfo = pallet_staking::weights::SubstrateWeight; + type BenchmarkingConfig = (); +} + +impl pallet_ddc_staking::Config for Test { + type BondingDuration = BondingDuration; + type Currency = Balances; + type Event = Event; + type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; +} + +impl pallet_ddc_validator::Config for Test { + type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; + type Event = Event; + type Randomness = RandomnessCollectiveFlip; + type Call = Call; + type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; + type TimeProvider = pallet_timestamp::Pallet; +} + +impl SendTransactionTypes for Test + where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} + +parameter_types! { + pub const ExistentialDeposit: u64 = 1; + pub const MaxLocks: u32 = 10; +} + +impl pallet_balances::Config for Test { + type Balance = Balance; + type DustRemoval = (); + type Event = Event; + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = (); +} + +// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + frame_system::GenesisConfig::default().build_storage::().unwrap().into() +} + +pub type Extrinsic = TestXt; + +impl SigningTypes for Test { + type Public = ::Signer; + type Signature = Signature; +} + +// impl frame_system::offchain::SendTransactionTypes for Test +// where +// Call: From, +// { +// type OverarchingCall = Call; +// type Extrinsic = TestXt; +// } + +// impl SendTransactionTypes for Test +// where +// Call: From, +// { +// type OverarchingCall = Call; +// type Extrinsic = Extrinsic; +// } + +impl CreateSignedTransaction for Test + where + Call: From, +{ + fn create_transaction>( + call: Call, + _public: ::Signer, + _account: AccountId, + nonce: u64, + ) -> Option<(Call, ::SignaturePayload)> { + Some((call, (nonce, ()))) + } +} \ No newline at end of file diff --git a/frame/ddc-validator/src/tests.rs b/frame/ddc-validator/src/tests.rs new file mode 100644 index 0000000000000..e69de29bb2d1d From 4d1fc2297ce97d31de78ce4b882d1c2f9f7d59b4 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 16 Mar 2023 20:22:00 +0600 Subject: [PATCH 046/192] An empty DAC Validator pallet crate --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + frame/ddc-validator/Cargo.toml | 19 +++++++++++++++++++ frame/ddc-validator/README.md | 1 + frame/ddc-validator/src/lib.rs | 26 ++++++++++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 frame/ddc-validator/Cargo.toml create mode 100644 frame/ddc-validator/README.md create mode 100644 frame/ddc-validator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index f4ae07eae343e..4864f553e6bf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5847,6 +5847,16 @@ dependencies = [ "substrate-test-utils", ] +[[package]] +name = "pallet-ddc-validator" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", +] + [[package]] name = "pallet-democracy" version = "4.0.0-dev" diff --git a/Cargo.toml b/Cargo.toml index 680a7394d7221..dd98b83860671 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ members = [ "frame/ddc-metrics-offchain-worker", "frame/ddc-pallet", "frame/ddc-staking", + "frame/ddc-validator", "frame/executive", "frame/gilt", "frame/grandpa", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml new file mode 100644 index 0000000000000..d9525b4dfd9af --- /dev/null +++ b/frame/ddc-validator/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pallet-ddc-validator" +version = "0.1.0" +edition = "2021" + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", +] diff --git a/frame/ddc-validator/README.md b/frame/ddc-validator/README.md new file mode 100644 index 0000000000000..3a21d9d6cc2b5 --- /dev/null +++ b/frame/ddc-validator/README.md @@ -0,0 +1 @@ +# DDC Validator diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs new file mode 100644 index 0000000000000..0cecc036b50ce --- /dev/null +++ b/frame/ddc-validator/src/lib.rs @@ -0,0 +1,26 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +pub use frame_support::pallet_prelude::*; +pub use frame_system::pallet_prelude::*; +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + type Event: From> + IsType<::Event>; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event {} + + #[pallet::error] + pub enum Error {} +} From c56af3c7e3f7cd7d6c1e20a0714a93886c6ac48a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 15:31:57 +0600 Subject: [PATCH 047/192] Add DAC Validator pallet to the runtime --- Cargo.lock | 1 + bin/node/runtime/Cargo.toml | 2 ++ bin/node/runtime/src/lib.rs | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 4864f553e6bf3..0cf086b5577c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5050,6 +5050,7 @@ dependencies = [ "pallet-contracts-rpc-runtime-api", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", + "pallet-ddc-validator", "pallet-democracy", "pallet-election-provider-multi-phase", "pallet-election-provider-support-benchmarking", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 853f66ce02f58..178490e3e7106 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -99,6 +99,7 @@ pallet-erc721 = { version = "2.0.0", default-features = false, path = "../../../ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../frame/erc20" } pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } +pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } @@ -174,6 +175,7 @@ std = [ "pallet-child-bounties/std", "pallet-ddc-metrics-offchain-worker/std", "pallet-ddc-staking/std", + "pallet-ddc-validator/std", ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fee2436f242a5..131614d8d7fa5 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -81,6 +81,7 @@ pub use pallet_cere_ddc; pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; +pub use pallet_ddc_validator; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1232,6 +1233,10 @@ impl pallet_ddc_staking::Config for Runtime { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +impl pallet_ddc_validator::Config for Runtime { + type Event = Event; +} + construct_runtime!( pub enum Runtime where Block = Block, @@ -1282,6 +1287,7 @@ construct_runtime!( Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, DdcStaking: pallet_ddc_staking, + DdcValidator: pallet_ddc_validator, } ); From 2a4317f66a27e7d209f7e7d75cfdf9b63bea8ffe Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 16:38:15 +0600 Subject: [PATCH 048/192] Era counter with privileged inc for DAC Validator --- frame/ddc-validator/src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0cecc036b50ce..a9b77496845ff 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -17,10 +17,29 @@ pub mod pallet { type Event: From> + IsType<::Event>; } + #[pallet::storage] + #[pallet::getter(fn global_era_counter)] + pub type GlobalEraCounter = StorageValue<_, u32>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event {} #[pallet::error] pub enum Error {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(100_000)] + pub fn inc_era(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + if let Some(era) = >::get() { + let new_era = era.checked_add(1).unwrap_or_default(); + >::put(new_era); + } else { + >::put(1); + } + Ok(()) + } + } } From 29eedabd03bf0303f5ada063af0ccf3ef05a1a43 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 17 Mar 2023 17:15:04 +0600 Subject: [PATCH 049/192] Hooks and storage items for DAC Validator --- frame/ddc-validator/Cargo.toml | 13 ++++++++ frame/ddc-validator/src/lib.rs | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index d9525b4dfd9af..057a274a2ec1b 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -7,7 +7,14 @@ edition = "2021" codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +log = { version = "0.4.17", default-features = false } +pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } [features] default = ["std"] @@ -15,5 +22,11 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", + "pallet-ddc-staking/std", + "pallet-staking/std", "scale-info/std", + "sp-io/std", + "sp-runtime/std", + "sp-staking/std", + "sp-std/std", ] diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a9b77496845ff..df766e3695547 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,8 +1,27 @@ #![cfg_attr(not(feature = "std"), no_std)] -pub use frame_support::pallet_prelude::*; +pub use frame_support::{pallet_prelude::*, parameter_types, weights::Weight, BoundedVec}; pub use frame_system::pallet_prelude::*; +pub use pallet_ddc_staking::{self as ddc_staking}; +pub use pallet_staking::{self as staking}; pub use pallet::*; +pub use sp_std::prelude::*; + +parameter_types! { + pub DdcValidatorsQuorumSize: u32 = 3; +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum ValidationMethodKind { + ProofOfDelivery, +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct Decision { + pub decision: Option, + pub method: ValidationMethodKind, + pub validator: AccountId, +} #[frame_support::pallet] pub mod pallet { @@ -13,14 +32,27 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config { + pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { type Event: From> + IsType<::Event>; } + #[pallet::storage] + #[pallet::getter(fn tasks)] + pub type Tasks = StorageMap< + _, + Twox64Concat, + T::AccountId, + BoundedVec, DdcValidatorsQuorumSize>, + >; + #[pallet::storage] #[pallet::getter(fn global_era_counter)] pub type GlobalEraCounter = StorageValue<_, u32>; + #[pallet::storage] + #[pallet::getter(fn last_managed_era)] + pub type LastManagedEra = StorageValue<_, u32>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event {} @@ -42,4 +74,24 @@ pub mod pallet { Ok(()) } } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(block_number: T::BlockNumber) -> Weight { + let validators: Vec = >::iter_keys().collect(); + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators: {:?}, edges: {:?}", + block_number, + >::get(), + >::get(), + validators, + edges, + ); + 0 + } + fn offchain_worker(block_number: T::BlockNumber) { + log::info!("Off-chain worker at block {:?}", block_number); + } + } } From 0c06af2e3fff2b36bd0967249d886056fb52afa1 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 17 Mar 2023 14:25:39 +0100 Subject: [PATCH 050/192] add function fetch_tasks --- frame/ddc-validator/src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index df766e3695547..77d1d894b2257 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -94,4 +94,20 @@ pub mod pallet { log::info!("Off-chain worker at block {:?}", block_number); } } + + impl Pallet { + /// Fetch the tasks related to current validator + fn fetch_tasks(validator: T::AccountId) -> Vec { + let mut cdn_nodes: Vec = vec![]; + for (cdn_id, cdn_tasks) in >::iter() { + for decision in cdn_tasks.iter() { + if decision.validator == validator { + cdn_nodes.push(cdn_id); + break; + } + } + } + cdn_nodes + } + } } From e4ff6e9901640ecaa0c28c722bfdc6cecd1899c0 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 1 Mar 2023 09:33:18 +0100 Subject: [PATCH 051/192] Create OCW, read from redis, compare bytes sent and bytes recieved, send tx onchain with the comparison Make http request from ocw Fix arbitary http request Try to connect to redis Clean up Try use redis via webdis Introduce structs Small improvements Code quality improvements Call offchain validator once per era Set shorted duration Update EPOCH_DURATION_IN_BLOCKS Fix short durations Integrade OCW to staking Specify ValidatorId type in OCW config trait bound Add logs Try fix issue with storage Increase points via on_initialize hook Try to send tx from offchain worker Read and store data from OCW Configure keys for OCW Use more advanced data storage Make save_validated_data with updated storage Revert "Fix short durations" This reverts commit d36efd54594d2c698e7c713d85120c6c5b86e9ca. Revert "Update EPOCH_DURATION_IN_BLOCKS" This reverts commit 87c67b138f3f593fffd5b745a4261c682dd0a24a. Revert "Set shorted duration" This reverts commit e5283c8202308364d32418ff4c483ef0b9996f0f. Fix logs for --chain=local Add signer to validation results and fix bug with KeyTypeId Validated fetched data and send tx Migrate to the latest pallet macro Remove ocw with old pallet macro --- Cargo.lock | 886 ++++++++---------- bin/node/runtime/Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 8 + .../dac-validator-offchain-worker/Cargo.toml | 58 ++ .../dac-validator-offchain-worker/src/lib.rs | 360 +++++++ frame/staking/src/lib.rs | 14 + frame/staking/src/pallet/impls.rs | 39 +- frame/staking/src/pallet/mod.rs | 13 +- 8 files changed, 888 insertions(+), 491 deletions(-) create mode 100644 frame/dac-validator-offchain-worker/Cargo.toml create mode 100644 frame/dac-validator-offchain-worker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0cf086b5577c0..3d241900ddcb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,7 @@ checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ "getrandom 0.2.3", "once_cell", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -255,7 +255,7 @@ dependencies = [ "fastrand", "futures-lite", "libc", - "log 0.4.14", + "log 0.4.17", "nb-connect", "once_cell", "parking", @@ -301,7 +301,7 @@ dependencies = [ "futures-lite", "gloo-timers", "kv-log-macro", - "log 0.4.14", + "log 0.4.17", "memchr", "num_cpus", "once_cell", @@ -369,7 +369,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4401f0a3622dad2e0763fa79e0eb328bc70fb7dccfdd645341f00d671247d6" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-sink", "futures-util", "memchr", @@ -382,7 +382,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0de5164e5edbf51c45fb8c2d9664ae1c095cce1b265ecf7569093c0d66ef690" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-sink", "futures-util", "memchr", @@ -479,6 +479,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -509,7 +515,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "hex", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-chain-spec", @@ -531,7 +537,6 @@ dependencies = [ "sp-finality-grandpa", "sp-keyring", "sp-keystore", - "sp-mmr-primitives", "sp-runtime", "sp-tracing", "strum", @@ -554,7 +559,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-rpc", @@ -574,7 +579,7 @@ dependencies = [ "env_logger 0.9.0", "hex", "hex-literal", - "log 0.4.14", + "log 0.4.17", "tiny-keccak", ] @@ -858,9 +863,15 @@ dependencies = [ [[package]] name = "bytes" -version = "1.1.0" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bzip2-sys" @@ -1188,11 +1199,11 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38faa2a16616c8e78a18d37b4726b98bfd2de192f2fdc8a39ddf568a408a0f75" +checksum = "9516ba6b2ba47b4cbf63b713f75b432fafa0a0e0464ec8381ec76e6efe931ab3" dependencies = [ - "cranelift-entity 0.82.3", + "cranelift-entity 0.80.0", ] [[package]] @@ -1206,7 +1217,7 @@ dependencies = [ "cranelift-codegen-shared 0.76.0", "cranelift-entity 0.76.0", "gimli 0.25.0", - "log 0.4.14", + "log 0.4.17", "regalloc 0.0.31", "smallvec 1.8.0", "target-lexicon", @@ -1214,17 +1225,17 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f192472a3ba23860afd07d2b0217dc628f21fcc72617aa1336d98e1671f33b" +checksum = "489e5d0081f7edff6be12d71282a8bf387b5df64d5592454b75d662397f2d642" dependencies = [ - "cranelift-bforest 0.82.3", - "cranelift-codegen-meta 0.82.3", - "cranelift-codegen-shared 0.82.3", - "cranelift-entity 0.82.3", + "cranelift-bforest 0.80.0", + "cranelift-codegen-meta 0.80.0", + "cranelift-codegen-shared 0.80.0", + "cranelift-entity 0.80.0", "gimli 0.26.1", - "log 0.4.14", - "regalloc 0.0.34", + "log 0.4.17", + "regalloc 0.0.33", "smallvec 1.8.0", "target-lexicon", ] @@ -1241,11 +1252,11 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32ddb89e9b89d3d9b36a5b7d7ea3261c98235a76ac95ba46826b8ec40b1a24" +checksum = "d36ee1140371bb0f69100e734b30400157a4adf7b86148dee8b0a438763ead48" dependencies = [ - "cranelift-codegen-shared 0.82.3", + "cranelift-codegen-shared 0.80.0", ] [[package]] @@ -1256,9 +1267,9 @@ checksum = "9dabb5fe66e04d4652e434195b45ae65b5c8172d520247b8f66d8df42b2b45dc" [[package]] name = "cranelift-codegen-shared" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01fd0d9f288cc1b42d9333b7a776b17e278fc888c28e6a0f09b5573d45a150bc" +checksum = "981da52d8f746af1feb96290c83977ff8d41071a7499e991d8abae0d4869f564" [[package]] name = "cranelift-entity" @@ -1268,9 +1279,9 @@ checksum = "3329733e4d4b8e91c809efcaa4faee80bf66f20164e3dd16d707346bd3494799" [[package]] name = "cranelift-entity" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3bfe172b83167604601faf9dc60453e0d0a93415b57a9c4d1a7ae6849185cf" +checksum = "a2906740053dd3bcf95ce53df0fd9b5649c68ae4bd9adada92b406f059eae461" dependencies = [ "serde", ] @@ -1282,47 +1293,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279afcc0d3e651b773f94837c3d581177b348c8d69e928104b2e9fccb226f921" dependencies = [ "cranelift-codegen 0.76.0", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "target-lexicon", ] [[package]] name = "cranelift-frontend" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a006e3e32d80ce0e4ba7f1f9ddf66066d052a8c884a110b91d05404d6ce26dce" +checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7" dependencies = [ - "cranelift-codegen 0.82.3", - "log 0.4.14", + "cranelift-codegen 0.80.0", + "log 0.4.17", "smallvec 1.8.0", "target-lexicon", ] [[package]] name = "cranelift-native" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501241b0cdf903412ec9075385ac9f2b1eb18a89044d1538e97fab603231f70c" +checksum = "166028ca0343a6ee7bddac0e70084e142b23f99c701bd6f6ea9123afac1a7a46" dependencies = [ - "cranelift-codegen 0.82.3", + "cranelift-codegen 0.80.0", "libc", "target-lexicon", ] [[package]] name = "cranelift-wasm" -version = "0.82.3" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d9e4211bbc3268042a96dd4de5bd979cda22434991d035f5f8eacba987fad2" +checksum = "5012a1cde0c8b3898770b711490d803018ae9bec2d60674ba0e5b2058a874f80" dependencies = [ - "cranelift-codegen 0.82.3", - "cranelift-entity 0.82.3", - "cranelift-frontend 0.82.3", + "cranelift-codegen 0.80.0", + "cranelift-entity 0.80.0", + "cranelift-frontend 0.80.0", "itertools", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", - "wasmparser 0.83.0", + "wasmparser 0.81.0", "wasmtime-types", ] @@ -1712,17 +1723,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" -dependencies = [ - "libc", - "redox_users 0.3.5", - "winapi 0.3.9", -] - [[package]] name = "dirs-sys" version = "0.3.6" @@ -1730,7 +1730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" dependencies = [ "libc", - "redox_users 0.4.0", + "redox_users", "winapi 0.3.9", ] @@ -1741,7 +1741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users 0.4.0", + "redox_users", "winapi 0.3.9", ] @@ -1890,12 +1890,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "enum-as-inner" version = "0.3.3" @@ -1977,7 +1971,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime 1.3.0", - "log 0.4.14", + "log 0.4.17", "regex", "termcolor", ] @@ -1988,7 +1982,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ - "log 0.4.14", + "log 0.4.17", "regex", ] @@ -2000,7 +1994,7 @@ checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime 2.1.0", - "log 0.4.14", + "log 0.4.17", "regex", "termcolor", ] @@ -2061,9 +2055,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.7.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" dependencies = [ "instant", ] @@ -2094,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" dependencies = [ "env_logger 0.7.1", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -2106,7 +2100,7 @@ dependencies = [ "either", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "num-traits", "parity-scale-codec", "parking_lot 0.11.2", @@ -2191,7 +2185,7 @@ dependencies = [ "frame-system", "hex-literal", "linregress", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "paste", "scale-info", @@ -2222,19 +2216,16 @@ dependencies = [ "itertools", "kvdb", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "memory-db", "parity-scale-codec", - "prettytable-rs", "rand 0.8.4", - "rand_pcg 0.3.1", "sc-block-builder", "sc-cli", "sc-client-api", "sc-client-db", "sc-executor", "sc-service", - "sc-sysinfo", "serde", "serde_json 1.0.79", "serde_nanos", @@ -2247,9 +2238,9 @@ dependencies = [ "sp-keystore", "sp-runtime", "sp-state-machine", + "sp-std", "sp-storage", "sp-trie", - "tempfile", "thousands", ] @@ -2346,7 +2337,7 @@ dependencies = [ "frame-support-procedural", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "once_cell", "parity-scale-codec", "parity-util-mem", @@ -2450,7 +2441,7 @@ version = "4.0.0-dev" dependencies = [ "criterion", "frame-support", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "serde", @@ -2717,7 +2708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" dependencies = [ "typenum", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -2785,7 +2776,7 @@ dependencies = [ "bitflags", "libc", "libgit2-sys", - "log 0.4.14", + "log 0.4.17", "url 2.2.1", ] @@ -2804,7 +2795,7 @@ dependencies = [ "aho-corasick", "bstr", "fnv", - "log 0.4.14", + "log 0.4.17", "regex", ] @@ -2838,7 +2829,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -2863,7 +2854,7 @@ version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ - "log 0.4.14", + "log 0.4.17", "pest", "pest_derive", "quick-error 2.0.0", @@ -2886,6 +2877,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + [[package]] name = "hashbrown" version = "0.11.2" @@ -3005,7 +3002,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "itoa 0.4.8", ] @@ -3016,7 +3013,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "http", "pin-project-lite 0.2.6", ] @@ -3073,7 +3070,7 @@ version = "0.14.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -3100,7 +3097,7 @@ dependencies = [ "ct-logs", "futures-util", "hyper 0.14.16", - "log 0.4.14", + "log 0.4.17", "rustls 0.19.1", "rustls-native-certs 0.5.0", "tokio", @@ -3114,7 +3111,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "hyper 0.14.16", "native-tls", "tokio", @@ -3182,7 +3179,7 @@ dependencies = [ "if-addrs", "ipnet", "libc", - "log 0.4.14", + "log 0.4.17", "winapi 0.3.9", ] @@ -3206,9 +3203,9 @@ dependencies = [ [[package]] name = "impl-trait-for-tuples" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" dependencies = [ "proc-macro2", "quote", @@ -3217,12 +3214,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg 1.0.1", - "hashbrown 0.11.2", + "hashbrown 0.9.1", "serde", ] @@ -3246,9 +3243,12 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.5.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6" +checksum = "f6ef6787e7f0faedc040f95716bdd0e62bcfcf4ba93da053b62dea2691c13864" +dependencies = [ + "winapi 0.3.9", +] [[package]] name = "iovec" @@ -3334,7 +3334,7 @@ dependencies = [ "hyper-tls", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "serde", "serde_json 1.0.79", "tokio", @@ -3351,7 +3351,7 @@ dependencies = [ "futures 0.3.21", "futures-executor", "futures-util", - "log 0.4.14", + "log 0.4.17", "serde", "serde_derive", "serde_json 1.0.79", @@ -3389,7 +3389,7 @@ dependencies = [ "hyper 0.14.16", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "net2", "parking_lot 0.11.2", "unicase 2.6.0", @@ -3404,7 +3404,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "parity-tokio-ipc", "parking_lot 0.11.2", "tower-service", @@ -3419,7 +3419,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.11.2", "rand 0.7.3", "serde", @@ -3431,12 +3431,12 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "globset", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.17", "tokio", "tokio-stream", "tokio-util 0.6.7", @@ -3452,7 +3452,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.17", "parity-ws", "parking_lot 0.11.2", "slab", @@ -3482,7 +3482,7 @@ dependencies = [ "jsonrpsee-types", "pin-project 1.0.10", "rustls-native-certs 0.6.1", - "soketto", + "soketto 0.7.1", "thiserror", "tokio", "tokio-rustls 0.23.2", @@ -3508,7 +3508,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json 1.0.79", - "soketto", + "soketto 0.7.1", "thiserror", "tokio", "tracing", @@ -3596,7 +3596,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -3628,7 +3628,7 @@ checksum = "ca7fbdfd71cd663dceb0faf3367a99f8cf724514933e9867cec4995b6027cbc1" dependencies = [ "fs-swap", "kvdb", - "log 0.4.14", + "log 0.4.17", "num_cpus", "owning_ref", "parity-util-mem", @@ -3702,9 +3702,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.1" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libp2p" @@ -3713,7 +3713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bec54343492ba5940a6c555e512c6721139835d28c59bc22febece72dfd0d9d" dependencies = [ "atomic", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "lazy_static", "libp2p-core", @@ -3762,7 +3762,7 @@ dependencies = [ "futures-timer", "lazy_static", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "multiaddr", "multihash 0.14.0", "multistream-select", @@ -3801,7 +3801,7 @@ dependencies = [ "async-std-resolver", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "smallvec 1.8.0", "trust-dns-resolver", ] @@ -3817,7 +3817,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3833,13 +3833,13 @@ dependencies = [ "asynchronous-codec 0.6.0", "base64 0.13.0", "byteorder", - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures 0.3.21", "hex_fmt", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3859,7 +3859,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "lru 0.6.6", "prost", "prost-build", @@ -3875,13 +3875,13 @@ checksum = "a2297dc0ca285f3a09d1368bde02449e539b46f94d32d53233f53f6625bcd3ba" dependencies = [ "arrayvec 0.5.2", "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "either", "fnv", "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.7.3", @@ -3907,7 +3907,7 @@ dependencies = [ "lazy_static", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "smallvec 1.8.0", "socket2 0.4.4", @@ -3935,10 +3935,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f2cd64ef597f40e14bfce0497f50ecb63dd6d201c61796daeb4227078834fbf" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "nohash-hasher", "parking_lot 0.11.2", "rand 0.7.3", @@ -3952,12 +3952,12 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8772c7a99088221bb7ca9c5c0574bf55046a7ab4c319f3619b275f28c8fb87a" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "curve25519-dalek 3.0.2", "futures 0.3.21", "lazy_static", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.8.4", @@ -3977,7 +3977,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "void", "wasm-timer", @@ -3990,10 +3990,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fba1a6ff33e4a274c89a3b1d78b9f34f32af13265cc5c46c16938262d4e945a" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "unsigned-varint 0.7.0", @@ -4007,7 +4007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1a458bbda880107b5b36fcb9b5a1ef0c329685da0e203ed692a8ebe64cc92c" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "rand 0.7.3", "salsa20", @@ -4021,12 +4021,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2852b61c90fa8ce3c8fcc2aba76e6cefc20d648f9df29157d6b3a916278ef3e3" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "futures-timer", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "prost", "prost-build", @@ -4048,7 +4048,7 @@ dependencies = [ "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", + "log 0.4.17", "prost", "prost-build", "rand 0.8.4", @@ -4066,12 +4066,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a877a4ced6d46bf84677e1974e8cf61fb434af73b2e96fb48d6cb6223a4634d8" dependencies = [ "async-trait", - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", "libp2p-core", "libp2p-swarm", - "log 0.4.14", - "lru 0.7.5", + "log 0.4.17", + "lru 0.7.3", "rand 0.7.3", "smallvec 1.8.0", "unsigned-varint 0.7.0", @@ -4087,7 +4087,7 @@ dependencies = [ "either", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "smallvec 1.8.0", "void", @@ -4117,7 +4117,7 @@ dependencies = [ "ipnet", "libc", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "socket2 0.4.4", ] @@ -4130,7 +4130,7 @@ dependencies = [ "async-std", "futures 0.3.21", "libp2p-core", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -4157,10 +4157,10 @@ dependencies = [ "futures 0.3.21", "futures-rustls", "libp2p-core", - "log 0.4.14", + "log 0.4.17", "quicksink", "rw-stream-sink", - "soketto", + "soketto 0.7.1", "url 2.2.1", "webpki-roots 0.21.0", ] @@ -4280,9 +4280,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.42" +version = "0.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7" +checksum = "a261afc61b7a5e323933b402ca6a1765183687c614789b1e4db7762ed4230bca" [[package]] name = "lite-json" @@ -4326,14 +4326,14 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.14", + "log 0.4.17", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", "value-bag", @@ -4371,9 +4371,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.5" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32613e41de4c47ab04970c348ca7ae7382cf116625755af070b008a15516a889" +checksum = "fcb87f3080f6d1d69e8c564c0fcfde1d7aa8cc451ce40cae89479111f03bc0eb" dependencies = [ "hashbrown 0.11.2", ] @@ -4567,7 +4567,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.14", + "log 0.4.17", "miow 0.2.2", "net2", "slab", @@ -4581,7 +4581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" dependencies = [ "libc", - "log 0.4.14", + "log 0.4.17", "miow 0.3.6", "ntapi", "winapi 0.3.9", @@ -4594,7 +4594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "slab", ] @@ -4712,9 +4712,9 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d91ec0a2440aaff5f78ec35631a7027d50386c6163aa975f7caa0d5da4b6ff8" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "pin-project 1.0.10", "smallvec 1.8.0", "unsigned-varint 0.7.0", @@ -4766,7 +4766,7 @@ checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "openssl", "openssl-probe", "openssl-sys", @@ -4823,7 +4823,7 @@ dependencies = [ "kvdb", "kvdb-rocksdb", "lazy_static", - "log 0.4.14", + "log 0.4.17", "node-primitives", "node-runtime", "node-testing", @@ -4861,7 +4861,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.21", "hex-literal", - "log 0.4.14", + "log 0.4.17", "nix", "node-executor", "node-inspect", @@ -4897,13 +4897,12 @@ dependencies = [ "sc-service", "sc-service-test", "sc-sync-state-rpc", - "sc-sysinfo", "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", "serde", "serde_json 1.0.79", - "soketto", + "soketto 0.4.2", "sp-api", "sp-authority-discovery", "sp-authorship", @@ -5033,7 +5032,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "frame-try-runtime", "hex-literal", - "log 0.4.14", + "log 0.4.17", "node-primitives", "pallet-authority-discovery", "pallet-authorship", @@ -5048,12 +5047,12 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", + "pallet-dac-validator-offchain-worker", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", "pallet-ddc-validator", "pallet-democracy", "pallet-election-provider-multi-phase", - "pallet-election-provider-support-benchmarking", "pallet-elections-phragmen", "pallet-erc20", "pallet-erc721", @@ -5063,8 +5062,6 @@ dependencies = [ "pallet-indices", "pallet-membership", "pallet-multisig", - "pallet-nomination-pools", - "pallet-nomination-pools-benchmarking", "pallet-offences", "pallet-offences-benchmarking", "pallet-proxy", @@ -5122,7 +5119,7 @@ dependencies = [ "frame-system", "fs_extra", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "node-executor", "node-primitives", "node-runtime", @@ -5168,7 +5165,7 @@ checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" dependencies = [ "memchr", "minimal-lexical", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -5288,9 +5285,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "oorandom" @@ -5482,7 +5479,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -5511,7 +5508,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -5539,7 +5536,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-bags-list", "pallet-staking", "remote-externalities", @@ -5558,7 +5555,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-transaction-payment", "parity-scale-codec", "scale-info", @@ -5597,9 +5594,10 @@ dependencies = [ "hex", "hex-literal", "k256", - "log 0.4.14", + "log 0.4.17", "pallet-beefy", "pallet-mmr", + "pallet-mmr-primitives", "pallet-session", "parity-scale-codec", "scale-info", @@ -5618,7 +5616,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-treasury", "parity-scale-codec", @@ -5669,7 +5667,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-bounties", "pallet-treasury", @@ -5688,7 +5686,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -5708,7 +5706,7 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-contracts-primitives", "pallet-contracts-proc-macro", @@ -5806,6 +5804,34 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-dac-validator-offchain-worker" +version = "0.0.1" +dependencies = [ + "alt_serde", + "frame-support", + "frame-system", + "hex", + "hex-literal", + "lite-json", + "pallet-balances", + "pallet-contracts", + "pallet-randomness-collective-flip", + "pallet-session", + "pallet-staking", + "pallet-timestamp", + "parity-scale-codec", + "pretty_assertions 0.6.1", + "scale-info", + "serde", + "serde_json 1.0.44", + "sp-core", + "sp-io", + "sp-keystore", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-ddc-metrics-offchain-worker" version = "2.0.1" @@ -5854,8 +5880,15 @@ version = "0.1.0" dependencies = [ "frame-support", "frame-system", + "log 0.4.17", + "pallet-ddc-staking", + "pallet-staking", "parity-scale-codec", "scale-info", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", ] [[package]] @@ -5884,9 +5917,8 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", - "pallet-election-provider-support-benchmarking", "parity-scale-codec", "parking_lot 0.12.0", "rand 0.7.3", @@ -5902,18 +5934,6 @@ dependencies = [ "strum", ] -[[package]] -name = "pallet-election-provider-support-benchmarking" -version = "4.0.0-dev" -dependencies = [ - "frame-benchmarking", - "frame-election-provider-support", - "frame-system", - "parity-scale-codec", - "sp-npos-elections", - "sp-runtime", -] - [[package]] name = "pallet-elections-phragmen" version = "5.0.0-dev" @@ -5921,7 +5941,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -5997,7 +6017,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -6042,7 +6062,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-session", "parity-scale-codec", @@ -6096,7 +6116,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -6115,11 +6135,27 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", + "pallet-mmr-primitives", "parity-scale-codec", "scale-info", "sp-core", "sp-io", - "sp-mmr-primitives", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-mmr-primitives" +version = "4.0.0-dev" +dependencies = [ + "frame-support", + "frame-system", + "hex-literal", + "log 0.4.17", + "parity-scale-codec", + "serde", + "sp-api", + "sp-core", "sp-runtime", "sp-std", ] @@ -6131,13 +6167,13 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", + "pallet-mmr-primitives", "parity-scale-codec", "serde", "serde_json 1.0.79", "sp-api", "sp-blockchain", "sp-core", - "sp-mmr-primitives", "sp-runtime", ] @@ -6178,52 +6214,12 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", - "scale-info", - "sp-core", - "sp-io", - "sp-runtime", - "sp-std", -] - -[[package]] -name = "pallet-nomination-pools" -version = "1.0.0" -dependencies = [ - "frame-support", - "frame-system", - "pallet-balances", - "parity-scale-codec", - "scale-info", - "sp-core", - "sp-io", - "sp-runtime", - "sp-staking", - "sp-std", - "sp-tracing", -] - -[[package]] -name = "pallet-nomination-pools-benchmarking" -version = "1.0.0" -dependencies = [ - "frame-benchmarking", - "frame-election-provider-support", - "frame-support", - "frame-system", - "pallet-bags-list", - "pallet-balances", - "pallet-nomination-pools", - "pallet-staking", - "pallet-staking-reward-curve", - "pallet-timestamp", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", "sp-io", "sp-runtime", - "sp-staking", "sp-std", ] @@ -6233,7 +6229,7 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6361,7 +6357,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-preimage", "parity-scale-codec", "scale-info", @@ -6394,7 +6390,7 @@ dependencies = [ "frame-support", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "pallet-timestamp", "parity-scale-codec", "scale-info", @@ -6455,7 +6451,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-authorship", "pallet-bags-list", "pallet-balances", @@ -6492,7 +6488,7 @@ dependencies = [ name = "pallet-staking-reward-fn" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "sp-arithmetic", ] @@ -6517,7 +6513,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -6535,7 +6531,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "pallet-treasury", "parity-scale-codec", @@ -6637,7 +6633,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6670,7 +6666,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.17", "pallet-balances", "parity-scale-codec", "scale-info", @@ -6709,7 +6705,7 @@ dependencies = [ "fs2", "hex", "libc", - "log 0.4.14", + "log 0.4.17", "lz4", "memmap2 0.2.1", "parking_lot 0.11.2", @@ -6757,7 +6753,7 @@ checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" dependencies = [ "futures 0.3.21", "libc", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "tokio", "winapi 0.3.9", @@ -6814,7 +6810,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "httparse", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "mio-extras", "rand 0.7.3", @@ -7115,7 +7111,7 @@ checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" dependencies = [ "cfg-if 0.1.10", "libc", - "log 0.4.14", + "log 0.4.17", "wepoll-sys", "winapi 0.3.9", ] @@ -7200,20 +7196,6 @@ dependencies = [ "output_vt100", ] -[[package]] -name = "prettytable-rs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" -dependencies = [ - "atty", - "csv", - "encode_unicode", - "lazy_static", - "term", - "unicode-width", -] - [[package]] name = "primitive-types" version = "0.11.1" @@ -7256,7 +7238,7 @@ dependencies = [ "proc-macro2", "quote", "syn", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -7267,7 +7249,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -7299,7 +7281,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "prost-derive", ] @@ -7309,11 +7291,11 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "heck 0.3.2", "itertools", "lazy_static", - "log 0.4.14", + "log 0.4.17", "multimap", "petgraph", "prost", @@ -7342,7 +7324,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "prost", ] @@ -7394,7 +7376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger 0.8.4", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", ] @@ -7695,17 +7677,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" -dependencies = [ - "getrandom 0.1.16", - "redox_syscall 0.1.57", - "rust-argon2", -] - [[package]] name = "redox_users" version = "0.4.0" @@ -7742,18 +7713,18 @@ version = "0.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" dependencies = [ - "log 0.4.14", + "log 0.4.17", "rustc-hash", "smallvec 1.8.0", ] [[package]] name = "regalloc" -version = "0.0.34" +version = "0.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" +checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" dependencies = [ - "log 0.4.14", + "log 0.4.17", "rustc-hash", "smallvec 1.8.0", ] @@ -7816,7 +7787,7 @@ dependencies = [ "env_logger 0.9.0", "frame-support", "jsonrpsee", - "log 0.4.14", + "log 0.4.17", "pallet-elections-phragmen", "parity-scale-codec", "serde", @@ -7922,18 +7893,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "rust-argon2" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" -dependencies = [ - "base64 0.13.0", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils 0.8.5", -] - [[package]] name = "rustc-demangle" version = "0.1.18" @@ -7972,9 +7931,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.33.5" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03627528abcc4a365554d32a9f3bbf67f7694c102cfeda792dc86a2d6057cc85" +checksum = "b2dcfc2778a90e38f56a708bfc90572422e11d6c7ee233d053d1f782cf9df6d2" dependencies = [ "bitflags", "errno", @@ -7991,7 +7950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", - "log 0.4.14", + "log 0.4.17", "ring", "sct 0.6.0", "webpki 0.21.4", @@ -8003,7 +7962,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" dependencies = [ - "log 0.4.14", + "log 0.4.17", "ring", "sct 0.7.0", "webpki 0.22.0", @@ -8102,7 +8061,7 @@ dependencies = [ name = "sc-allocator" version = "4.1.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "sp-core", "sp-wasm-interface", "thiserror", @@ -8117,7 +8076,7 @@ dependencies = [ "futures-timer", "ip_network", "libp2p", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "prost", "prost-build", @@ -8143,7 +8102,7 @@ version = "0.10.0-dev" dependencies = [ "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8214,7 +8173,7 @@ dependencies = [ "futures 0.3.21", "hex", "libp2p", - "log 0.4.14", + "log 0.4.17", "names", "parity-scale-codec", "rand 0.7.3", @@ -8249,7 +8208,7 @@ dependencies = [ "fnv", "futures 0.3.21", "hash-db", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-executor", @@ -8281,7 +8240,7 @@ dependencies = [ "kvdb-memorydb", "kvdb-rocksdb", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "parity-db", "parity-scale-codec", "parking_lot 0.12.0", @@ -8308,7 +8267,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "sc-client-api", "sc-utils", @@ -8330,7 +8289,7 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8368,7 +8327,7 @@ dependencies = [ "async-trait", "fork-tree", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "merlin", "num-bigint", "num-rational 0.2.4", @@ -8462,7 +8421,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-basic-authorship", "sc-client-api", @@ -8498,7 +8457,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-client-api", @@ -8522,7 +8481,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-consensus", @@ -8558,7 +8517,7 @@ dependencies = [ "env_logger 0.9.0", "hex-literal", "lazy_static", - "lru 0.7.5", + "lru 0.7.3", "parity-scale-codec", "parking_lot 0.12.0", "paste", @@ -8610,7 +8569,7 @@ dependencies = [ name = "sc-executor-wasmi" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-allocator", "sc-executor-common", @@ -8627,7 +8586,7 @@ version = "0.10.0-dev" dependencies = [ "cfg-if 1.0.0", "libc", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-wasm 0.42.2", "sc-allocator", @@ -8654,7 +8613,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "hex", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "rand 0.8.4", @@ -8698,7 +8657,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-block-builder", "sc-client-api", @@ -8722,7 +8681,7 @@ dependencies = [ "ansi_term 0.12.1", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-util-mem", "sc-client-api", "sc-network", @@ -8755,7 +8714,7 @@ dependencies = [ "async-trait", "asynchronous-codec 0.5.0", "bitflags", - "bytes 1.1.0", + "bytes 1.4.0", "cid", "either", "fnv", @@ -8767,8 +8726,8 @@ dependencies = [ "libp2p", "linked-hash-map", "linked_hash_set", - "log 0.4.14", - "lru 0.7.5", + "log 0.4.17", + "lru 0.7.3", "parity-scale-codec", "parking_lot 0.12.0", "pin-project 1.0.10", @@ -8811,8 +8770,8 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", - "lru 0.7.5", + "log 0.4.17", + "lru 0.7.3", "quickcheck", "sc-network", "sp-runtime", @@ -8830,7 +8789,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "rand 0.7.3", "sc-block-builder", @@ -8852,7 +8811,7 @@ dependencies = [ name = "sc-offchain" version = "4.0.0-dev" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", "futures 0.3.21", "futures-timer", @@ -8890,7 +8849,7 @@ version = "4.0.0-dev" dependencies = [ "futures 0.3.21", "libp2p", - "log 0.4.14", + "log 0.4.17", "rand 0.7.3", "sc-utils", "serde_json 1.0.79", @@ -8901,7 +8860,7 @@ dependencies = [ name = "sc-proposer-metrics" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "substrate-prometheus-endpoint", ] @@ -8915,7 +8874,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-pubsub", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -8951,7 +8910,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-chain-spec", @@ -8977,7 +8936,7 @@ dependencies = [ "jsonrpc-ipc-server", "jsonrpc-pubsub", "jsonrpc-ws-server", - "log 0.4.14", + "log 0.4.17", "serde_json 1.0.79", "substrate-prometheus-endpoint", "tokio", @@ -9010,7 +8969,7 @@ dependencies = [ "hash-db", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parking_lot 0.12.0", @@ -9028,7 +8987,6 @@ dependencies = [ "sc-offchain", "sc-rpc", "sc-rpc-server", - "sc-sysinfo", "sc-telemetry", "sc-tracing", "sc-transaction-pool", @@ -9072,7 +9030,7 @@ dependencies = [ "futures 0.3.21", "hex", "hex-literal", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "sc-block-builder", @@ -9104,7 +9062,7 @@ dependencies = [ name = "sc-state-db" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parity-util-mem-derive", @@ -9133,24 +9091,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "sc-sysinfo" -version = "6.0.0-dev" -dependencies = [ - "futures 0.3.21", - "libc", - "log 0.4.14", - "rand 0.7.3", - "rand_pcg 0.2.1", - "regex", - "sc-telemetry", - "serde", - "serde_json 1.0.79", - "sp-core", - "sp-io", - "sp-std", -] - [[package]] name = "sc-telemetry" version = "4.0.0-dev" @@ -9158,7 +9098,7 @@ dependencies = [ "chrono", "futures 0.3.21", "libp2p", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "pin-project 1.0.10", "rand 0.7.3", @@ -9178,7 +9118,7 @@ dependencies = [ "criterion", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "once_cell", "parking_lot 0.12.0", "regex", @@ -9219,7 +9159,7 @@ dependencies = [ "futures-timer", "hex", "linked-hash-map", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "parking_lot 0.12.0", @@ -9248,7 +9188,7 @@ name = "sc-transaction-pool-api" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "serde", "sp-blockchain", "sp-runtime", @@ -9262,7 +9202,7 @@ dependencies = [ "futures 0.3.21", "futures-timer", "lazy_static", - "log 0.4.14", + "log 0.4.17", "parking_lot 0.12.0", "prometheus", "tokio-test", @@ -9744,6 +9684,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "soketto" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c71ed3d54db0a699f4948e1bb3e45b450fa31fe602621dee6680361d569c88" +dependencies = [ + "base64 0.12.3", + "bytes 0.5.6", + "futures 0.3.21", + "httparse", + "log 0.4.17", + "rand 0.7.3", + "sha-1 0.9.4", +] + [[package]] name = "soketto" version = "0.7.1" @@ -9751,11 +9706,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.0", - "bytes 1.1.0", + "bytes 1.4.0", "flate2", "futures 0.3.21", "httparse", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "sha-1 0.9.4", ] @@ -9765,7 +9720,7 @@ name = "sp-api" version = "4.0.0-dev" dependencies = [ "hash-db", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-api-proc-macro", "sp-core", @@ -9794,7 +9749,7 @@ version = "2.0.1" dependencies = [ "criterion", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "rustversion", "sc-block-builder", @@ -9899,8 +9854,8 @@ name = "sp-blockchain" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "log 0.4.14", - "lru 0.7.5", + "log 0.4.17", + "lru 0.7.3", "parity-scale-codec", "parking_lot 0.12.0", "sp-api", @@ -9918,7 +9873,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-inherents", @@ -10023,7 +9978,7 @@ dependencies = [ "impl-serde", "lazy_static", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "merlin", "num-traits", "parity-scale-codec", @@ -10109,7 +10064,7 @@ name = "sp-finality-grandpa" version = "4.0.0-dev" dependencies = [ "finality-grandpa", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "serde", @@ -10142,7 +10097,7 @@ dependencies = [ "futures 0.3.21", "hash-db", "libsecp256k1", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parking_lot 0.12.0", "secp256k1", @@ -10195,21 +10150,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "sp-mmr-primitives" -version = "4.0.0-dev" -dependencies = [ - "hex-literal", - "log 0.4.14", - "parity-scale-codec", - "serde", - "sp-api", - "sp-core", - "sp-debug-derive", - "sp-runtime", - "sp-std", -] - [[package]] name = "sp-npos-elections" version = "4.0.0-dev" @@ -10273,7 +10213,7 @@ dependencies = [ "either", "hash256-std-hasher", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "parity-util-mem", "paste", @@ -10369,7 +10309,7 @@ name = "sp-sandbox" version = "0.10.0-dev" dependencies = [ "assert_matches", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-io", @@ -10416,7 +10356,7 @@ version = "0.12.0" dependencies = [ "hash-db", "hex-literal", - "log 0.4.14", + "log 0.4.17", "num-traits", "parity-scale-codec", "parking_lot 0.12.0", @@ -10454,7 +10394,7 @@ dependencies = [ name = "sp-tasks" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-core", "sp-externalities", @@ -10481,7 +10421,7 @@ version = "4.0.0-dev" dependencies = [ "async-trait", "futures-timer", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-api", "sp-inherents", @@ -10514,7 +10454,7 @@ name = "sp-transaction-storage-proof" version = "4.0.0-dev" dependencies = [ "async-trait", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-core", @@ -10576,7 +10516,7 @@ name = "sp-wasm-interface" version = "6.0.0" dependencies = [ "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sp-std", "wasmi", @@ -10732,7 +10672,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-rpc-api", @@ -10753,7 +10693,7 @@ version = "0.10.0-dev" dependencies = [ "futures-util", "hyper 0.14.16", - "log 0.4.14", + "log 0.4.17", "prometheus", "thiserror", "tokio", @@ -10766,7 +10706,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "sc-client-api", "sc-rpc-api", @@ -10817,7 +10757,7 @@ dependencies = [ "frame-system", "frame-system-rpc-runtime-api", "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "memory-db", "pallet-babe", "pallet-timestamp", @@ -10975,29 +10915,18 @@ checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" [[package]] name = "tempfile" -version = "3.3.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", - "fastrand", "libc", + "rand 0.8.4", "redox_syscall 0.2.10", "remove_dir_all", "winapi 0.3.9", ] -[[package]] -name = "term" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" -dependencies = [ - "byteorder", - "dirs", - "winapi 0.3.9", -] - [[package]] name = "termcolor" version = "1.1.2" @@ -11147,7 +11076,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "libc", "memchr", "mio 0.8.0", @@ -11190,7 +11119,7 @@ checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", "futures 0.1.31", - "log 0.4.14", + "log 0.4.17", ] [[package]] @@ -11223,7 +11152,7 @@ dependencies = [ "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", - "log 0.4.14", + "log 0.4.17", "mio 0.6.23", "num_cpus", "parking_lot 0.9.0", @@ -11297,7 +11226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" dependencies = [ "async-stream", - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "tokio", "tokio-stream", @@ -11320,10 +11249,10 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "futures-sink", - "log 0.4.14", + "log 0.4.17", "pin-project-lite 0.2.6", "tokio", ] @@ -11334,7 +11263,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "futures-core", "futures-io", "futures-sink", @@ -11364,7 +11293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", - "log 0.4.14", + "log 0.4.17", "pin-project-lite 0.2.6", "tracing-attributes", "tracing-core", @@ -11407,7 +11336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" dependencies = [ "lazy_static", - "log 0.4.14", + "log 0.4.17", "tracing-core", ] @@ -11480,7 +11409,7 @@ checksum = "d32d034c0d3db64b43c31de38e945f15b40cd4ca6d2dcfc26d4798ce8de4ab83" dependencies = [ "hash-db", "hashbrown 0.12.0", - "log 0.4.14", + "log 0.4.17", "rustc-hex", "smallvec 1.8.0", ] @@ -11520,7 +11449,7 @@ dependencies = [ "idna 0.2.2", "ipnet", "lazy_static", - "log 0.4.14", + "log 0.4.17", "rand 0.8.4", "smallvec 1.8.0", "thiserror", @@ -11538,7 +11467,7 @@ dependencies = [ "futures-util", "ipconfig", "lazy_static", - "log 0.4.14", + "log 0.4.17", "lru-cache", "parking_lot 0.11.2", "resolv-conf", @@ -11559,7 +11488,7 @@ version = "0.10.0-dev" dependencies = [ "clap 3.1.6", "jsonrpsee", - "log 0.4.14", + "log 0.4.17", "parity-scale-codec", "remote-externalities", "sc-chain-spec", @@ -11655,7 +11584,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] @@ -11723,7 +11652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35581ff83d4101e58b582e607120c7f5ffb17e632a980b1f38334d76b36908b2" dependencies = [ "asynchronous-codec 0.5.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures-io", "futures-util", ] @@ -11735,7 +11664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f8d425fafb8cd76bc3f22aace4af471d3156301d7508f2107e98fbeae10bc7f" dependencies = [ "asynchronous-codec 0.6.0", - "bytes 1.1.0", + "bytes 1.4.0", "futures-io", "futures-util", ] @@ -11771,11 +11700,12 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.0.0-alpha.6" +version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", + "version_check 0.9.2", ] [[package]] @@ -11798,9 +11728,9 @@ checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "void" @@ -11840,7 +11770,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.14", + "log 0.4.17", "try-lock", ] @@ -11874,7 +11804,7 @@ checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.14", + "log 0.4.17", "proc-macro2", "quote", "syn", @@ -11928,7 +11858,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c32691b6c7e6c14e7f8fd55361a9088b507aa49620fcd06c09b3a1082186b9" dependencies = [ - "log 0.4.14", + "log 0.4.17", "parity-wasm 0.32.0", "rustc-demangle", ] @@ -12204,33 +12134,34 @@ checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" [[package]] name = "wasmparser" -version = "0.83.0" +version = "0.81.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a" +checksum = "98930446519f63d00a836efdc22f67766ceae8dbcc1571379f2bcabc6b2b9abc" [[package]] name = "wasmtime" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ffb4705016d5ca91e18a72ed6822dab50e6d5ddd7045461b17ef19071cdef1" +checksum = "414be1bc5ca12e755ffd3ff7acc3a6d1979922f8237fc34068b2156cebcc3270" dependencies = [ "anyhow", "backtrace", "bincode", "cfg-if 1.0.0", + "cpp_demangle", "indexmap", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "object 0.27.1", - "once_cell", "paste", "psm", "rayon", "region 2.2.0", + "rustc-demangle", "serde", "target-lexicon", - "wasmparser 0.83.0", + "wasmparser 0.81.0", "wasmtime-cache", "wasmtime-cranelift", "wasmtime-environ", @@ -12241,16 +12172,16 @@ dependencies = [ [[package]] name = "wasmtime-cache" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c6ab24291fa7cb3a181f5669f6c72599b7ef781669759b45c7828c5999d0c0" +checksum = "8b9b4cd1949206fda9241faf8c460a7d797aa1692594d3dd6bc1cbfa57ee20d0" dependencies = [ "anyhow", "base64 0.13.0", "bincode", "directories-next", "file-per-thread-logger", - "log 0.4.14", + "log 0.4.17", "rustix", "serde", "sha2 0.9.8", @@ -12261,96 +12192,82 @@ dependencies = [ [[package]] name = "wasmtime-cranelift" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04c810078a491b7bc4866ebe045f714d2b95e6b539e1f64009a4a7606be11de" +checksum = "a4693d33725773615a4c9957e4aa731af57b27dca579702d1d8ed5750760f1a9" dependencies = [ "anyhow", - "cranelift-codegen 0.82.3", - "cranelift-entity 0.82.3", - "cranelift-frontend 0.82.3", + "cranelift-codegen 0.80.0", + "cranelift-entity 0.80.0", + "cranelift-frontend 0.80.0", "cranelift-native", "cranelift-wasm", "gimli 0.26.1", - "log 0.4.14", + "log 0.4.17", "more-asserts", "object 0.27.1", "target-lexicon", "thiserror", - "wasmparser 0.83.0", + "wasmparser 0.81.0", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61448266ea164b1ac406363cdcfac81c7c44db4d94c7a81c8620ac6c5c6cdf59" +checksum = "5b17e47116a078b9770e6fb86cff8b9a660826623cebcfff251b047c8d8993ef" dependencies = [ "anyhow", - "cranelift-entity 0.82.3", + "cranelift-entity 0.80.0", "gimli 0.26.1", "indexmap", - "log 0.4.14", + "log 0.4.17", "more-asserts", "object 0.27.1", "serde", "target-lexicon", "thiserror", - "wasmparser 0.83.0", + "wasmparser 0.81.0", "wasmtime-types", ] [[package]] name = "wasmtime-jit" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "156b4623c6b0d4b8c24afb846c20525922f538ef464cc024abab7ea8de2109a2" +checksum = "60ea5b380bdf92e32911400375aeefb900ac9d3f8e350bb6ba555a39315f2ee7" dependencies = [ "addr2line", "anyhow", "bincode", "cfg-if 1.0.0", - "cpp_demangle", "gimli 0.26.1", - "log 0.4.14", "object 0.27.1", "region 2.2.0", - "rustc-demangle", "rustix", "serde", "target-lexicon", "thiserror", "wasmtime-environ", - "wasmtime-jit-debug", "wasmtime-runtime", "winapi 0.3.9", ] -[[package]] -name = "wasmtime-jit-debug" -version = "0.35.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5dc31f811760a6c76b2672c404866fd19b75e5fb3b0075a3e377a6846490654" -dependencies = [ - "lazy_static", - "object 0.27.1", - "rustix", -] - [[package]] name = "wasmtime-runtime" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f907beaff69d4d920fa4688411ee4cc75c0f01859e424677f9e426e2ef749864" +checksum = "abc7cd79937edd6e238b337608ebbcaf9c086a8457f01dfd598324f7fa56d81a" dependencies = [ "anyhow", "backtrace", "cc", "cfg-if 1.0.0", "indexmap", + "lazy_static", "libc", - "log 0.4.14", + "log 0.4.17", "mach", "memoffset", "more-asserts", @@ -12359,20 +12276,19 @@ dependencies = [ "rustix", "thiserror", "wasmtime-environ", - "wasmtime-jit-debug", "winapi 0.3.9", ] [[package]] name = "wasmtime-types" -version = "0.35.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ef0e5fd197b9609dc9eb74beba0c84d5a12b2417cbae55534633329ba4852" +checksum = "d9e5e51a461a2cf2b69e1fc48f325b17d78a8582816e18479e8ead58844b23f8" dependencies = [ - "cranelift-entity 0.82.3", + "cranelift-entity 0.80.0", "serde", "thiserror", - "wasmparser 0.83.0", + "wasmparser 0.81.0", ] [[package]] @@ -12639,7 +12555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.17", "nohash-hasher", "parking_lot 0.11.2", "rand 0.8.4", @@ -12669,18 +12585,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.10.0+zstd.1.5.2" +version = "0.9.0+zstd.1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd" +checksum = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.4+zstd.1.5.2" +version = "4.1.1+zstd.1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee" +checksum = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079" dependencies = [ "libc", "zstd-sys", @@ -12688,9 +12604,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.3+zstd.1.5.2" +version = "1.6.1+zstd.1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" +checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33" dependencies = [ "cc", "libc", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 178490e3e7106..eaf555fbc4f3f 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -100,6 +100,7 @@ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../f pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } +pallet-dac-validator-offchain-worker = { version = "0.0.1", default-features = false, path = "../../../frame/dac-validator-offchain-worker" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 131614d8d7fa5..25dfc5d6d4802 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -82,6 +82,7 @@ pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; pub use pallet_ddc_validator; +pub use pallet_dac_validator_offchain_worker; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1217,6 +1218,12 @@ parameter_types! { pub const OcwBlockInterval: u32 = pallet_ddc_metrics_offchain_worker::BLOCK_INTERVAL; } +impl pallet_dac_validator_offchain_worker::Config for Runtime { + type Call = Call; + + type AuthorityId = pallet_dac_validator_offchain_worker::crypto::TestAuthId; +} + impl pallet_ddc_metrics_offchain_worker::Config for Runtime { type BlockInterval = OcwBlockInterval; @@ -1286,6 +1293,7 @@ construct_runtime!( Erc721: pallet_erc721::{Pallet, Call, Storage, Event}, Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, + DdcValidatorOffchainWorker: pallet_dac_validator_offchain_worker::{Pallet, Call, Storage}, DdcStaking: pallet_ddc_staking, DdcValidator: pallet_ddc_validator, } diff --git a/frame/dac-validator-offchain-worker/Cargo.toml b/frame/dac-validator-offchain-worker/Cargo.toml new file mode 100644 index 0000000000000..7f978de7fe981 --- /dev/null +++ b/frame/dac-validator-offchain-worker/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "pallet-dac-validator-offchain-worker" +version = "0.0.1" +authors = ["Parity Technologies "] +edition = "2021" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet for offchain worker" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["full"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +serde = { version = "1.0.101", optional = true } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } +lite-json = { version = "0.2.0", default-features = false } +alt_serde = { version = "1", default-features = false, features = ["derive"] } +serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } +hex-literal = "^0.3.1" +hex = { version = "0.4", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } +pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-keystore", + "frame-support/std", + "frame-system/std", + "serde", + "lite-json/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", + "pallet-contracts/std", + "pallet-staking/std", + "pallet-session/std", + # "pallet-contracts-rpc-runtime-api/std", +] + +[dev-dependencies] +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } +pretty_assertions = "0.6.1" \ No newline at end of file diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs new file mode 100644 index 0000000000000..5670f9689dcb7 --- /dev/null +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -0,0 +1,360 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use alloc::{format, string::String}; +use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +use frame_support::{ + decl_event, decl_module, decl_storage, + log::{error, info, warn}, + traits::Currency, + weights::{Weight}, + dispatch::DispatchResult, + RuntimeDebug, + BoundedVec, + parameter_types, +}; +use frame_system::{ensure_signed, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; +use pallet_staking::{self as staking}; +use sp_core::crypto::{KeyTypeId, UncheckedFrom};use sp_io::offchain::timestamp; +use sp_runtime::offchain::{http, Duration}; +use sp_std::{vec, vec::Vec}; +extern crate alloc; +use pallet_session as session; + +parameter_types! { + pub DdcValidatorsQuorumSize: u32 = 3; +} + +use core::fmt::Debug; +use scale_info::TypeInfo; + +type BalanceOf = <::Currency as Currency< + ::AccountId, +>>::Balance; + +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); + +pub const HTTP_TIMEOUT_MS: u64 = 30_000; + +const TIME_START_MS: u64 = 1_672_531_200_000; +const ERA_DURATION_MS: u64 = 120_000; +const ERA_IN_BLOCKS: u8 = 20; + +const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; + +type ResultStr = Result; + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct RedisFtAggregate { + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: (u32, Vec), +} + +#[derive(Clone)] +struct BytesSent { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesSent { + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let (_, values) = aggregate.ft_aggregate; + + BytesSent { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } +} + +#[derive(Clone)] +struct BytesReceived { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesReceived { + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let (_, values) = aggregate.ft_aggregate; + + BytesReceived { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } +} + +#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] +pub struct ValidationResult { + block_number: BlockNumber, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, +} + +// use sp_std::fmt; +// impl fmt::Display for ValidationResult +// { +// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +// write!(f, "ValidationResult signer {}", self.signer) +// } +// } + +pub mod crypto { + use super::KEY_TYPE; + use frame_system::offchain::AppCrypto; + use sp_core::sr25519::Signature as Sr25519Signature; + use sp_runtime::{ + app_crypto::{app_crypto, sr25519}, + traits::Verify, + }; + app_crypto!(sr25519, KEY_TYPE); + + use sp_runtime::{MultiSignature, MultiSigner}; + + pub struct TestAuthId; + + impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } + + impl AppCrypto for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::without_storage_info] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: + frame_system::Config + + pallet_contracts::Config + + pallet_session::Config::AccountId> + + pallet_staking::Config + + CreateSignedTransaction> + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + /// The overarching dispatch call type. + type Call: From>; + + type AuthorityId: AppCrypto; + } + + #[pallet::hooks] + impl Hooks> for Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + fn offchain_worker(block_number: T::BlockNumber) { + let res = Self::offchain_worker_main(block_number); + + match res { + Ok(()) => info!("[DDC Validator] DDC Validator is suspended."), + Err(err) => error!("[DDC Validator] Error in Offchain Worker: {}", err), + }; + } + } + + #[pallet::call] + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + #[pallet::weight(1000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, block_number: T::BlockNumber) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + info!("author: {:?}", signer); + let mut v_results = ValidationResults::::get(); + + let cur_validation = ValidationResult:: { + block_number, + val_res, + cdn_node_pub_key, + signer, + }; + + v_results.push(cur_validation); + + ValidationResults::::set(v_results); + + Ok(()) + } + } + + #[pallet::storage] + #[pallet::getter(fn validation_results)] + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; +} + +impl Pallet +where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, +{ + fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { + return Ok(()) + } + + let signer = match Self::get_signer() { + Err(e) => { + warn!("{:?}", e); + return Ok(()); + } + Ok(signer) => signer, + }; + + info!("ValidationResults: {:?}", ValidationResults::::get()); + + // Read data from DataModel and do dumb validation + let current_era = Self::get_current_era(); + let (bytes_sent, bytes_received) = Self::fetch_data(current_era); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let cdn_node_pub_key = bytes_sent.node_public_key.clone(); + let tx_res = signer.send_signed_transaction(|_acct| { + info!("Sending save_validated_data tx"); + + // This is the on-chain function + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), block_number } + }); + + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting save_validated_data TX") + } + Some((_, Ok(()))) => {} + } + + info!("save_validated_data: {:?}", ValidationResults::::get()); + + + Ok(()) + } + + fn get_signer() -> ResultStr> { + let signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + return Err("[DDC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + } + Ok(signer) + } + + fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { + return if bytes_sent.sum == bytes_received.sum { + true + } else { + false + } + } + + fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + info!("[DDC Validator] DDC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::new(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::new(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_current_era() -> u64 { + // info!("timestamp: {}", timestamp().unix_millis()); + // info!("TIME_START_MS: {}", TIME_START_MS); + // info!("ERA_DURATION_MS: {}", ERA_DURATION_MS); + // info!("1: {}", timestamp().unix_millis() - TIME_START_MS); + // info!("2: {}", (timestamp().unix_millis() - TIME_START_MS) % ERA_DURATION_MS); + + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + } + + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + } + + fn http_get_json(url: &str) -> ResultStr { + let body = Self::http_get_request(url).map_err(|err| { + error!("[DDC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" + })?; + + let parsed = serde_json::from_slice(&body).map_err(|err| { + warn!("[DDC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" + }); + + parsed + } + + fn http_get_request(http_url: &str) -> Result, http::Error> { + info!("[DDC Validator] Sending request to: {:?}", http_url); + + // Initiate an external HTTP GET request. This is using high-level wrappers from + // `sp_runtime`. + let request = http::Request::get(http_url); + + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + warn!("[DDC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + // Next we fully read the response body and collect it to a vector of bytes. + Ok(response.body().collect::>()) + } + + fn validators() -> Vec<::ValidatorId> { + >::validators() + } + + /// Reward a validator. + pub fn reward_by_ids( + validators_points: impl IntoIterator, + ) -> Result<(), ()> { + >::reward_by_ids(validators_points); + Ok(()) + } +} \ No newline at end of file diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 037dcc86ab2a8..61f093c8b26bd 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -380,6 +380,20 @@ impl Default for EraRewardPoints { } } +#[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] +pub struct EraDacRewardPoints { + /// Total number of points. Equals the sum of reward points for each validator. + total: RewardPoint, + /// The reward points earned by a given validator. + individual: BTreeMap, +} + +impl Default for EraDacRewardPoints { + fn default() -> Self { + EraDacRewardPoints { total: Default::default(), individual: BTreeMap::new() } + } +} + /// Indicates the initial status of the staker. #[derive(RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, Clone))] diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index f6d60a27b3fcd..7485765ae0b3a 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -16,12 +16,13 @@ // limitations under the License. //! Implementations for the Staking FRAME Pallet. - use frame_election_provider_support::{ data_provider, ElectionDataProvider, ElectionProvider, ScoreProvider, SortedListProvider, Supports, VoteWeight, VoterOf, }; use frame_support::{ + decl_event, decl_module, decl_storage, + log::{error, info, warn}, pallet_prelude::*, traits::{ Currency, CurrencyToVote, Defensive, EstimateNextNewSession, Get, Imbalance, @@ -44,7 +45,7 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, Exposure, ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, - SessionInterface, StakingLedger, ValidatorPrefs, + SessionInterface, StakingLedger, ValidatorPrefs, EraDacRewardPoints, }; use super::{pallet::*, STAKING_ID}; @@ -627,6 +628,40 @@ impl Pallet { era_rewards.total += points; } }); + let erasRewardPoints = >::get(active_era.index); + } + } + + pub fn reward_by_ids2(validators_points: impl IntoIterator) { + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + let mut p = EraDacRewardPoints { + total: era_rewards.total.clone(), + individual: era_rewards.individual.clone(), + }; + for (validator, points) in validators_points.into_iter() { + *p.individual.entry(validator).or_default() += points; + p.total += points; + } + return p; + }); + let dacRewardPoints = >::get(active_era.index); + } + } + + pub fn get_reward_points(validators_points: impl IntoIterator) { + info!("Hello from get_reward_points"); + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + for (validator, points) in validators_points.into_iter() { + // info!("validator: {:?}, points: {:?}", validator, points); + // *era_rewards.individual.entry(validator).or_default() += points; + info!("reading era_rewards.total: {:?}", era_rewards.total); + // era_rewards.total += points; + } + }); + let dacRewardPoints = >::get(active_era.index); + info!("reading dacRewardPoints: {:?}", dacRewardPoints); } } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 5af51be85e3c7..65058d6b3998d 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -40,10 +40,10 @@ mod impls; pub use impls::*; use crate::{ - slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, EraRewardPoints, Exposure, - Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, Releases, - RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, - ValidatorPrefs, + slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraDacRewardPoints, EraPayout, + EraRewardPoints, Exposure, Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, + PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakingLedger, + UnappliedSlash, UnlockChunk, ValidatorPrefs, }; const STAKING_ID: LockIdentifier = *b"staking "; @@ -399,6 +399,11 @@ pub mod pallet { pub type ErasRewardPoints = StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; + #[pallet::storage] + #[pallet::getter(fn eras_dac_reward_points)] + pub type ErasDacRewardPoints = + StorageMap<_, Twox64Concat, EraIndex, EraDacRewardPoints, ValueQuery>; + /// The total amount staked for the last `HISTORY_DEPTH` eras. /// If total hasn't been set or has been removed then 0 stake is returned. #[pallet::storage] From 3f57edcf2307cd7db5efff1ab7db04e4a3621c6a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 11:54:31 +0600 Subject: [PATCH 052/192] Assign a random DAC validator for each edge --- bin/node/runtime/src/lib.rs | 1 + frame/ddc-validator/src/lib.rs | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 25dfc5d6d4802..549d5e911e4bf 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1242,6 +1242,7 @@ impl pallet_ddc_staking::Config for Runtime { impl pallet_ddc_validator::Config for Runtime { type Event = Event; + type Randomness = RandomnessCollectiveFlip; } construct_runtime!( diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 77d1d894b2257..bb2dd76411c31 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,6 +1,8 @@ #![cfg_attr(not(feature = "std"), no_std)] -pub use frame_support::{pallet_prelude::*, parameter_types, weights::Weight, BoundedVec}; +pub use frame_support::{ + pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, +}; pub use frame_system::pallet_prelude::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; @@ -34,6 +36,7 @@ pub mod pallet { #[pallet::config] pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { type Event: From> + IsType<::Event>; + type Randomness: Randomness; } #[pallet::storage] @@ -79,15 +82,34 @@ pub mod pallet { impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; let edges: Vec = >::iter_keys().collect(); log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators: {:?}, edges: {:?}", + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, >::get(), >::get(), + validators_count, validators, edges, ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); + } + Tasks::::insert(edge, decisions); + } 0 } fn offchain_worker(block_number: T::BlockNumber) { @@ -109,5 +131,31 @@ pub mod pallet { } cdn_nodes } + + fn choose(total: u32) -> Option { + if total == 0 { + return None + } + let mut random_number = Self::generate_random_number(0); + + // Best effort attempt to remove bias from modulus operator. + for i in 1..128 { + if random_number < u32::MAX - u32::MAX % total { + break + } + + random_number = Self::generate_random_number(i); + } + + Some(random_number % total) + } + + fn generate_random_number(seed: u32) -> u32 { + let (random_seed, _) = T::Randomness::random(&(b"ddc-validator", seed).encode()); + let random_number = ::decode(&mut random_seed.as_ref()) + .expect("secure hashes should always be bigger than u32; qed"); + + random_number + } } } From 1b203e285ba6c721e5fc20d6e1857fcead8176a3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 12:29:00 +0600 Subject: [PATCH 053/192] Update Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3d241900ddcb1..dc15ef872f506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11705,7 +11705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", - "version_check 0.9.2", + "version_check 0.9.4", ] [[package]] From ab2d07fc545c6c3af905ef93f364cf899add4a1b Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 14:27:15 +0600 Subject: [PATCH 054/192] Assign DAC validation tasks on era increment --- frame/ddc-validator/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index bb2dd76411c31..a88d229d435e7 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -81,6 +81,19 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { + match (>::get(), >::get()) { + (Some(global_era_counter), Some(last_managed_era)) => { + if last_managed_era >= global_era_counter { + return 0 + } + >::put(global_era_counter); + }, + (Some(global_era_counter), None) => { + >::put(global_era_counter); + }, + _ => { return 0 }, + }; + let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; let edges: Vec = >::iter_keys().collect(); From 11b3f218a731a08cd54b849d258535c52cd8fae6 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 20 Mar 2023 14:54:08 +0600 Subject: [PATCH 055/192] Format DAC Validator files --- frame/ddc-validator/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a88d229d435e7..344048de95113 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -4,9 +4,9 @@ pub use frame_support::{ pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, }; pub use frame_system::pallet_prelude::*; +pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; -pub use pallet::*; pub use sp_std::prelude::*; parameter_types! { @@ -91,7 +91,7 @@ pub mod pallet { (Some(global_era_counter), None) => { >::put(global_era_counter); }, - _ => { return 0 }, + _ => return 0, }; let validators: Vec = >::iter_keys().collect(); @@ -125,6 +125,7 @@ pub mod pallet { } 0 } + fn offchain_worker(block_number: T::BlockNumber) { log::info!("Off-chain worker at block {:?}", block_number); } @@ -138,7 +139,7 @@ pub mod pallet { for decision in cdn_tasks.iter() { if decision.validator == validator { cdn_nodes.push(cdn_id); - break; + break } } } From 09ccc057075b336e329ed9bd529ee1b2e5dbbbee Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Tue, 21 Mar 2023 15:15:39 +0100 Subject: [PATCH 056/192] adjust era --- frame/ddc-validator/src/lib.rs | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 344048de95113..10f72a1ac84ff 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -8,11 +8,16 @@ pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; pub use sp_std::prelude::*; +pub use sp_io::offchain::timestamp; parameter_types! { pub DdcValidatorsQuorumSize: u32 = 3; } +const TIME_START_MS: u64 = 1_672_531_200_000; +const ERA_DURATION_MS: u64 = 120_000; +const ERA_IN_BLOCKS: u8 = 20; + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { ProofOfDelivery, @@ -48,10 +53,6 @@ pub mod pallet { BoundedVec, DdcValidatorsQuorumSize>, >; - #[pallet::storage] - #[pallet::getter(fn global_era_counter)] - pub type GlobalEraCounter = StorageValue<_, u32>; - #[pallet::storage] #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, u32>; @@ -63,21 +64,6 @@ pub mod pallet { #[pallet::error] pub enum Error {} - #[pallet::call] - impl Pallet { - #[pallet::weight(100_000)] - pub fn inc_era(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - if let Some(era) = >::get() { - let new_era = era.checked_add(1).unwrap_or_default(); - >::put(new_era); - } else { - >::put(1); - } - Ok(()) - } - } - #[pallet::hooks] impl Hooks> for Pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { @@ -100,7 +86,7 @@ pub mod pallet { log::info!( "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, - >::get(), + Self::get_current_era(), >::get(), validators_count, validators, @@ -170,6 +156,9 @@ pub mod pallet { .expect("secure hashes should always be bigger than u32; qed"); random_number + // Get the current era; Shall we start era count from 0 or from 1? + fn get_current_era() -> u64 { + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS } } } From 1215c4756234103d8bf122bae16f96b76f6d12e3 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Tue, 21 Mar 2023 15:40:37 +0100 Subject: [PATCH 057/192] fix era --- frame/ddc-validator/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 10f72a1ac84ff..3d708a0a132c2 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -156,6 +156,8 @@ pub mod pallet { .expect("secure hashes should always be bigger than u32; qed"); random_number + } + // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS From 0319c85be440acdbede5039cd3aeac5e426fc39d Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 21 Mar 2023 10:45:34 +0100 Subject: [PATCH 058/192] Prepare for demo --- .../dac-validator-offchain-worker/src/lib.rs | 110 +++++++++++++----- 1 file changed, 83 insertions(+), 27 deletions(-) diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs index 5670f9689dcb7..161a646514e13 100644 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -49,9 +49,20 @@ type ResultStr = Result; #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec), + pub ft_aggregate: (u32, Vec, Vec), + + // This struct should be correct for any length but there is error while parsing JSON + // #[serde(rename = "FT.AGGREGATE")] + // pub ft_aggregate: Vec, } +// #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +// #[serde(crate = "alt_serde")] +// pub enum FtAggregate { +// Length(u32), +// Node(Vec), +// } + #[derive(Clone)] struct BytesSent { node_public_key: String, @@ -61,7 +72,7 @@ struct BytesSent { impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values) = aggregate.ft_aggregate; + let (_, values, values2) = aggregate.ft_aggregate; BytesSent { node_public_key: values[1].clone(), @@ -71,6 +82,24 @@ impl BytesSent { } } +// impl BytesSent { +// pub fn new(aggregate: RedisFtAggregate) -> BytesSent { +// // let (_, values, values2) = aggregate.ft_aggregate; +// let data = aggregate.ft_aggregate[1].clone(); +// +// match data { +// FtAggregate::Node(node) => { +// return BytesSent { +// node_public_key: node[1].clone(), +// era: node[3].clone(), +// sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), +// } +// } +// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), +// } +// } +// } + #[derive(Clone)] struct BytesReceived { node_public_key: String, @@ -80,7 +109,7 @@ struct BytesReceived { impl BytesReceived { pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values) = aggregate.ft_aggregate; + let (_, values, values2) = aggregate.ft_aggregate; BytesReceived { node_public_key: values[1].clone(), @@ -90,9 +119,28 @@ impl BytesReceived { } } +// impl BytesReceived { +// pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { +// // let (_, values, values2) = aggregate.ft_aggregate; +// +// let data = aggregate.ft_aggregate[1].clone(); +// +// match data { +// FtAggregate::Node(node) => { +// return BytesReceived { +// node_public_key: node[1].clone(), +// era: node[3].clone(), +// sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), +// } +// } +// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), +// } +// } +// } + #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - block_number: BlockNumber, +pub struct ValidationResult { + era: String, signer: AccountId, val_res: bool, cdn_node_pub_key: String, @@ -173,8 +221,8 @@ pub mod pallet { let res = Self::offchain_worker_main(block_number); match res { - Ok(()) => info!("[DDC Validator] DDC Validator is suspended."), - Err(err) => error!("[DDC Validator] Error in Offchain Worker: {}", err), + Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), }; } } @@ -185,20 +233,26 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { - #[pallet::weight(1000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, block_number: T::BlockNumber) -> DispatchResult { + #[pallet::weight(10000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; - info!("author: {:?}", signer); + info!("[DAC Validator] author: {:?}", signer); let mut v_results = ValidationResults::::get(); - let cur_validation = ValidationResult:: { - block_number, + let cur_validation = ValidationResult:: { + era, val_res, cdn_node_pub_key, signer, }; + // ValidationResults::::append(cur_validation); + // ValidationResults::::mutate(|results| { + // results.push(cur_validation); + // + // results.clone() + // }); v_results.push(cur_validation); ValidationResults::::set(v_results); @@ -209,7 +263,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; } impl Pallet @@ -218,6 +272,8 @@ where as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { return Ok(()) } @@ -230,19 +286,19 @@ where Ok(signer) => signer, }; - info!("ValidationResults: {:?}", ValidationResults::::get()); + info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era(); + let current_era = Self::get_current_era() - 1u64; let (bytes_sent, bytes_received) = Self::fetch_data(current_era); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let cdn_node_pub_key = bytes_sent.node_public_key.clone(); let tx_res = signer.send_signed_transaction(|_acct| { - info!("Sending save_validated_data tx"); + info!("[DAC Validator] Sending save_validated_data tx"); // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), block_number } + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } }); match &tx_res { @@ -252,8 +308,7 @@ where Some((_, Ok(()))) => {} } - info!("save_validated_data: {:?}", ValidationResults::::get()); - + // info!("[DAC Validator] save_validated_data: {:?}", ValidationResults::::get()); Ok(()) } @@ -261,8 +316,9 @@ where fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { - return Err("[DDC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); } + Ok(signer) } @@ -275,18 +331,18 @@ where } fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ - info!("[DDC Validator] DDC Validator is running. Current era is {}", era); + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("Bytes sent sum is fetched: {:?}", bytes_sent_res); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error let bytes_received_query = Self::get_bytes_received_query_url(era); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); - info!("Bytes received sum is fetched:: {:?}", bytes_received_res); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); (bytes_sent, bytes_received) @@ -312,12 +368,12 @@ where fn http_get_json(url: &str) -> ResultStr { let body = Self::http_get_request(url).map_err(|err| { - error!("[DDC Validator] Error while getting {}: {:?}", url, err); + error!("[DAC Validator] Error while getting {}: {:?}", url, err); "HTTP GET error" })?; let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DDC Validator] Error while parsing JSON from {}: {:?}", url, err); + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); "HTTP JSON parse error" }); @@ -325,7 +381,7 @@ where } fn http_get_request(http_url: &str) -> Result, http::Error> { - info!("[DDC Validator] Sending request to: {:?}", http_url); + info!("[DAC Validator] Sending request to: {:?}", http_url); // Initiate an external HTTP GET request. This is using high-level wrappers from // `sp_runtime`. @@ -338,7 +394,7 @@ where let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; if response.code != 200 { - warn!("[DDC Validator] http_get_request unexpected status code: {}", response.code); + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); return Err(http::Error::Unknown) } From 495f40a1f08e7b13665193bd141bfbb54b3a549f Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 11:32:21 +0100 Subject: [PATCH 059/192] merge offchain worker with ddc validator --- Cargo.lock | 7 + bin/node/runtime/src/lib.rs | 2 + frame/ddc-validator/Cargo.toml | 12 ++ frame/ddc-validator/src/lib.rs | 322 +++++++++++++++++++++++++++++++-- 4 files changed, 326 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc15ef872f506..6359b04d3e23c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5878,14 +5878,21 @@ dependencies = [ name = "pallet-ddc-validator" version = "0.1.0" dependencies = [ + "alt_serde", "frame-support", "frame-system", "log 0.4.17", + "pallet-contracts", "pallet-ddc-staking", + "pallet-session", "pallet-staking", "parity-scale-codec", "scale-info", + "serde", + "serde_json 1.0.44", + "sp-core", "sp-io", + "sp-keystore", "sp-runtime", "sp-staking", "sp-std", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 549d5e911e4bf..ba7f7f460155a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1243,6 +1243,8 @@ impl pallet_ddc_staking::Config for Runtime { impl pallet_ddc_validator::Config for Runtime { type Event = Event; type Randomness = RandomnessCollectiveFlip; + type Call = Call; + type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; } construct_runtime!( diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 057a274a2ec1b..55172f6848949 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -4,13 +4,20 @@ version = "0.1.0" edition = "2021" [dependencies] +alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } log = { version = "0.4.17", default-features = false } +pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } +pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.101", optional = true } +serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } @@ -22,9 +29,14 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", + "pallet-contracts/std", "pallet-ddc-staking/std", "pallet-staking/std", + "pallet-session/std", "scale-info/std", + "serde", + "sp-keystore", + "sp-core/std", "sp-io/std", "sp-runtime/std", "sp-staking/std", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 3d708a0a132c2..9cab6656ca750 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,23 +1,53 @@ #![cfg_attr(not(feature = "std"), no_std)] +pub use alloc::{format, string::String}; +pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +pub use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +pub use core::fmt::Debug; pub use frame_support::{ - pallet_prelude::*, parameter_types, traits::Randomness, weights::Weight, BoundedVec, + decl_event, decl_module, decl_storage, + log::{error, info, warn}, + pallet_prelude::*, + traits::{Randomness, Currency}, + weights::Weight, + dispatch::DispatchResult, + RuntimeDebug, + BoundedVec, + parameter_types, }; -pub use frame_system::pallet_prelude::*; +pub use frame_system::{ensure_signed, pallet_prelude::*, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_staking::{self as staking}; +pub use pallet_session as session; +pub use scale_info::TypeInfo; +pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; +pub use sp_runtime::offchain::{http, Duration}; pub use sp_std::prelude::*; pub use sp_io::offchain::timestamp; +extern crate alloc; parameter_types! { pub DdcValidatorsQuorumSize: u32 = 3; } +type BalanceOf = <::Currency as Currency< + ::AccountId, +>>::Balance; + +type ResultStr = Result; + + +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); + +pub const HTTP_TIMEOUT_MS: u64 = 30_000; + const TIME_START_MS: u64 = 1_672_531_200_000; const ERA_DURATION_MS: u64 = 120_000; const ERA_IN_BLOCKS: u8 = 20; +const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { ProofOfDelivery, @@ -30,18 +60,112 @@ pub struct Decision { pub validator: AccountId, } +#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] +pub struct ValidationResult { + era: String, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, +} + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct RedisFtAggregate { + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: (u32, Vec, Vec), +} + +#[derive(Clone)] +struct BytesSent { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesSent { + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let (_, values, values2) = aggregate.ft_aggregate; + + BytesSent { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } +} + +#[derive(Clone)] +struct BytesReceived { + node_public_key: String, + era: String, + sum: u32, +} + +impl BytesReceived { + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let (_, values, values2) = aggregate.ft_aggregate; + + BytesReceived { + node_public_key: values[1].clone(), + era: values[3].clone(), + sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } +} + +pub mod crypto { + use super::KEY_TYPE; + use frame_system::offchain::AppCrypto; + use sp_core::sr25519::Signature as Sr25519Signature; + use sp_runtime::{ + app_crypto::{app_crypto, sr25519}, + traits::Verify, + }; + app_crypto!(sr25519, KEY_TYPE); + + use sp_runtime::{MultiSignature, MultiSigner}; + + pub struct TestAuthId; + + impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } + + impl AppCrypto for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } +} + #[frame_support::pallet] pub mod pallet { use super::*; #[pallet::pallet] + #[pallet::without_storage_info] #[pallet::generate_store(pub(super) trait Store)] pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config + pallet_staking::Config + ddc_staking::Config { + pub trait Config: + frame_system::Config + + pallet_contracts::Config + + pallet_session::Config::AccountId> + + pallet_staking::Config + + ddc_staking::Config + + CreateSignedTransaction> + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { type Event: From> + IsType<::Event>; type Randomness: Randomness; + type Call: From>; + type AuthorityId: AppCrypto; } #[pallet::storage] @@ -55,26 +179,34 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn last_managed_era)] - pub type LastManagedEra = StorageValue<_, u32>; + pub type LastManagedEra = StorageValue<_, u64>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event {} + pub enum Event + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + {} #[pallet::error] pub enum Error {} #[pallet::hooks] - impl Hooks> for Pallet { + impl Hooks> for Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { fn on_initialize(block_number: T::BlockNumber) -> Weight { - match (>::get(), >::get()) { - (Some(global_era_counter), Some(last_managed_era)) => { + match (Self::get_current_era(), >::get()) { + (global_era_counter, Some(last_managed_era)) => { if last_managed_era >= global_era_counter { return 0 } >::put(global_era_counter); }, - (Some(global_era_counter), None) => { + (global_era_counter, None) => { >::put(global_era_counter); }, _ => return 0, @@ -113,11 +245,172 @@ pub mod pallet { } fn offchain_worker(block_number: T::BlockNumber) { - log::info!("Off-chain worker at block {:?}", block_number); + let res = Self::offchain_worker_main(block_number); + + match res { + Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), + }; } } - impl Pallet { + #[pallet::call] + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + #[pallet::weight(10000)] + pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + info!("[DAC Validator] author: {:?}", signer); + let mut v_results = ValidationResults::::get(); + + let cur_validation = ValidationResult:: { + era, + val_res, + cdn_node_pub_key, + signer, + }; + + v_results.push(cur_validation); + + ValidationResults::::set(v_results); + + Ok(()) + } + } + + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { + return Ok(()) + } + + let signer = match Self::get_signer() { + Err(e) => { + warn!("{:?}", e); + return Ok(()); + } + Ok(signer) => signer, + }; + + info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); + + // Read data from DataModel and do dumb validation + let current_era = Self::get_current_era() - 1u64; + let (bytes_sent, bytes_received) = Self::fetch_data(current_era); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let cdn_node_pub_key = bytes_sent.node_public_key.clone(); + let tx_res = signer.send_signed_transaction(|_acct| { + info!("[DAC Validator] Sending save_validated_data tx"); + + // This is the on-chain function + Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } + }); + + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting save_validated_data TX") + } + Some((_, Ok(()))) => {} + } + + Ok(()) + } + + fn get_signer() -> ResultStr> { + let signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + } + + Ok(signer) + } + + // Get the current era; Shall we start era count from 0 or from 1? + fn get_current_era() -> u64 { + (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + } + + fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::new(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::new(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + } + + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + } + + fn http_get_json(url: &str) -> ResultStr { + let body = Self::http_get_request(url).map_err(|err| { + error!("[DAC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" + })?; + + let parsed = serde_json::from_slice(&body).map_err(|err| { + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" + }); + + parsed + } + + fn http_get_request(http_url: &str) -> Result, http::Error> { + info!("[DAC Validator] Sending request to: {:?}", http_url); + + // Initiate an external HTTP GET request. This is using high-level wrappers from + // `sp_runtime`. + let request = http::Request::get(http_url); + + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + // Next we fully read the response body and collect it to a vector of bytes. + Ok(response.body().collect::>()) + } + + fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { + return if bytes_sent.sum == bytes_received.sum { + true + } else { + false + } + } + /// Fetch the tasks related to current validator fn fetch_tasks(validator: T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; @@ -151,16 +444,11 @@ pub mod pallet { } fn generate_random_number(seed: u32) -> u32 { - let (random_seed, _) = T::Randomness::random(&(b"ddc-validator", seed).encode()); + let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); let random_number = ::decode(&mut random_seed.as_ref()) .expect("secure hashes should always be bigger than u32; qed"); random_number } - - // Get the current era; Shall we start era count from 0 or from 1? - fn get_current_era() -> u64 { - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS - } } } From 8ea8d8e3dfc26eacbc530e9b3b69279d8111c506 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 14:43:26 +0100 Subject: [PATCH 060/192] add proof of delivery --- frame/ddc-validator/src/lib.rs | 75 +++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 9cab6656ca750..52219d6dc0481 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -181,6 +181,10 @@ pub mod pallet { #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, u64>; + #[pallet::storage] + #[pallet::getter(fn validation_results)] + pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event @@ -209,7 +213,6 @@ pub mod pallet { (global_era_counter, None) => { >::put(global_era_counter); }, - _ => return 0, }; let validators: Vec = >::iter_keys().collect(); @@ -280,6 +283,27 @@ pub mod pallet { Ok(()) } + + #[pallet::weight(10000)] + pub fn proof_of_delivery(origin: OriginFor, era: u64) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + for cdn_node_id in cdn_nodes_to_validate { + let (bytes_sent, bytes_received) = Self::fetch_data(era, &cdn_node_id); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + let decisions_for_cdn = >::get(cdn_node_id); + for decision in decisions_for_cdn.unwrap().iter_mut() { + if decision.validator == signer { + decision.decision = Some(val_res); + decision.method = ValidationMethodKind::ProofOfDelivery; + } + } + } + + Ok(()) + } } impl Pallet @@ -306,23 +330,26 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - let (bytes_sent, bytes_received) = Self::fetch_data(current_era); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Sending save_validated_data tx"); + // for decision in &mut cdn_nodes_to_validate { - // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - }); + // } + // let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + // let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting save_validated_data TX") - } - Some((_, Ok(()))) => {} - } + // let tx_res = signer.send_signed_transaction(|_acct| { + // info!("[DAC Validator] Sending save_validated_data tx"); + + // // This is the on-chain function + // Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } + // }); + + // match &tx_res { + // None | Some((_, Err(()))) => { + // return Err("Error while submitting save_validated_data TX") + // } + // Some((_, Ok(()))) => {} + // } Ok(()) } @@ -341,16 +368,16 @@ pub mod pallet { (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS } - fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ + fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_query = Self::get_bytes_sent_query_url(era, cdn_node); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); @@ -359,12 +386,12 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + fn get_bytes_sent_query_url(era: u64, cdn_node: &T::AccountId) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era, *cdn_node) } - fn get_bytes_received_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + fn get_bytes_received_query_url(era: u64, cdn_node: &T::AccountId) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era, *cdn_node) } fn http_get_json(url: &str) -> ResultStr { @@ -412,11 +439,11 @@ pub mod pallet { } /// Fetch the tasks related to current validator - fn fetch_tasks(validator: T::AccountId) -> Vec { + fn fetch_tasks(validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; for (cdn_id, cdn_tasks) in >::iter() { for decision in cdn_tasks.iter() { - if decision.validator == validator { + if decision.validator == *validator { cdn_nodes.push(cdn_id); break } From ce82be59d87f063dd1cfe36835fbc608a9534e66 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 22 Mar 2023 14:45:48 +0100 Subject: [PATCH 061/192] add PoD trigger in offchain worker --- frame/ddc-validator/src/lib.rs | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 52219d6dc0481..d599567bef4aa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -331,25 +331,20 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - // for decision in &mut cdn_nodes_to_validate { + + let tx_res = signer.send_signed_transaction(|_acct| { + info!("[DAC Validator] Trigger proof of delivery"); - // } - // let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - // let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - - // let tx_res = signer.send_signed_transaction(|_acct| { - // info!("[DAC Validator] Sending save_validated_data tx"); - - // // This is the on-chain function - // Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - // }); + // This is the on-chain function + Call::proof_of_delivery { era: current_era } + }); - // match &tx_res { - // None | Some((_, Err(()))) => { - // return Err("Error while submitting save_validated_data TX") - // } - // Some((_, Ok(()))) => {} - // } + match &tx_res { + None | Some((_, Err(()))) => { + return Err("Error while submitting proof of delivery TX") + } + Some((_, Ok(()))) => {} + } Ok(()) } From 54dd70cefc3c5a38eb27a2bf3191966753af7292 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 22 Mar 2023 19:15:46 +0100 Subject: [PATCH 062/192] Fix json parse error for cases when in the response not 2 cdn nodes --- .../dac-validator-offchain-worker/src/lib.rs | 90 +++++++------------ 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs index 161a646514e13..6be8983dc768b 100644 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ b/frame/dac-validator-offchain-worker/src/lib.rs @@ -49,19 +49,16 @@ type ResultStr = Result; #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec, Vec), - - // This struct should be correct for any length but there is error while parsing JSON - // #[serde(rename = "FT.AGGREGATE")] - // pub ft_aggregate: Vec, + pub ft_aggregate: Vec, } -// #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -// #[serde(crate = "alt_serde")] -// pub enum FtAggregate { -// Length(u32), -// Node(Vec), -// } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(untagged)] +pub enum FtAggregate { + Length(u32), + Node(Vec), +} #[derive(Clone)] struct BytesSent { @@ -72,34 +69,21 @@ struct BytesSent { impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesSent { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } } -// impl BytesSent { -// pub fn new(aggregate: RedisFtAggregate) -> BytesSent { -// // let (_, values, values2) = aggregate.ft_aggregate; -// let data = aggregate.ft_aggregate[1].clone(); -// -// match data { -// FtAggregate::Node(node) => { -// return BytesSent { -// node_public_key: node[1].clone(), -// era: node[3].clone(), -// sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), -// } -// } -// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), -// } -// } -// } - #[derive(Clone)] struct BytesReceived { node_public_key: String, @@ -109,35 +93,21 @@ struct BytesReceived { impl BytesReceived { pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesReceived { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } } -// impl BytesReceived { -// pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { -// // let (_, values, values2) = aggregate.ft_aggregate; -// -// let data = aggregate.ft_aggregate[1].clone(); -// -// match data { -// FtAggregate::Node(node) => { -// return BytesReceived { -// node_public_key: node[1].clone(), -// era: node[3].clone(), -// sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), -// } -// } -// FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), -// } -// } -// } - #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { era: String, From cf131e6c9306ec9ab59ba404245efb64f2be4128 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 23 Mar 2023 11:47:35 +0100 Subject: [PATCH 063/192] Move JSON parsing fix to ddc-validator --- frame/ddc-validator/src/lib.rs | 58 ++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index d599567bef4aa..2b1d5245eb286 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -72,8 +72,16 @@ pub struct ValidationResult { #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] pub struct RedisFtAggregate { - #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: (u32, Vec, Vec), + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(untagged)] +pub enum FtAggregate { + Length(u32), + Node(Vec), } #[derive(Clone)] @@ -84,15 +92,20 @@ struct BytesSent { } impl BytesSent { - pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesSent { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } } #[derive(Clone)] @@ -103,15 +116,20 @@ struct BytesReceived { } impl BytesReceived { - pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let (_, values, values2) = aggregate.ft_aggregate; - - BytesReceived { - node_public_key: values[1].clone(), - era: values[3].clone(), - sum: values[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => { + return BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + } + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } } pub mod crypto { From 9b650dbd9e1684b9370c15e6fea0dd0b6610a843 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Thu, 23 Mar 2023 15:23:09 +0100 Subject: [PATCH 064/192] fix timestamp bug --- bin/node/runtime/src/lib.rs | 1 + frame/ddc-validator/src/lib.rs | 96 ++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index ba7f7f460155a..47a5a133320c7 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1245,6 +1245,7 @@ impl pallet_ddc_validator::Config for Runtime { type Randomness = RandomnessCollectiveFlip; type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; + type TimeProvider = pallet_timestamp::Pallet; } construct_runtime!( diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 2b1d5245eb286..e7481fe6ed9bc 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -8,7 +8,7 @@ pub use frame_support::{ decl_event, decl_module, decl_storage, log::{error, info, warn}, pallet_prelude::*, - traits::{Randomness, Currency}, + traits::{Randomness, Currency, UnixTime}, weights::Weight, dispatch::DispatchResult, RuntimeDebug, @@ -22,9 +22,9 @@ pub use pallet_staking::{self as staking}; pub use pallet_session as session; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; -pub use sp_runtime::offchain::{http, Duration}; +pub use sp_runtime::offchain::{http, Duration, Timestamp}; pub use sp_std::prelude::*; -pub use sp_io::offchain::timestamp; + extern crate alloc; parameter_types! { @@ -42,8 +42,8 @@ pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); pub const HTTP_TIMEOUT_MS: u64 = 30_000; -const TIME_START_MS: u64 = 1_672_531_200_000; -const ERA_DURATION_MS: u64 = 120_000; +const TIME_START_MS: u128 = 1_672_531_200_000; +const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; @@ -184,6 +184,7 @@ pub mod pallet { type Randomness: Randomness; type Call: From>; type AuthorityId: AppCrypto; + type TimeProvider: UnixTime; } #[pallet::storage] @@ -221,48 +222,53 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - match (Self::get_current_era(), >::get()) { - (global_era_counter, Some(last_managed_era)) => { - if last_managed_era >= global_era_counter { - return 0 + if block_number != 0u32.into() && block_number != 1u32.into() { + let era = Self::get_current_era(); + match (era, >::get()) { + (global_era_counter, Some(last_managed_era)) => { + if last_managed_era >= global_era_counter { + return 0 + } + >::put(global_era_counter); + }, + (global_era_counter, None) => { + >::put(global_era_counter); + }, + }; + + let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", + block_number, + era, + >::get(), + validators_count, + validators, + edges, + ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); } - >::put(global_era_counter); - }, - (global_era_counter, None) => { - >::put(global_era_counter); - }, - }; - - let validators: Vec = >::iter_keys().collect(); - let validators_count = validators.len() as u32; - let edges: Vec = >::iter_keys().collect(); - log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", - block_number, - Self::get_current_era(), - >::get(), - validators_count, - validators, - edges, - ); - - // A naive approach assigns random validators for each edge. - for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = - Default::default(); - while !decisions.is_full() { - let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - let validator: T::AccountId = validators[validator_idx].clone(); - let assignment = Decision { - validator, - method: ValidationMethodKind::ProofOfDelivery, - decision: None, - }; - decisions.try_push(assignment).unwrap(); + Tasks::::insert(edge, decisions); } - Tasks::::insert(edge, decisions); + 0 + } else { + 0 } - 0 } fn offchain_worker(block_number: T::BlockNumber) { @@ -378,7 +384,7 @@ pub mod pallet { // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS + ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS).try_into().unwrap() } fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { From 09afaf1773fd37eb1a914c7a9e22495dc954791c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 24 Mar 2023 11:17:31 +0600 Subject: [PATCH 065/192] Autoformat DAC Validator files --- frame/ddc-validator/Cargo.toml | 8 +- frame/ddc-validator/src/lib.rs | 205 +++++++++++++++++---------------- 2 files changed, 109 insertions(+), 104 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 55172f6848949..af7a0108ea9e0 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -11,14 +11,14 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../sys log = { version = "0.4.17", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } +pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } -sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } @@ -31,13 +31,13 @@ std = [ "frame-system/std", "pallet-contracts/std", "pallet-ddc-staking/std", - "pallet-staking/std", "pallet-session/std", + "pallet-staking/std", "scale-info/std", "serde", - "sp-keystore", "sp-core/std", "sp-io/std", + "sp-keystore", "sp-runtime/std", "sp-staking/std", "sp-std/std", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index e7481fe6ed9bc..a2e6586a88120 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -2,24 +2,27 @@ pub use alloc::{format, string::String}; pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; -pub use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; +pub use codec::{Decode, Encode, HasCompact, MaxEncodedLen}; pub use core::fmt::Debug; pub use frame_support::{ decl_event, decl_module, decl_storage, - log::{error, info, warn}, - pallet_prelude::*, - traits::{Randomness, Currency, UnixTime}, - weights::Weight, dispatch::DispatchResult, - RuntimeDebug, - BoundedVec, + log::{error, info, warn}, + pallet_prelude::*, parameter_types, + traits::{Currency, Randomness, UnixTime}, + weights::Weight, + BoundedVec, RuntimeDebug, +}; +pub use frame_system::{ + ensure_signed, + offchain::{AppCrypto, CreateSignedTransaction, SendSignedTransaction, Signer, SigningTypes}, + pallet_prelude::*, }; -pub use frame_system::{ensure_signed, pallet_prelude::*, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; -pub use pallet_staking::{self as staking}; pub use pallet_session as session; +pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_runtime::offchain::{http, Duration, Timestamp}; @@ -32,12 +35,11 @@ parameter_types! { } type BalanceOf = <::Currency as Currency< - ::AccountId, + ::AccountId, >>::Balance; type ResultStr = Result; - pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); pub const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -62,10 +64,10 @@ pub struct Decision { #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { - era: String, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, + era: String, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -86,9 +88,9 @@ pub enum FtAggregate { #[derive(Clone)] struct BytesSent { - node_public_key: String, - era: String, - sum: u32, + node_public_key: String, + era: String, + sum: u32, } impl BytesSent { @@ -96,13 +98,12 @@ impl BytesSent { let data = aggregate.ft_aggregate[1].clone(); match data { - FtAggregate::Node(node) => { + FtAggregate::Node(node) => return BytesSent { node_public_key: node[1].clone(), era: node[3].clone(), sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } @@ -110,9 +111,9 @@ impl BytesSent { #[derive(Clone)] struct BytesReceived { - node_public_key: String, - era: String, - sum: u32, + node_public_key: String, + era: String, + sum: u32, } impl BytesReceived { @@ -120,13 +121,14 @@ impl BytesReceived { let data = aggregate.ft_aggregate[1].clone(); match data { - FtAggregate::Node(node) => { + FtAggregate::Node(node) => return BytesReceived { node_public_key: node[1].clone(), era: node[3].clone(), - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertable to u32"), + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } @@ -137,8 +139,8 @@ pub mod crypto { use frame_system::offchain::AppCrypto; use sp_core::sr25519::Signature as Sr25519Signature; use sp_runtime::{ - app_crypto::{app_crypto, sr25519}, - traits::Verify, + app_crypto::{app_crypto, sr25519}, + traits::Verify, }; app_crypto!(sr25519, KEY_TYPE); @@ -147,15 +149,15 @@ pub mod crypto { pub struct TestAuthId; impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; } impl AppCrypto for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; } } @@ -169,21 +171,21 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: + pub trait Config: frame_system::Config + pallet_contracts::Config - + pallet_session::Config::AccountId> + + pallet_session::Config::AccountId> + pallet_staking::Config + ddc_staking::Config + CreateSignedTransaction> - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { type Event: From> + IsType<::Event>; type Randomness: Randomness; type Call: From>; - type AuthorityId: AppCrypto; + type AuthorityId: AppCrypto; type TimeProvider: UnixTime; } @@ -202,28 +204,28 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; + pub(super) type ValidationResults = + StorageValue<_, Vec>, ValueQuery>; #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event + pub enum Event where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - {} + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} #[pallet::error] pub enum Error {} #[pallet::hooks] - impl Hooks> for Pallet + impl Hooks> for Pallet where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { if block_number != 0u32.into() && block_number != 1u32.into() { - let era = Self::get_current_era(); + let era = Self::get_current_era(); match (era, >::get()) { (global_era_counter, Some(last_managed_era)) => { if last_managed_era >= global_era_counter { @@ -235,10 +237,11 @@ pub mod pallet { >::put(global_era_counter); }, }; - + let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; - let edges: Vec = >::iter_keys().collect(); + let edges: Vec = + >::iter_keys().collect(); log::info!( "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", block_number, @@ -248,7 +251,7 @@ pub mod pallet { validators, edges, ); - + // A naive approach assigns random validators for each edge. for edge in edges { let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = @@ -282,24 +285,25 @@ pub mod pallet { } #[pallet::call] - impl Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + impl Pallet + where + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { #[pallet::weight(10000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { + pub fn save_validated_data( + origin: OriginFor, + val_res: bool, + cdn_node_pub_key: String, + era: String, + ) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; info!("[DAC Validator] author: {:?}", signer); let mut v_results = ValidationResults::::get(); - let cur_validation = ValidationResult:: { - era, - val_res, - cdn_node_pub_key, - signer, - }; + let cur_validation = + ValidationResult:: { era, val_res, cdn_node_pub_key, signer }; v_results.push(cur_validation); @@ -330,24 +334,27 @@ pub mod pallet { } } - impl Pallet + impl Pallet where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + ::AccountId: AsRef<[u8]> + UncheckedFrom, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); + info!( + "[DAC Validator] Validation data stored onchain: {:?}", + ValidationResults::::get() + ); if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) + return Ok(()) } let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()); - } - Ok(signer) => signer, + Err(e) => { + warn!("{:?}", e); + return Ok(()) + }, + Ok(signer) => signer, }; info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); @@ -355,19 +362,17 @@ pub mod pallet { // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1u64; - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Trigger proof of delivery"); + info!("[DAC Validator] Trigger proof of delivery"); - // This is the on-chain function - Call::proof_of_delivery { era: current_era } + // This is the on-chain function + Call::proof_of_delivery { era: current_era } }); match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting proof of delivery TX") - } - Some((_, Ok(()))) => {} + None | Some((_, Err(()))) => + return Err("Error while submitting proof of delivery TX"), + Some((_, Ok(()))) => {}, } Ok(()) @@ -376,7 +381,7 @@ pub mod pallet { fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { - return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); } Ok(signer) @@ -384,7 +389,9 @@ pub mod pallet { // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> u64 { - ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS).try_into().unwrap() + ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) + .try_into() + .unwrap() } fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { @@ -398,7 +405,7 @@ pub mod pallet { // Todo: handle the error let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); + Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); @@ -415,13 +422,13 @@ pub mod pallet { fn http_get_json(url: &str) -> ResultStr { let body = Self::http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); - "HTTP GET error" + error!("[DAC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" })?; let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); - "HTTP JSON parse error" + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" }); parsed @@ -438,23 +445,20 @@ pub mod pallet { let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; - let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + let response = + pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); - return Err(http::Error::Unknown) + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) } // Next we fully read the response body and collect it to a vector of bytes. Ok(response.body().collect::>()) - } + } fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { - true - } else { - false - } + return if bytes_sent.sum == bytes_received.sum { true } else { false } } /// Fetch the tasks related to current validator @@ -490,7 +494,8 @@ pub mod pallet { } fn generate_random_number(seed: u32) -> u32 { - let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); + let (random_seed, _) = + ::Randomness::random(&(b"ddc-validator", seed).encode()); let random_number = ::decode(&mut random_seed.as_ref()) .expect("secure hashes should always be bigger than u32; qed"); From a5aafacf1bd43e679aafec5feb398fed13322fa7 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 24 Mar 2023 12:02:07 +0100 Subject: [PATCH 066/192] add filtering of requests --- frame/ddc-validator/Cargo.toml | 1 + frame/ddc-validator/src/lib.rs | 91 +++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index af7a0108ea9e0..0e0fe0ea95000 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] +array-bytes = "6.0.0" alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a2e6586a88120..5a553ca1dde7a 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -107,6 +107,27 @@ impl BytesSent { FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec!(); + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + }; + + res.push(node); + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res; + } } #[derive(Clone)] @@ -132,6 +153,27 @@ impl BytesReceived { FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec!(); + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone(), + sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + }; + + res.push(node); + } + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res; + } } pub mod crypto { @@ -317,8 +359,9 @@ pub mod pallet { let signer: T::AccountId = ensure_signed(origin)?; let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + let (s, r) = Self::fetch_data1(era); for cdn_node_id in cdn_nodes_to_validate { - let (bytes_sent, bytes_received) = Self::fetch_data(era, &cdn_node_id); + let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let decisions_for_cdn = >::get(cdn_node_id); @@ -397,13 +440,13 @@ pub mod pallet { fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era, cdn_node); + let bytes_sent_query = Self::get_bytes_sent_query_url(era); let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era, cdn_node); + let bytes_received_query = Self::get_bytes_received_query_url(era); let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); @@ -412,12 +455,46 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64, cdn_node: &T::AccountId) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era, *cdn_node) + fn account_to_string(account: T::AccountId) -> String { + let to32 = T::AccountId::encode(&account); + let pub_key_str = array_bytes::bytes2hex("", to32); + + pub_key_str + } + + fn filter_data(s: &Vec, r: &Vec, a: &T::AccountId) -> (BytesSent, BytesReceived){ + let ac = Self::account_to_string(a.clone()); + + let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); + let filtered_r = &*r.into_iter().find(|br| br.node_public_key == ac).unwrap(); + + (filtered_s.clone(), filtered_r.clone()) + } + + fn fetch_data1(era: u64 ) -> (Vec, Vec){ + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::get_all(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::get_all(bytes_received_res); + + (bytes_sent, bytes_received) + } + + fn get_bytes_sent_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) } - fn get_bytes_received_query_url(era: u64, cdn_node: &T::AccountId) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/{:?}/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era, *cdn_node) + fn get_bytes_received_query_url(era: u64) -> String { + format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) } fn http_get_json(url: &str) -> ResultStr { From fee7b819427554493ab74317f4c63a80e0b92aab Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 14:38:27 +0600 Subject: [PATCH 067/192] Refactor guard condition --- frame/ddc-validator/src/lib.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 5a553ca1dde7a..dc5b4688cdb6d 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -268,18 +268,12 @@ pub mod pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { if block_number != 0u32.into() && block_number != 1u32.into() { let era = Self::get_current_era(); - match (era, >::get()) { - (global_era_counter, Some(last_managed_era)) => { - if last_managed_era >= global_era_counter { - return 0 - } - >::put(global_era_counter); - }, - (global_era_counter, None) => { - >::put(global_era_counter); - }, - }; - + if let Some(last_managed_era) = >::get() { + if last_managed_era >= era { + return 0 + } + } + >::put(era); let validators: Vec = >::iter_keys().collect(); let validators_count = validators.len() as u32; let edges: Vec = From ed167c868e8139beff75a40531278198887259de Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 15:30:57 +0600 Subject: [PATCH 068/192] Refactor a guard condition to decrease indentation --- frame/ddc-validator/src/lib.rs | 79 +++++++++++++++++----------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index dc5b4688cdb6d..cf4423173bc88 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -266,48 +266,49 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - if block_number != 0u32.into() && block_number != 1u32.into() { - let era = Self::get_current_era(); - if let Some(last_managed_era) = >::get() { - if last_managed_era >= era { - return 0 - } + if block_number < 1u32.into() { + return 0 + } + + let era = Self::get_current_era(); + if let Some(last_managed_era) = >::get() { + if last_managed_era >= era { + return 0 } - >::put(era); - let validators: Vec = >::iter_keys().collect(); - let validators_count = validators.len() as u32; - let edges: Vec = - >::iter_keys().collect(); - log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", - block_number, - era, - >::get(), - validators_count, - validators, - edges, - ); - - // A naive approach assigns random validators for each edge. - for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = - Default::default(); - while !decisions.is_full() { - let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - let validator: T::AccountId = validators[validator_idx].clone(); - let assignment = Decision { - validator, - method: ValidationMethodKind::ProofOfDelivery, - decision: None, - }; - decisions.try_push(assignment).unwrap(); - } - Tasks::::insert(edge, decisions); + } + >::put(era); + + let validators: Vec = >::iter_keys().collect(); + let validators_count = validators.len() as u32; + let edges: Vec = >::iter_keys().collect(); + log::info!( + "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", + block_number, + era, + >::get(), + validators_count, + validators, + edges, + ); + + // A naive approach assigns random validators for each edge. + for edge in edges { + let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + Default::default(); + while !decisions.is_full() { + let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + let validator: T::AccountId = validators[validator_idx].clone(); + let assignment = Decision { + validator, + method: ValidationMethodKind::ProofOfDelivery, + decision: None, + }; + decisions.try_push(assignment).unwrap(); } - 0 - } else { - 0 + Tasks::::insert(edge, decisions); } + + 0 } fn offchain_worker(block_number: T::BlockNumber) { From 2856c6ed17b85d5ce8afc660023a3ba02da98ff2 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 27 Mar 2023 16:05:56 +0200 Subject: [PATCH 069/192] Removed poc pallet --- Cargo.lock | 36 +- bin/node/runtime/Cargo.toml | 1 - bin/node/runtime/src/lib.rs | 8 - .../dac-validator-offchain-worker/Cargo.toml | 58 --- .../dac-validator-offchain-worker/src/lib.rs | 386 ------------------ 5 files changed, 7 insertions(+), 482 deletions(-) delete mode 100644 frame/dac-validator-offchain-worker/Cargo.toml delete mode 100644 frame/dac-validator-offchain-worker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6359b04d3e23c..b2869c996bec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,6 +141,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "698b65a961a9d730fb45b6b0327e20207810c9f61ee421b082b27ba003f49e2b" +[[package]] +name = "array-bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f72e9d6fac4bc80778ea470b20197b88d28c292bb7d60c3fb099280003cd19" + [[package]] name = "arrayref" version = "0.3.6" @@ -5047,7 +5053,6 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", - "pallet-dac-validator-offchain-worker", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", "pallet-ddc-validator", @@ -5804,34 +5809,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "pallet-dac-validator-offchain-worker" -version = "0.0.1" -dependencies = [ - "alt_serde", - "frame-support", - "frame-system", - "hex", - "hex-literal", - "lite-json", - "pallet-balances", - "pallet-contracts", - "pallet-randomness-collective-flip", - "pallet-session", - "pallet-staking", - "pallet-timestamp", - "parity-scale-codec", - "pretty_assertions 0.6.1", - "scale-info", - "serde", - "serde_json 1.0.44", - "sp-core", - "sp-io", - "sp-keystore", - "sp-runtime", - "sp-std", -] - [[package]] name = "pallet-ddc-metrics-offchain-worker" version = "2.0.1" @@ -5879,6 +5856,7 @@ name = "pallet-ddc-validator" version = "0.1.0" dependencies = [ "alt_serde", + "array-bytes", "frame-support", "frame-system", "log 0.4.17", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index eaf555fbc4f3f..178490e3e7106 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -100,7 +100,6 @@ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../f pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } -pallet-dac-validator-offchain-worker = { version = "0.0.1", default-features = false, path = "../../../frame/dac-validator-offchain-worker" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 47a5a133320c7..072543a8ad79a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -82,7 +82,6 @@ pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; pub use pallet_ddc_validator; -pub use pallet_dac_validator_offchain_worker; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1218,12 +1217,6 @@ parameter_types! { pub const OcwBlockInterval: u32 = pallet_ddc_metrics_offchain_worker::BLOCK_INTERVAL; } -impl pallet_dac_validator_offchain_worker::Config for Runtime { - type Call = Call; - - type AuthorityId = pallet_dac_validator_offchain_worker::crypto::TestAuthId; -} - impl pallet_ddc_metrics_offchain_worker::Config for Runtime { type BlockInterval = OcwBlockInterval; @@ -1297,7 +1290,6 @@ construct_runtime!( Erc721: pallet_erc721::{Pallet, Call, Storage, Event}, Erc20: pallet_erc20::{Pallet, Call, Storage, Event}, DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, - DdcValidatorOffchainWorker: pallet_dac_validator_offchain_worker::{Pallet, Call, Storage}, DdcStaking: pallet_ddc_staking, DdcValidator: pallet_ddc_validator, } diff --git a/frame/dac-validator-offchain-worker/Cargo.toml b/frame/dac-validator-offchain-worker/Cargo.toml deleted file mode 100644 index 7f978de7fe981..0000000000000 --- a/frame/dac-validator-offchain-worker/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -[package] -name = "pallet-dac-validator-offchain-worker" -version = "0.0.1" -authors = ["Parity Technologies "] -edition = "2021" -license = "Unlicense" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "FRAME example pallet for offchain worker" -readme = "README.md" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["full"] } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -serde = { version = "1.0.101", optional = true } -sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore", optional = true } -sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } -sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } -lite-json = { version = "0.2.0", default-features = false } -alt_serde = { version = "1", default-features = false, features = ["derive"] } -serde_json = { version = "1", default-features = false, git = "https://github.com/Cerebellum-Network/json", branch = "no-std-cere", features = ["alloc"] } -hex-literal = "^0.3.1" -hex = { version = "0.4", default-features = false } -scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } -pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } - -[features] -default = ["std"] -std = [ - "codec/std", - "sp-keystore", - "frame-support/std", - "frame-system/std", - "serde", - "lite-json/std", - "sp-core/std", - "sp-io/std", - "sp-runtime/std", - "sp-std/std", - "pallet-contracts/std", - "pallet-staking/std", - "pallet-session/std", - # "pallet-contracts-rpc-runtime-api/std", -] - -[dev-dependencies] -pallet-balances = { version = "4.0.0-dev", path = "../balances" } -pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } -pretty_assertions = "0.6.1" \ No newline at end of file diff --git a/frame/dac-validator-offchain-worker/src/lib.rs b/frame/dac-validator-offchain-worker/src/lib.rs deleted file mode 100644 index 6be8983dc768b..0000000000000 --- a/frame/dac-validator-offchain-worker/src/lib.rs +++ /dev/null @@ -1,386 +0,0 @@ -#![cfg_attr(not(feature = "std"), no_std)] - -use alloc::{format, string::String}; -use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; -use codec::{Encode, Decode, MaxEncodedLen, HasCompact}; -use frame_support::{ - decl_event, decl_module, decl_storage, - log::{error, info, warn}, - traits::Currency, - weights::{Weight}, - dispatch::DispatchResult, - RuntimeDebug, - BoundedVec, - parameter_types, -}; -use frame_system::{ensure_signed, offchain::{CreateSignedTransaction, Signer, SigningTypes, AppCrypto, SendSignedTransaction}}; -use pallet_staking::{self as staking}; -use sp_core::crypto::{KeyTypeId, UncheckedFrom};use sp_io::offchain::timestamp; -use sp_runtime::offchain::{http, Duration}; -use sp_std::{vec, vec::Vec}; -extern crate alloc; -use pallet_session as session; - -parameter_types! { - pub DdcValidatorsQuorumSize: u32 = 3; -} - -use core::fmt::Debug; -use scale_info::TypeInfo; - -type BalanceOf = <::Currency as Currency< - ::AccountId, ->>::Balance; - -pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); - -pub const HTTP_TIMEOUT_MS: u64 = 30_000; - -const TIME_START_MS: u64 = 1_672_531_200_000; -const ERA_DURATION_MS: u64 = 120_000; -const ERA_IN_BLOCKS: u8 = 20; - -const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; - -type ResultStr = Result; - -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct RedisFtAggregate { - #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(untagged)] -pub enum FtAggregate { - Length(u32), - Node(Vec), -} - -#[derive(Clone)] -struct BytesSent { - node_public_key: String, - era: String, - sum: u32, -} - -impl BytesSent { - pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => { - return BytesSent { - node_public_key: node[1].clone(), - era: node[3].clone(), - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), - } - } - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } -} - -#[derive(Clone)] -struct BytesReceived { - node_public_key: String, - era: String, - sum: u32, -} - -impl BytesReceived { - pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => { - return BytesReceived { - node_public_key: node[1].clone(), - era: node[3].clone(), - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), - } - } - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } -} - -#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - era: String, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, -} - -// use sp_std::fmt; -// impl fmt::Display for ValidationResult -// { -// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { -// write!(f, "ValidationResult signer {}", self.signer) -// } -// } - -pub mod crypto { - use super::KEY_TYPE; - use frame_system::offchain::AppCrypto; - use sp_core::sr25519::Signature as Sr25519Signature; - use sp_runtime::{ - app_crypto::{app_crypto, sr25519}, - traits::Verify, - }; - app_crypto!(sr25519, KEY_TYPE); - - use sp_runtime::{MultiSignature, MultiSigner}; - - pub struct TestAuthId; - - impl AppCrypto<::Signer, Sr25519Signature> for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; - } - - impl AppCrypto for TestAuthId { - type RuntimeAppPublic = Public; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; - } -} - -pub use pallet::*; - -#[frame_support::pallet] -pub mod pallet { - use super::*; - use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; - - #[pallet::pallet] - #[pallet::without_storage_info] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: - frame_system::Config - + pallet_contracts::Config - + pallet_session::Config::AccountId> - + pallet_staking::Config - + CreateSignedTransaction> - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - /// The overarching dispatch call type. - type Call: From>; - - type AuthorityId: AppCrypto; - } - - #[pallet::hooks] - impl Hooks> for Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - fn offchain_worker(block_number: T::BlockNumber) { - let res = Self::offchain_worker_main(block_number); - - match res { - Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), - Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), - }; - } - } - - #[pallet::call] - impl Pallet - where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - #[pallet::weight(10000)] - pub fn save_validated_data(origin: OriginFor, val_res: bool, cdn_node_pub_key: String, era: String) -> DispatchResult { - let signer: T::AccountId = ensure_signed(origin)?; - - info!("[DAC Validator] author: {:?}", signer); - let mut v_results = ValidationResults::::get(); - - let cur_validation = ValidationResult:: { - era, - val_res, - cdn_node_pub_key, - signer, - }; - - // ValidationResults::::append(cur_validation); - // ValidationResults::::mutate(|results| { - // results.push(cur_validation); - // - // results.clone() - // }); - v_results.push(cur_validation); - - ValidationResults::::set(v_results); - - Ok(()) - } - } - - #[pallet::storage] - #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = StorageValue<_, Vec>, ValueQuery>; -} - -impl Pallet -where - ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, -{ - fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!("[DAC Validator] Validation data stored onchain: {:?}", ValidationResults::::get()); - - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) - } - - let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()); - } - Ok(signer) => signer, - }; - - info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); - - // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; - let (bytes_sent, bytes_received) = Self::fetch_data(current_era); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - - let cdn_node_pub_key = bytes_sent.node_public_key.clone(); - let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Sending save_validated_data tx"); - - // This is the on-chain function - Call::save_validated_data { val_res, cdn_node_pub_key: cdn_node_pub_key.clone(), era: bytes_sent.era.clone() } - }); - - match &tx_res { - None | Some((_, Err(()))) => { - return Err("Error while submitting save_validated_data TX") - } - Some((_, Ok(()))) => {} - } - - // info!("[DAC Validator] save_validated_data: {:?}", ValidationResults::::get()); - - Ok(()) - } - - fn get_signer() -> ResultStr> { - let signer = Signer::<_, _>::any_account(); - if !signer.can_sign() { - return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); - } - - Ok(signer) - } - - fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { - true - } else { - false - } - } - - fn fetch_data(era: u64 ) -> (BytesSent, BytesReceived){ - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); - // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); - let bytes_sent = BytesSent::new(bytes_sent_res); - - // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); - let bytes_received = BytesReceived::new(bytes_received_res); - - (bytes_sent, bytes_received) - } - - fn get_current_era() -> u64 { - // info!("timestamp: {}", timestamp().unix_millis()); - // info!("TIME_START_MS: {}", TIME_START_MS); - // info!("ERA_DURATION_MS: {}", ERA_DURATION_MS); - // info!("1: {}", timestamp().unix_millis() - TIME_START_MS); - // info!("2: {}", (timestamp().unix_millis() - TIME_START_MS) % ERA_DURATION_MS); - - (timestamp().unix_millis() - TIME_START_MS) / ERA_DURATION_MS - } - - fn get_bytes_sent_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) - } - - fn get_bytes_received_query_url(era: u64) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) - } - - fn http_get_json(url: &str) -> ResultStr { - let body = Self::http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); - "HTTP GET error" - })?; - - let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); - "HTTP JSON parse error" - }); - - parsed - } - - fn http_get_request(http_url: &str) -> Result, http::Error> { - info!("[DAC Validator] Sending request to: {:?}", http_url); - - // Initiate an external HTTP GET request. This is using high-level wrappers from - // `sp_runtime`. - let request = http::Request::get(http_url); - - let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); - - let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; - - let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; - - if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); - return Err(http::Error::Unknown) - } - - // Next we fully read the response body and collect it to a vector of bytes. - Ok(response.body().collect::>()) - } - - fn validators() -> Vec<::ValidatorId> { - >::validators() - } - - /// Reward a validator. - pub fn reward_by_ids( - validators_points: impl IntoIterator, - ) -> Result<(), ()> { - >::reward_by_ids(validators_points); - Ok(()) - } -} \ No newline at end of file From 749018142be6141f27e4dd54dfa3b7656d0f063c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 27 Mar 2023 20:04:43 +0600 Subject: [PATCH 070/192] Use era type from staking primitives --- frame/ddc-validator/src/lib.rs | 37 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index cf4423173bc88..1de5200c696ec 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -26,6 +26,7 @@ pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_runtime::offchain::{http, Duration, Timestamp}; +pub use sp_staking::EraIndex; pub use sp_std::prelude::*; extern crate alloc; @@ -64,7 +65,7 @@ pub struct Decision { #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { - era: String, + era: EraIndex, signer: AccountId, val_res: bool, cdn_node_pub_key: String, @@ -89,7 +90,7 @@ pub enum FtAggregate { #[derive(Clone)] struct BytesSent { node_public_key: String, - era: String, + era: EraIndex, sum: u32, } @@ -101,7 +102,8 @@ impl BytesSent { FtAggregate::Node(node) => return BytesSent { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), @@ -116,7 +118,8 @@ impl BytesSent { FtAggregate::Node(node) => { let node = BytesSent { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), }; @@ -133,7 +136,7 @@ impl BytesSent { #[derive(Clone)] struct BytesReceived { node_public_key: String, - era: String, + era: EraIndex, sum: u32, } @@ -145,7 +148,8 @@ impl BytesReceived { FtAggregate::Node(node) => return BytesReceived { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5] .parse::() .expect("bytesReceivedSum must be convertable to u32"), @@ -162,7 +166,8 @@ impl BytesReceived { FtAggregate::Node(node) => { let node = BytesReceived { node_public_key: node[1].clone(), - era: node[3].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), }; @@ -242,7 +247,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn last_managed_era)] - pub type LastManagedEra = StorageValue<_, u64>; + pub type LastManagedEra = StorageValue<_, EraIndex>; #[pallet::storage] #[pallet::getter(fn validation_results)] @@ -332,7 +337,7 @@ pub mod pallet { origin: OriginFor, val_res: bool, cdn_node_pub_key: String, - era: String, + era: EraIndex, ) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; @@ -350,7 +355,7 @@ pub mod pallet { } #[pallet::weight(10000)] - pub fn proof_of_delivery(origin: OriginFor, era: u64) -> DispatchResult { + pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; let cdn_nodes_to_validate = Self::fetch_tasks(&signer); @@ -398,7 +403,7 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; + let current_era = Self::get_current_era() - 1; let tx_res = signer.send_signed_transaction(|_acct| { info!("[DAC Validator] Trigger proof of delivery"); @@ -426,13 +431,13 @@ pub mod pallet { } // Get the current era; Shall we start era count from 0 or from 1? - fn get_current_era() -> u64 { + fn get_current_era() -> EraIndex { ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) .try_into() .unwrap() } - fn fetch_data(era: u64, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { + fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); @@ -466,7 +471,7 @@ pub mod pallet { (filtered_s.clone(), filtered_r.clone()) } - fn fetch_data1(era: u64 ) -> (Vec, Vec){ + fn fetch_data1(era: EraIndex) -> (Vec, Vec){ info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); @@ -484,11 +489,11 @@ pub mod pallet { (bytes_sent, bytes_received) } - fn get_bytes_sent_query_url(era: u64) -> String { + fn get_bytes_sent_query_url(era: EraIndex) -> String { format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) } - fn get_bytes_received_query_url(era: u64) -> String { + fn get_bytes_received_query_url(era: EraIndex) -> String { format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) } From 3b4f852838475e71a6be55caa934e91cb8c22584 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:46:36 +0600 Subject: [PATCH 071/192] Fix faulty hook on block initialization --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 1de5200c696ec..edbaba17d690b 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -271,7 +271,7 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - if block_number < 1u32.into() { + if block_number <= 1u32.into() { return 0 } From df14bb4ace06a205321100a453359b643c0d620e Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:52:12 +0600 Subject: [PATCH 072/192] Autoformat DAC Validator files --- frame/ddc-validator/src/lib.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index edbaba17d690b..c0fe9def0cc93 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -111,7 +111,7 @@ impl BytesSent { } pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec!(); + let mut res: Vec = vec![]; for i in 1..aggregation.ft_aggregate.len() { let data = aggregation.ft_aggregate[i].clone(); match data { @@ -120,16 +120,18 @@ impl BytesSent { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + sum: node[5] + .parse::() + .expect("bytesSentSum must be convertable to u32"), }; res.push(node); - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } - return res; + return res } } @@ -159,7 +161,7 @@ impl BytesReceived { } pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec!(); + let mut res: Vec = vec![]; for i in 1..aggregation.ft_aggregate.len() { let data = aggregation.ft_aggregate[i].clone(); match data { @@ -168,16 +170,18 @@ impl BytesReceived { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesReceivedSum must be convertable to u32"), + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertable to u32"), }; res.push(node); - } + }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } } - return res; + return res } } @@ -462,7 +466,11 @@ pub mod pallet { pub_key_str } - fn filter_data(s: &Vec, r: &Vec, a: &T::AccountId) -> (BytesSent, BytesReceived){ + fn filter_data( + s: &Vec, + r: &Vec, + a: &T::AccountId, + ) -> (BytesSent, BytesReceived) { let ac = Self::account_to_string(a.clone()); let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); @@ -471,7 +479,7 @@ pub mod pallet { (filtered_s.clone(), filtered_r.clone()) } - fn fetch_data1(era: EraIndex) -> (Vec, Vec){ + fn fetch_data1(era: EraIndex) -> (Vec, Vec) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = Self::get_bytes_sent_query_url(era); From 0746e1e15edd623969ba29b0726f44276066601c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 12:53:59 +0600 Subject: [PATCH 073/192] Typo fix --- frame/ddc-validator/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index c0fe9def0cc93..a30027c7d1779 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -104,7 +104,7 @@ impl BytesSent { node_public_key: node[1].clone(), era: node[3].clone().parse::().expect("era must be convertible u32") as EraIndex, - sum: node[5].parse::().expect("bytesSentSum must be convertable to u32"), + sum: node[5].parse::().expect("bytesSentSum must be convertible to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } @@ -122,7 +122,7 @@ impl BytesSent { as EraIndex, sum: node[5] .parse::() - .expect("bytesSentSum must be convertable to u32"), + .expect("bytesSentSum must be convertible to u32"), }; res.push(node); @@ -154,7 +154,7 @@ impl BytesReceived { as EraIndex, sum: node[5] .parse::() - .expect("bytesReceivedSum must be convertable to u32"), + .expect("bytesReceivedSum must be convertible to u32"), }, FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), } @@ -172,7 +172,7 @@ impl BytesReceived { as EraIndex, sum: node[5] .parse::() - .expect("bytesReceivedSum must be convertable to u32"), + .expect("bytesReceivedSum must be convertible to u32"), }; res.push(node); From 43b6ddcc120d62fb1aa870447bad6b17c2a41845 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 28 Mar 2023 13:45:04 +0600 Subject: [PATCH 074/192] Introduce era key for DAC validator tasks --- frame/ddc-validator/src/lib.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a30027c7d1779..32354fa1933ee 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -242,9 +242,11 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn tasks)] - pub type Tasks = StorageMap< + pub type Tasks = StorageDoubleMap< _, Twox64Concat, + EraIndex, + Twox64Concat, T::AccountId, BoundedVec, DdcValidatorsQuorumSize>, >; @@ -314,7 +316,7 @@ pub mod pallet { }; decisions.try_push(assignment).unwrap(); } - Tasks::::insert(edge, decisions); + Tasks::::insert(era, edge, decisions); } 0 @@ -361,14 +363,14 @@ pub mod pallet { #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; - - let cdn_nodes_to_validate = Self::fetch_tasks(&signer); + let era = Self::get_current_era(); + let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); let (s, r) = Self::fetch_data1(era); for cdn_node_id in cdn_nodes_to_validate { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let decisions_for_cdn = >::get(cdn_node_id); + let decisions_for_cdn = >::get(era, cdn_node_id); for decision in decisions_for_cdn.unwrap().iter_mut() { if decision.validator == signer { decision.decision = Some(val_res); @@ -547,9 +549,9 @@ pub mod pallet { } /// Fetch the tasks related to current validator - fn fetch_tasks(validator: &T::AccountId) -> Vec { + fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; - for (cdn_id, cdn_tasks) in >::iter() { + for (cdn_id, cdn_tasks) in >::iter_prefix(era) { for decision in cdn_tasks.iter() { if decision.validator == *validator { cdn_nodes.push(cdn_id); From 4ce11d2cb5a8f6e853bc850b811876fdbaf55374 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 13:02:18 +0600 Subject: [PATCH 075/192] Set DAC Validators quorum size in runtime crate --- bin/node/runtime/src/lib.rs | 5 +++++ frame/ddc-validator/src/lib.rs | 11 +++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 072543a8ad79a..fb131673c4b7d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1233,7 +1233,12 @@ impl pallet_ddc_staking::Config for Runtime { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +parameter_types! { + pub const DdcValidatorsQuorumSize: u32 = 3; +} + impl pallet_ddc_validator::Config for Runtime { + type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; type Event = Event; type Randomness = RandomnessCollectiveFlip; type Call = Call; diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 32354fa1933ee..eb284d97a7595 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -31,10 +31,6 @@ pub use sp_std::prelude::*; extern crate alloc; -parameter_types! { - pub DdcValidatorsQuorumSize: u32 = 3; -} - type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -238,6 +234,9 @@ pub mod pallet { type Call: From>; type AuthorityId: AppCrypto; type TimeProvider: UnixTime; + + #[pallet::constant] + type DdcValidatorsQuorumSize: Get; } #[pallet::storage] @@ -248,7 +247,7 @@ pub mod pallet { EraIndex, Twox64Concat, T::AccountId, - BoundedVec, DdcValidatorsQuorumSize>, + BoundedVec, T::DdcValidatorsQuorumSize>, >; #[pallet::storage] @@ -304,7 +303,7 @@ pub mod pallet { // A naive approach assigns random validators for each edge. for edge in edges { - let mut decisions: BoundedVec, DdcValidatorsQuorumSize> = + let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = Default::default(); while !decisions.is_full() { let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; From 3359fe7532d2f1f339551c9a7589759346319e8a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 15:26:38 +0600 Subject: [PATCH 076/192] Print pubkeys available to offchain worker --- frame/ddc-validator/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index eb284d97a7595..6c06457a1b4ae 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -25,6 +25,7 @@ pub use pallet_session as session; pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; +pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, Duration, Timestamp}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; @@ -322,6 +323,16 @@ pub mod pallet { } fn offchain_worker(block_number: T::BlockNumber) { + let pubkeys = sr25519_public_keys(KEY_TYPE); + if pubkeys.is_empty() { + log::info!("No local sr25519 accounts available to offchain worker."); + return + } + log::info!( + "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", + pubkeys, pubkeys.first().unwrap() + ); + let res = Self::offchain_worker_main(block_number); match res { From ddbf26127291e9109dcdb8662f8223e80f588a35 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:46:42 +0200 Subject: [PATCH 077/192] merge with latest commit --- frame/ddc-validator/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6c06457a1b4ae..229e01f05a2fa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -32,6 +32,10 @@ pub use sp_std::prelude::*; extern crate alloc; +parameter_types! { + pub const ValidationThreshold: f32 = 5.0; +} + type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -555,7 +559,9 @@ pub mod pallet { } fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { - return if bytes_sent.sum == bytes_received.sum { true } else { false } + let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); + + return if percentage_difference > 0.0 && (ValidationThreshold::get() - percentage_difference) > 0.0 { true } else { false } } /// Fetch the tasks related to current validator From 5a2f0d3d57bdd11a15a53eaaa36119a6eec73570 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:49:55 +0200 Subject: [PATCH 078/192] fix merge --- frame/ddc-validator/src/lib.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 229e01f05a2fa..b39e005488fce 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -88,8 +88,8 @@ pub enum FtAggregate { Node(Vec), } -#[derive(Clone)] -struct BytesSent { +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesSent { node_public_key: String, era: EraIndex, sum: u32, @@ -136,8 +136,8 @@ impl BytesSent { } } -#[derive(Clone)] -struct BytesReceived { +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesReceived { node_public_key: String, era: EraIndex, sum: u32, @@ -375,8 +375,9 @@ pub mod pallet { } #[pallet::weight(10000)] - pub fn proof_of_delivery(origin: OriginFor, era: EraIndex) -> DispatchResult { + pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { let signer: T::AccountId = ensure_signed(origin)?; + let era = Self::get_current_era(); let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); let (s, r) = Self::fetch_data1(era); @@ -423,15 +424,12 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1; + let current_era = Self::get_current_era() - 1u64; + let (s, r) = Self::fetch_data1(current_era); let tx_res = signer.send_signed_transaction(|_acct| { - info!("[DAC Validator] Trigger proof of delivery"); - - // This is the on-chain function - Call::proof_of_delivery { era: current_era } + Call::proof_of_delivery { s: s.clone(), r: r.clone() } }); - match &tx_res { None | Some((_, Err(()))) => return Err("Error while submitting proof of delivery TX"), From acb9b0f0fcaf322387de69e10495983193202526 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Wed, 29 Mar 2023 12:52:33 +0200 Subject: [PATCH 079/192] fix typo --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b39e005488fce..af78c5be65e82 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -424,7 +424,7 @@ pub mod pallet { info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1u64; + let current_era = Self::get_current_era() - 1; let (s, r) = Self::fetch_data1(current_era); let tx_res = signer.send_signed_transaction(|_acct| { From 15292ee6e3c36f83e426f6169cfc04116bf81131 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 16:35:22 +0600 Subject: [PATCH 080/192] Add doc comments --- frame/ddc-validator/src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index af78c5be65e82..eb28a1d59c77e 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -36,6 +36,7 @@ parameter_types! { pub const ValidationThreshold: f32 = 5.0; } +/// The balance type of this pallet. type BalanceOf = <::Currency as Currency< ::AccountId, >>::Balance; @@ -52,15 +53,22 @@ const ERA_IN_BLOCKS: u8 = 20; const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; +/// DAC Validation methods. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { + /// Compare amount of served content with amount of content consumed. ProofOfDelivery, } +/// Associates validation decision with the validator and the method used to produce it. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct Decision { + /// Individual validator's decision. Can be `None` if the validator did not produce a decision + /// (yet). pub decision: Option, + /// The method used to produce the decision. pub method: ValidationMethodKind, + /// The validator who produced the decision. pub validator: AccountId, } @@ -234,16 +242,27 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { + /// The overarching event type. type Event: From> + IsType<::Event>; + + /// Something that provides randomness in the runtime. Required by the tasks assignment + /// procedure. type Randomness: Randomness; + + /// A dispatchable call. type Call: From>; + type AuthorityId: AppCrypto; type TimeProvider: UnixTime; + /// Number of validators expected to produce an individual validation decision to form a + /// consensus. Tasks assignment procedure use this value to determine the number of + /// validators are getting the same task. Must be an odd number. #[pallet::constant] type DdcValidatorsQuorumSize: Get; } + /// The map from the era and CDN participant stash key to the validation decisions related. #[pallet::storage] #[pallet::getter(fn tasks)] pub type Tasks = StorageDoubleMap< @@ -255,6 +274,7 @@ pub mod pallet { BoundedVec, T::DdcValidatorsQuorumSize>, >; + /// The last era for which the tasks assignment produced. #[pallet::storage] #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, EraIndex>; @@ -576,6 +596,9 @@ pub mod pallet { cdn_nodes } + /// Randomly choose a number in range `[0, total)`. + /// Returns `None` for zero input. + /// Modification of `choose_ticket` from `pallet-lottery` version `4.0.0-dev`. fn choose(total: u32) -> Option { if total == 0 { return None @@ -594,6 +617,11 @@ pub mod pallet { Some(random_number % total) } + /// Generate a random number from a given seed. + /// Note that there is potential bias introduced by using modulus operator. + /// You should call this function with different seed values until the random + /// number lies within `u32::MAX - u32::MAX % n`. + /// Modification of `generate_random_number` from `pallet-lottery` version `4.0.0-dev`. fn generate_random_number(seed: u32) -> u32 { let (random_seed, _) = ::Randomness::random(&(b"ddc-validator", seed).encode()); From 0ef5dc53e5f27829a0c3b973c4d68d44e486d493 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 29 Mar 2023 18:00:40 +0600 Subject: [PATCH 081/192] Module level docs --- frame/ddc-validator/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index eb28a1d59c77e..6402d9f24b772 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,3 +1,37 @@ +//! # DDC Validator pallet +//! +//! The DDC Validator pallet is responsible for producing validation decisions based on activity +//! data from DAC DataModel. It is expected to work on validators nodes only. +//! +//! - [`Config`] +//! - [`Call`] +//! - [`Pallet`] +//! - [`Hooks`] +//! +//! ## Responsibility +//! +//! 1. Assign validation tasks on DAC Validators in the beginning of each era, +//! 2. Spin the offchain worker which tries to execute the validation tasks each era, +//! 3. Fetch the data required for validation from DAC DataModel, +//! 4. Execute validation method on this data, +//! 5. Produce validation decision and submit it to the chain. +//! +//! ## Usage +//! +//! 1. Run the node with `--validator` flag, +//! 2. Setup validator key with `author_insertKey` RPC call. Use `dacv` validator key type and the +//! same private key as the one used to generate the validator's session keys, +//! 3. Proceed a regular validator setup, +//! 4. Tasks assignment will assign you a task in the beginning of the era which has your account in +//! validators set. +//! +//! ## Notes +//! +//! - Era definition in this pallet is different than in the `pallet-staking`. In this pallet era is +//! a period of time during which the validator is expected to produce a validation decision. +//! Means staking era and DAC era are different and are not related to each other, +//! - You can set DAC Validators quorum size by specifying `DdcValidatorsQuorumSize` parameter, + #![cfg_attr(not(feature = "std"), no_std)] pub use alloc::{format, string::String}; From 3df787c49bd195dd88140ad1ba5efc42fb5f60e3 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 29 Mar 2023 22:51:19 +0200 Subject: [PATCH 082/192] Remove http request from tx --- frame/ddc-validator/src/lib.rs | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6402d9f24b772..0be9e9454a443 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -430,22 +430,32 @@ pub mod pallet { #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { + info!("[DAC Validator] processing proof_of_delivery"); let signer: T::AccountId = ensure_signed(origin)?; + info!("signer: {:?}", Self::account_to_string(signer.clone())); + let era = Self::get_current_era(); let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); - let (s, r) = Self::fetch_data1(era); + + info!("[DAC Validator] cdn_nodes_to_validate: {:?}", cdn_nodes_to_validate); + for cdn_node_id in cdn_nodes_to_validate { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); let decisions_for_cdn = >::get(era, cdn_node_id); - for decision in decisions_for_cdn.unwrap().iter_mut() { + for decision in decisions_for_cdn.clone().unwrap().iter_mut() { if decision.validator == signer { decision.decision = Some(val_res); decision.method = ValidationMethodKind::ProofOfDelivery; } } + + info!( + "[DAC Validator] decisions_for_cdn: {:?}", + decisions_for_cdn + ); } Ok(()) @@ -458,11 +468,6 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - info!( - "[DAC Validator] Validation data stored onchain: {:?}", - ValidationResults::::get() - ); - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { return Ok(()) } @@ -475,8 +480,6 @@ pub mod pallet { Ok(signer) => signer, }; - info!("[DAC Validator] ValidationResults: {:?}", ValidationResults::::get()); - // Read data from DataModel and do dumb validation let current_era = Self::get_current_era() - 1; let (s, r) = Self::fetch_data1(current_era); @@ -484,9 +487,13 @@ pub mod pallet { let tx_res = signer.send_signed_transaction(|_acct| { Call::proof_of_delivery { s: s.clone(), r: r.clone() } }); + match &tx_res { - None | Some((_, Err(()))) => - return Err("Error while submitting proof of delivery TX"), + None => return Err("Error while submitting proof of delivery TX"), + Some((_, Err(e))) => { + info!("Error while submitting proof of delivery TX: {:?}", e); + return Err("Error while submitting proof of delivery TX"); + }, Some((_, Ok(()))) => {}, } @@ -620,6 +627,8 @@ pub mod pallet { fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; for (cdn_id, cdn_tasks) in >::iter_prefix(era) { + info!("[DAC Validator] tasks assigned to {:?}: {:?}", cdn_id, cdn_tasks); + for decision in cdn_tasks.iter() { if decision.validator == *validator { cdn_nodes.push(cdn_id); From b59d09b350fc88376eaab34d0141832e02ffc999 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 30 Mar 2023 14:51:59 +0600 Subject: [PATCH 083/192] Submit DAC Validation decision call --- frame/ddc-validator/src/lib.rs | 50 ++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0be9e9454a443..b456baf621728 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -323,10 +323,22 @@ pub mod pallet { pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + /// DAC Validator successfully published the validation decision. + ValidationDecisionSubmitted, + } #[pallet::error] - pub enum Error {} + pub enum Error { + /// Validation decision attempts to submit the result for the wrong era (not the current + /// one). + BadEra, + /// Can't submit the validation decision twice. + DecisionAlreadySubmitted, + /// Task does not exist for a given era, CDN participant, and DAC validator. + TaskNotFound, + } #[pallet::hooks] impl Hooks> for Pallet @@ -428,6 +440,40 @@ pub mod pallet { Ok(()) } + /// Set validation decision in tasks assignment. + /// + /// `origin` must be a DAC Validator assigned to the task. + /// `era` must be a current era, otherwise the decision will be rejected. + /// `subject` is a CDN participant stash. + /// + /// Emits `ValidationDecisionSubmitted` event. + #[pallet::weight(100_000)] + pub fn submit_validation_decision( + origin: OriginFor, + era: EraIndex, + subject: T::AccountId, + method: ValidationMethodKind, + decision: bool, + ) -> DispatchResult { + let account = ensure_signed(origin)?; + + ensure!(Self::get_current_era() == era, Error::::BadEra); + + Tasks::::try_mutate_exists(era, &subject, |maybe_tasks| { + let mut tasks = maybe_tasks.take().ok_or(Error::::TaskNotFound)?; + let mut task = tasks + .iter_mut() + .find(|task| task.validator == account && task.method == method) + .ok_or(Error::::TaskNotFound)?; + ensure!(task.decision.is_none(), Error::::DecisionAlreadySubmitted); + task.decision = Some(decision); + + Self::deposit_event(Event::ValidationDecisionSubmitted); + + Ok(()) + }) + } + #[pallet::weight(10000)] pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { info!("[DAC Validator] processing proof_of_delivery"); From bdec8e77b92a4f597f8cca71c2f8f61235aa7799 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Fri, 14 Jan 2022 12:27:35 +0300 Subject: [PATCH 084/192] Set shorted duration --- bin/node/runtime/src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fb131673c4b7d..aa256885dd098 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -684,12 +684,12 @@ impl pallet_bags_list::Config for Runtime { } parameter_types! { - pub const LaunchPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const VotingPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const FastTrackVotingPeriod: BlockNumber = 3 * 60 * MINUTES; - pub const MinimumDeposit: Balance = 5000 * DOLLARS; - pub const EnactmentPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; - pub const CooloffPeriod: BlockNumber = 7 * 24 * 60 * MINUTES; + pub const LaunchPeriod: BlockNumber = 5 * MINUTES; + pub const VotingPeriod: BlockNumber = 5 * MINUTES; + pub const FastTrackVotingPeriod: BlockNumber = 5 * MINUTES; + pub const MinimumDeposit: Balance = 10 * DOLLARS; + pub const EnactmentPeriod: BlockNumber = 5 * MINUTES; + pub const CooloffPeriod: BlockNumber = 5 * MINUTES; pub const MaxProposals: u32 = 100; } @@ -745,7 +745,7 @@ impl pallet_democracy::Config for Runtime { } parameter_types! { - pub const CouncilMotionDuration: BlockNumber = 7 * DAYS; + pub const CouncilMotionDuration: BlockNumber = 5 * MINUTES; pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; } @@ -767,7 +767,7 @@ parameter_types! { // 1 storage item created, key size is 32 bytes, value size is 16+16. pub const VotingBondBase: Balance = deposit(1, 64); pub const VotingBondFactor: Balance = 1 * DOLLARS; - pub const TermDuration: BlockNumber = 7 * DAYS; + pub const TermDuration: BlockNumber = 5 * MINUTES; pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 20; pub const ElectionsPhragmenPalletId: LockIdentifier = *b"phrelect"; @@ -797,7 +797,7 @@ impl pallet_elections_phragmen::Config for Runtime { } parameter_types! { - pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS; + pub const TechnicalMotionDuration: BlockNumber = 5 * MINUTES; pub const TechnicalMaxProposals: u32 = 100; pub const TechnicalMaxMembers: u32 = 100; } @@ -834,9 +834,9 @@ impl pallet_membership::Config for Runtime { parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const SpendPeriod: BlockNumber = 1 * DAYS; + pub const SpendPeriod: BlockNumber = 5 * MINUTES; pub const Burn: Permill = Permill::from_percent(0); - pub const TipCountdown: BlockNumber = 1 * DAYS; + pub const TipCountdown: BlockNumber = 5 * MINUTES; pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const DataDepositPerByte: Balance = 1 * CENTS; @@ -1120,10 +1120,10 @@ parameter_types! { pub const CandidateDeposit: Balance = 10 * DOLLARS; pub const WrongSideDeduction: Balance = 2 * DOLLARS; pub const MaxStrikes: u32 = 10; - pub const RotationPeriod: BlockNumber = 80 * HOURS; + pub const RotationPeriod: BlockNumber = 5 * MINUTES; pub const PeriodSpend: Balance = 500 * DOLLARS; pub const MaxLockDuration: BlockNumber = 36 * 30 * DAYS; - pub const ChallengePeriod: BlockNumber = 7 * DAYS; + pub const ChallengePeriod: BlockNumber = 15 * MINUTES; pub const MaxCandidateIntake: u32 = 10; pub const SocietyPalletId: PalletId = PalletId(*b"py/socie"); } From 82bd1de9b26860f8e07e9f46c53db21d9e0f8d23 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 1 Nov 2022 09:37:49 +0300 Subject: [PATCH 085/192] Update EPOCH_DURATION_IN_BLOCKS --- bin/node/runtime/src/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index aa4dcd79be23d..f854b194e4e11 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -61,7 +61,7 @@ pub mod time { // 1 in 4 blocks (on average, not counting collisions) will be primary BABE blocks. pub const PRIMARY_PROBABILITY: (u64, u64) = (1, 4); - pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 4 * HOURS; + pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 5 * MINUTES; pub const EPOCH_DURATION_IN_SLOTS: u64 = { const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64; From 847d71ad7bd88df7e5beaa68b170d835e2a40d35 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 31 Mar 2023 10:27:35 +0200 Subject: [PATCH 086/192] Revert "Update EPOCH_DURATION_IN_BLOCKS" This reverts commit 3d4ea108ee378ae6ee089d85d8a335eb34ae53fb. --- bin/node/runtime/src/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index f854b194e4e11..aa4dcd79be23d 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -61,7 +61,7 @@ pub mod time { // 1 in 4 blocks (on average, not counting collisions) will be primary BABE blocks. pub const PRIMARY_PROBABILITY: (u64, u64) = (1, 4); - pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 5 * MINUTES; + pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 4 * HOURS; pub const EPOCH_DURATION_IN_SLOTS: u64 = { const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64; From db7d0d150e026163892588275b0fdfaaf94273cd Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 31 Mar 2023 10:28:14 +0200 Subject: [PATCH 087/192] Revert "Set shorted duration" This reverts commit 0239ff17acffbff56ce24a6492cdd03dc6906c49. --- bin/node/runtime/src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index aa256885dd098..fb131673c4b7d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -684,12 +684,12 @@ impl pallet_bags_list::Config for Runtime { } parameter_types! { - pub const LaunchPeriod: BlockNumber = 5 * MINUTES; - pub const VotingPeriod: BlockNumber = 5 * MINUTES; - pub const FastTrackVotingPeriod: BlockNumber = 5 * MINUTES; - pub const MinimumDeposit: Balance = 10 * DOLLARS; - pub const EnactmentPeriod: BlockNumber = 5 * MINUTES; - pub const CooloffPeriod: BlockNumber = 5 * MINUTES; + pub const LaunchPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const VotingPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const FastTrackVotingPeriod: BlockNumber = 3 * 60 * MINUTES; + pub const MinimumDeposit: Balance = 5000 * DOLLARS; + pub const EnactmentPeriod: BlockNumber = 1 * 24 * 60 * MINUTES; + pub const CooloffPeriod: BlockNumber = 7 * 24 * 60 * MINUTES; pub const MaxProposals: u32 = 100; } @@ -745,7 +745,7 @@ impl pallet_democracy::Config for Runtime { } parameter_types! { - pub const CouncilMotionDuration: BlockNumber = 5 * MINUTES; + pub const CouncilMotionDuration: BlockNumber = 7 * DAYS; pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; } @@ -767,7 +767,7 @@ parameter_types! { // 1 storage item created, key size is 32 bytes, value size is 16+16. pub const VotingBondBase: Balance = deposit(1, 64); pub const VotingBondFactor: Balance = 1 * DOLLARS; - pub const TermDuration: BlockNumber = 5 * MINUTES; + pub const TermDuration: BlockNumber = 7 * DAYS; pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 20; pub const ElectionsPhragmenPalletId: LockIdentifier = *b"phrelect"; @@ -797,7 +797,7 @@ impl pallet_elections_phragmen::Config for Runtime { } parameter_types! { - pub const TechnicalMotionDuration: BlockNumber = 5 * MINUTES; + pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS; pub const TechnicalMaxProposals: u32 = 100; pub const TechnicalMaxMembers: u32 = 100; } @@ -834,9 +834,9 @@ impl pallet_membership::Config for Runtime { parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const SpendPeriod: BlockNumber = 5 * MINUTES; + pub const SpendPeriod: BlockNumber = 1 * DAYS; pub const Burn: Permill = Permill::from_percent(0); - pub const TipCountdown: BlockNumber = 5 * MINUTES; + pub const TipCountdown: BlockNumber = 1 * DAYS; pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const DataDepositPerByte: Balance = 1 * CENTS; @@ -1120,10 +1120,10 @@ parameter_types! { pub const CandidateDeposit: Balance = 10 * DOLLARS; pub const WrongSideDeduction: Balance = 2 * DOLLARS; pub const MaxStrikes: u32 = 10; - pub const RotationPeriod: BlockNumber = 5 * MINUTES; + pub const RotationPeriod: BlockNumber = 80 * HOURS; pub const PeriodSpend: Balance = 500 * DOLLARS; pub const MaxLockDuration: BlockNumber = 36 * 30 * DAYS; - pub const ChallengePeriod: BlockNumber = 15 * MINUTES; + pub const ChallengePeriod: BlockNumber = 7 * DAYS; pub const MaxCandidateIntake: u32 = 10; pub const SocietyPalletId: PalletId = PalletId(*b"py/socie"); } From a21122706f93d1c1595107ef5e3708984debd3b3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 31 Mar 2023 17:16:21 +0600 Subject: [PATCH 088/192] Fix validation decision update --- frame/ddc-validator/src/lib.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b456baf621728..1f10608c9e245 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -490,18 +490,17 @@ pub mod pallet { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - let decisions_for_cdn = >::get(era, cdn_node_id); - for decision in decisions_for_cdn.clone().unwrap().iter_mut() { - if decision.validator == signer { - decision.decision = Some(val_res); - decision.method = ValidationMethodKind::ProofOfDelivery; - } - } - - info!( - "[DAC Validator] decisions_for_cdn: {:?}", - decisions_for_cdn - ); + >::mutate(era, cdn_node_id, |decisions_for_cdn| { + let decisions = + decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); + let mut decision = decisions + .iter_mut() + .find(|decision| decision.validator == signer) + .expect("unexpected validators set in tasks assignment"); + decision.decision = Some(val_res); + }); + + info!("[DAC Validator] decisions_for_cdn: {:?}", >::get(era, cdn_node_id)); } Ok(()) From 2b8151430ba16ff81a4fa36c60f274346efcc4d6 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 31 Mar 2023 17:33:09 +0600 Subject: [PATCH 089/192] Typo fix --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 1f10608c9e245..71c81b97f8499 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -490,7 +490,7 @@ pub mod pallet { let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - >::mutate(era, cdn_node_id, |decisions_for_cdn| { + >::mutate(era, &cdn_node_id, |decisions_for_cdn| { let decisions = decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); let mut decision = decisions From c2c6d5fd09d525f42b7fed2b131b0c42988e2a22 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 10 Apr 2023 12:24:52 +0600 Subject: [PATCH 090/192] Integer type ValidationThreshold --- frame/ddc-validator/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 71c81b97f8499..b1c65fe416b36 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -67,7 +67,7 @@ pub use sp_std::prelude::*; extern crate alloc; parameter_types! { - pub const ValidationThreshold: f32 = 5.0; + pub const ValidationThreshold: u32 = 5; } /// The balance type of this pallet. @@ -665,7 +665,13 @@ pub mod pallet { fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); - return if percentage_difference > 0.0 && (ValidationThreshold::get() - percentage_difference) > 0.0 { true } else { false } + return if percentage_difference > 0.0 && + (ValidationThreshold::get() as f32 - percentage_difference) > 0.0 + { + true + } else { + false + } } /// Fetch the tasks related to current validator From 7a0e601cb769d107d8dfe72347a0c41ee435d7a9 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 10 Apr 2023 12:27:05 +0600 Subject: [PATCH 091/192] Autoformat DAC Validator files --- frame/ddc-validator/src/lib.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b1c65fe416b36..976a5131f9387 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -400,7 +400,8 @@ pub mod pallet { } log::info!( "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", - pubkeys, pubkeys.first().unwrap() + pubkeys, + pubkeys.first().unwrap() ); let res = Self::offchain_worker_main(block_number); @@ -475,7 +476,11 @@ pub mod pallet { } #[pallet::weight(10000)] - pub fn proof_of_delivery(origin: OriginFor, s: Vec, r: Vec) -> DispatchResult { + pub fn proof_of_delivery( + origin: OriginFor, + s: Vec, + r: Vec, + ) -> DispatchResult { info!("[DAC Validator] processing proof_of_delivery"); let signer: T::AccountId = ensure_signed(origin)?; @@ -529,15 +534,16 @@ pub mod pallet { let current_era = Self::get_current_era() - 1; let (s, r) = Self::fetch_data1(current_era); - let tx_res = signer.send_signed_transaction(|_acct| { - Call::proof_of_delivery { s: s.clone(), r: r.clone() } + let tx_res = signer.send_signed_transaction(|_acct| Call::proof_of_delivery { + s: s.clone(), + r: r.clone(), }); match &tx_res { None => return Err("Error while submitting proof of delivery TX"), Some((_, Err(e))) => { info!("Error while submitting proof of delivery TX: {:?}", e); - return Err("Error while submitting proof of delivery TX"); + return Err("Error while submitting proof of delivery TX") }, Some((_, Ok(()))) => {}, } From 5e811fb38139a57c29ccd367bd262c42710d39f2 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 10 Apr 2023 13:20:16 +0600 Subject: [PATCH 092/192] Set PoD allowed deviation in runtime crate --- bin/node/runtime/src/lib.rs | 2 ++ frame/ddc-validator/src/lib.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fb131673c4b7d..f7141dd1d7b7a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1235,6 +1235,7 @@ impl pallet_ddc_staking::Config for Runtime { parameter_types! { pub const DdcValidatorsQuorumSize: u32 = 3; + pub const ValidationThreshold: u32 = 5; } impl pallet_ddc_validator::Config for Runtime { @@ -1244,6 +1245,7 @@ impl pallet_ddc_validator::Config for Runtime { type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; + type ValidationThreshold = ValidationThreshold; } construct_runtime!( diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 976a5131f9387..157d6fa0b2551 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -66,10 +66,6 @@ pub use sp_std::prelude::*; extern crate alloc; -parameter_types! { - pub const ValidationThreshold: u32 = 5; -} - /// The balance type of this pallet. type BalanceOf = <::Currency as Currency< ::AccountId, @@ -294,6 +290,13 @@ pub mod pallet { /// validators are getting the same task. Must be an odd number. #[pallet::constant] type DdcValidatorsQuorumSize: Get; + + /// Proof-of-Delivery parameter specifies an allowed deviation between bytes sent and bytes + /// received. The deviation is expressed as a percentage. For example, if the value is 10, + /// then the difference between bytes sent and bytes received is allowed to be up to 10%. + /// The value must be in range [0, 100]. + #[pallet::constant] + type ValidationThreshold: Get; } /// The map from the era and CDN participant stash key to the validation decisions related. @@ -672,7 +675,7 @@ pub mod pallet { let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); return if percentage_difference > 0.0 && - (ValidationThreshold::get() as f32 - percentage_difference) > 0.0 + (T::ValidationThreshold::get() as f32 - percentage_difference) > 0.0 { true } else { From 84d4a029691be026b8f7192af4dcc4101ee40ecc Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 13 Apr 2023 15:16:22 +0200 Subject: [PATCH 093/192] Update spec version --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index f7141dd1d7b7a..e03d634037f9d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -130,7 +130,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 30400, + spec_version: 30401, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, From b70df0712c091f89fbb7ed41d51c292a43fc83a1 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 14 Apr 2023 18:35:48 +0600 Subject: [PATCH 094/192] DAC Validators run a process by signal --- frame/ddc-validator/src/lib.rs | 183 +++++++++++++++++++++++---------- 1 file changed, 130 insertions(+), 53 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 157d6fa0b2551..0ebd3f3f0f020 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -81,7 +81,8 @@ const TIME_START_MS: u128 = 1_672_531_200_000; const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; -const DATA_PROVIDER_URL: &str = "http://localhost:7379/"; +/// Webdis in experimental cluster connected to Redis in dev. +const DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379/"; /// DAC Validation methods. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] @@ -299,6 +300,11 @@ pub mod pallet { type ValidationThreshold: Get; } + /// A signal to start a process on all the validators. + #[pallet::storage] + #[pallet::getter(fn signal)] + pub(super) type Signal = StorageValue<_, bool>; + /// The map from the era and CDN participant stash key to the validation decisions related. #[pallet::storage] #[pallet::getter(fn tasks)] @@ -350,69 +356,108 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - if block_number <= 1u32.into() { - return 0 - } - - let era = Self::get_current_era(); - if let Some(last_managed_era) = >::get() { - if last_managed_era >= era { - return 0 - } + // Reset the signal in the beginning of the block to keep it reset until an incoming + // transaction sets it to true. + if Signal::::get().unwrap_or(false) { + Signal::::set(Some(false)); } - >::put(era); - let validators: Vec = >::iter_keys().collect(); - let validators_count = validators.len() as u32; - let edges: Vec = >::iter_keys().collect(); - log::info!( - "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, validators: {:?}, edges: {:?}", - block_number, - era, - >::get(), - validators_count, - validators, - edges, - ); - - // A naive approach assigns random validators for each edge. - for edge in edges { - let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = - Default::default(); - while !decisions.is_full() { - let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - let validator: T::AccountId = validators[validator_idx].clone(); - let assignment = Decision { - validator, - method: ValidationMethodKind::ProofOfDelivery, - decision: None, - }; - decisions.try_push(assignment).unwrap(); - } - Tasks::::insert(era, edge, decisions); - } + // Old task manager. + // + // if block_number <= 1u32.into() { + // return 0 + // } + + // let era = Self::get_current_era(); + // if let Some(last_managed_era) = >::get() { + // if last_managed_era >= era { + // return 0 + // } + // } + // >::put(era); + + // let validators: Vec = >::iter_keys().collect(); + // let validators_count = validators.len() as u32; + // let edges: Vec = + // >::iter_keys().collect(); log::info!( + // "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, + // validators: {:?}, edges: {:?}", block_number, + // era, + // >::get(), + // validators_count, + // validators, + // edges, + // ); + + // // A naive approach assigns random validators for each edge. + // for edge in edges { + // let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = + // Default::default(); + // while !decisions.is_full() { + // let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + // let validator: T::AccountId = validators[validator_idx].clone(); + // let assignment = Decision { + // validator, + // method: ValidationMethodKind::ProofOfDelivery, + // decision: None, + // }; + // decisions.try_push(assignment).unwrap(); + // } + // Tasks::::insert(era, edge, decisions); + // } 0 } + /// Offchain worker entry point. + /// + /// 1. Listen to a signal, + /// 2. Run a process at the same time, + /// 3. Read data from DAC. fn offchain_worker(block_number: T::BlockNumber) { - let pubkeys = sr25519_public_keys(KEY_TYPE); - if pubkeys.is_empty() { - log::info!("No local sr25519 accounts available to offchain worker."); + // Skip if not a validator. + if !sp_io::offchain::is_validator() { + return + } + + // Wait for signal. + let signal = Signal::::get().unwrap_or(false); + if !signal { + log::info!("🔎 DAC Validator is idle at block {:?}, waiting for a signal, signal state is {:?}", block_number, signal); return } + + // Read from DAC. + let current_era = Self::get_current_era(); + let (sent_query, sent, received_query, received) = Self::fetch_data2(current_era - 1); log::info!( - "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", - pubkeys, - pubkeys.first().unwrap() + "🔎 DAC Validator is fetching data from DAC, current era: {:?}, bytes sent query: {:?}, bytes sent response: {:?}, bytes received query: {:?}, bytes received response: {:?}", + current_era, + sent_query, + sent, + received_query, + received, ); - let res = Self::offchain_worker_main(block_number); - - match res { - Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), - Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), - }; + // Old off-chain worker. + // + // let pubkeys = sr25519_public_keys(KEY_TYPE); + // if pubkeys.is_empty() { + // log::info!("No local sr25519 accounts available to offchain worker."); + // return + // } + // log::info!( + // "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", + // pubkeys, + // pubkeys.first().unwrap() + // ); + + // let res = Self::offchain_worker_main(block_number); + + // match res { + // Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + // Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), + // }; } } @@ -422,6 +467,25 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { + /// Run a process at the same time on all the validators. + #[pallet::weight(10_000)] + pub fn send_signal(origin: OriginFor) -> DispatchResult { + ensure_signed(origin)?; + + Signal::::set(Some(true)); + + Ok(()) + } + + #[pallet::weight(10_000)] + pub fn reset_signal(origin: OriginFor) -> DispatchResult { + ensure_signed(origin)?; + + Signal::::set(Some(false)); + + Ok(()) + } + #[pallet::weight(10000)] pub fn save_validated_data( origin: OriginFor, @@ -626,6 +690,19 @@ pub mod pallet { (bytes_sent, bytes_received) } + fn fetch_data2(era: EraIndex) -> (String, Vec, String, Vec) { + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + let bytes_sent = BytesSent::get_all(bytes_sent_res); + + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = + Self::http_get_json(&bytes_received_query).unwrap(); + let bytes_received = BytesReceived::get_all(bytes_received_res); + + (bytes_sent_query, bytes_sent, bytes_received_query, bytes_received) + } + fn get_bytes_sent_query_url(era: EraIndex) -> String { format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) } @@ -649,7 +726,7 @@ pub mod pallet { } fn http_get_request(http_url: &str) -> Result, http::Error> { - info!("[DAC Validator] Sending request to: {:?}", http_url); + // info!("[DAC Validator] Sending request to: {:?}", http_url); // Initiate an external HTTP GET request. This is using high-level wrappers from // `sp_runtime`. From c30b46e217efe8f2907146f4ecf2a8772f7db8db Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 14 Apr 2023 18:57:07 +0600 Subject: [PATCH 095/192] Update Cargo.lock --- Cargo.lock | 401 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 264 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2869c996bec3..6c0be527c6aa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -485,12 +485,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -543,6 +537,7 @@ dependencies = [ "sp-finality-grandpa", "sp-keyring", "sp-keystore", + "sp-mmr-primitives", "sp-runtime", "sp-tracing", "strum", @@ -867,12 +862,6 @@ dependencies = [ "iovec", ] -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.4.0" @@ -1205,11 +1194,11 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9516ba6b2ba47b4cbf63b713f75b432fafa0a0e0464ec8381ec76e6efe931ab3" +checksum = "38faa2a16616c8e78a18d37b4726b98bfd2de192f2fdc8a39ddf568a408a0f75" dependencies = [ - "cranelift-entity 0.80.0", + "cranelift-entity 0.82.3", ] [[package]] @@ -1231,17 +1220,17 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489e5d0081f7edff6be12d71282a8bf387b5df64d5592454b75d662397f2d642" +checksum = "26f192472a3ba23860afd07d2b0217dc628f21fcc72617aa1336d98e1671f33b" dependencies = [ - "cranelift-bforest 0.80.0", - "cranelift-codegen-meta 0.80.0", - "cranelift-codegen-shared 0.80.0", - "cranelift-entity 0.80.0", + "cranelift-bforest 0.82.3", + "cranelift-codegen-meta 0.82.3", + "cranelift-codegen-shared 0.82.3", + "cranelift-entity 0.82.3", "gimli 0.26.1", "log 0.4.17", - "regalloc 0.0.33", + "regalloc 0.0.34", "smallvec 1.8.0", "target-lexicon", ] @@ -1258,11 +1247,11 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36ee1140371bb0f69100e734b30400157a4adf7b86148dee8b0a438763ead48" +checksum = "0f32ddb89e9b89d3d9b36a5b7d7ea3261c98235a76ac95ba46826b8ec40b1a24" dependencies = [ - "cranelift-codegen-shared 0.80.0", + "cranelift-codegen-shared 0.82.3", ] [[package]] @@ -1273,9 +1262,9 @@ checksum = "9dabb5fe66e04d4652e434195b45ae65b5c8172d520247b8f66d8df42b2b45dc" [[package]] name = "cranelift-codegen-shared" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981da52d8f746af1feb96290c83977ff8d41071a7499e991d8abae0d4869f564" +checksum = "01fd0d9f288cc1b42d9333b7a776b17e278fc888c28e6a0f09b5573d45a150bc" [[package]] name = "cranelift-entity" @@ -1285,9 +1274,9 @@ checksum = "3329733e4d4b8e91c809efcaa4faee80bf66f20164e3dd16d707346bd3494799" [[package]] name = "cranelift-entity" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2906740053dd3bcf95ce53df0fd9b5649c68ae4bd9adada92b406f059eae461" +checksum = "9e3bfe172b83167604601faf9dc60453e0d0a93415b57a9c4d1a7ae6849185cf" dependencies = [ "serde", ] @@ -1306,11 +1295,11 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7" +checksum = "a006e3e32d80ce0e4ba7f1f9ddf66066d052a8c884a110b91d05404d6ce26dce" dependencies = [ - "cranelift-codegen 0.80.0", + "cranelift-codegen 0.82.3", "log 0.4.17", "smallvec 1.8.0", "target-lexicon", @@ -1318,28 +1307,28 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166028ca0343a6ee7bddac0e70084e142b23f99c701bd6f6ea9123afac1a7a46" +checksum = "501241b0cdf903412ec9075385ac9f2b1eb18a89044d1538e97fab603231f70c" dependencies = [ - "cranelift-codegen 0.80.0", + "cranelift-codegen 0.82.3", "libc", "target-lexicon", ] [[package]] name = "cranelift-wasm" -version = "0.80.0" +version = "0.82.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5012a1cde0c8b3898770b711490d803018ae9bec2d60674ba0e5b2058a874f80" +checksum = "16d9e4211bbc3268042a96dd4de5bd979cda22434991d035f5f8eacba987fad2" dependencies = [ - "cranelift-codegen 0.80.0", - "cranelift-entity 0.80.0", - "cranelift-frontend 0.80.0", + "cranelift-codegen 0.82.3", + "cranelift-entity 0.82.3", + "cranelift-frontend 0.82.3", "itertools", "log 0.4.17", "smallvec 1.8.0", - "wasmparser 0.81.0", + "wasmparser 0.83.0", "wasmtime-types", ] @@ -1729,6 +1718,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users 0.3.5", + "winapi 0.3.9", +] + [[package]] name = "dirs-sys" version = "0.3.6" @@ -1736,7 +1736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1747,7 +1747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1896,6 +1896,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "enum-as-inner" version = "0.3.3" @@ -2225,13 +2231,16 @@ dependencies = [ "log 0.4.17", "memory-db", "parity-scale-codec", + "prettytable-rs", "rand 0.8.4", + "rand_pcg 0.3.1", "sc-block-builder", "sc-cli", "sc-client-api", "sc-client-db", "sc-executor", "sc-service", + "sc-sysinfo", "serde", "serde_json 1.0.79", "serde_nanos", @@ -2244,9 +2253,9 @@ dependencies = [ "sp-keystore", "sp-runtime", "sp-state-machine", - "sp-std", "sp-storage", "sp-trie", + "tempfile", "thousands", ] @@ -3209,9 +3218,9 @@ dependencies = [ [[package]] name = "impl-trait-for-tuples" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ "proc-macro2", "quote", @@ -3249,12 +3258,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.4.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ef6787e7f0faedc040f95716bdd0e62bcfcf4ba93da053b62dea2691c13864" -dependencies = [ - "winapi 0.3.9", -] +checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6" [[package]] name = "iovec" @@ -3488,7 +3494,7 @@ dependencies = [ "jsonrpsee-types", "pin-project 1.0.10", "rustls-native-certs 0.6.1", - "soketto 0.7.1", + "soketto", "thiserror", "tokio", "tokio-rustls 0.23.2", @@ -3514,7 +3520,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json 1.0.79", - "soketto 0.7.1", + "soketto", "thiserror", "tokio", "tracing", @@ -4077,7 +4083,7 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.17", - "lru 0.7.3", + "lru 0.7.8", "rand 0.7.3", "smallvec 1.8.0", "unsigned-varint 0.7.0", @@ -4166,7 +4172,7 @@ dependencies = [ "log 0.4.17", "quicksink", "rw-stream-sink", - "soketto 0.7.1", + "soketto", "url 2.2.1", "webpki-roots 0.21.0", ] @@ -4286,9 +4292,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.36" +version = "0.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a261afc61b7a5e323933b402ca6a1765183687c614789b1e4db7762ed4230bca" +checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7" [[package]] name = "lite-json" @@ -4377,11 +4383,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.7.3" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb87f3080f6d1d69e8c564c0fcfde1d7aa8cc451ce40cae89479111f03bc0eb" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown 0.11.2", + "hashbrown 0.12.0", ] [[package]] @@ -4903,12 +4909,13 @@ dependencies = [ "sc-service", "sc-service-test", "sc-sync-state-rpc", + "sc-sysinfo", "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", "serde", "serde_json 1.0.79", - "soketto 0.4.2", + "soketto", "sp-api", "sp-authority-discovery", "sp-authorship", @@ -5058,6 +5065,7 @@ dependencies = [ "pallet-ddc-validator", "pallet-democracy", "pallet-election-provider-multi-phase", + "pallet-election-provider-support-benchmarking", "pallet-elections-phragmen", "pallet-erc20", "pallet-erc721", @@ -5067,6 +5075,8 @@ dependencies = [ "pallet-indices", "pallet-membership", "pallet-multisig", + "pallet-nomination-pools", + "pallet-nomination-pools-benchmarking", "pallet-offences", "pallet-offences-benchmarking", "pallet-proxy", @@ -5602,7 +5612,6 @@ dependencies = [ "log 0.4.17", "pallet-beefy", "pallet-mmr", - "pallet-mmr-primitives", "pallet-session", "parity-scale-codec", "scale-info", @@ -5904,6 +5913,7 @@ dependencies = [ "frame-system", "log 0.4.17", "pallet-balances", + "pallet-election-provider-support-benchmarking", "parity-scale-codec", "parking_lot 0.12.0", "rand 0.7.3", @@ -5919,6 +5929,18 @@ dependencies = [ "strum", ] +[[package]] +name = "pallet-election-provider-support-benchmarking" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", + "frame-system", + "parity-scale-codec", + "sp-npos-elections", + "sp-runtime", +] + [[package]] name = "pallet-elections-phragmen" version = "5.0.0-dev" @@ -6120,27 +6142,11 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "pallet-mmr-primitives", "parity-scale-codec", "scale-info", "sp-core", "sp-io", - "sp-runtime", - "sp-std", -] - -[[package]] -name = "pallet-mmr-primitives" -version = "4.0.0-dev" -dependencies = [ - "frame-support", - "frame-system", - "hex-literal", - "log 0.4.17", - "parity-scale-codec", - "serde", - "sp-api", - "sp-core", + "sp-mmr-primitives", "sp-runtime", "sp-std", ] @@ -6152,13 +6158,13 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "pallet-mmr-primitives", "parity-scale-codec", "serde", "serde_json 1.0.79", "sp-api", "sp-blockchain", "sp-core", + "sp-mmr-primitives", "sp-runtime", ] @@ -6208,6 +6214,46 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-nomination-pools" +version = "1.0.0" +dependencies = [ + "frame-support", + "frame-system", + "pallet-balances", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", + "sp-tracing", +] + +[[package]] +name = "pallet-nomination-pools-benchmarking" +version = "1.0.0" +dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", + "frame-support", + "frame-system", + "pallet-bags-list", + "pallet-balances", + "pallet-nomination-pools", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", +] + [[package]] name = "pallet-offences" version = "4.0.0-dev" @@ -7181,6 +7227,20 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "prettytable-rs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" +dependencies = [ + "atty", + "csv", + "encode_unicode", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "primitive-types" version = "0.11.1" @@ -7662,6 +7722,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + [[package]] name = "redox_users" version = "0.4.0" @@ -7705,9 +7776,9 @@ dependencies = [ [[package]] name = "regalloc" -version = "0.0.33" +version = "0.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" dependencies = [ "log 0.4.17", "rustc-hash", @@ -7878,6 +7949,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils 0.8.5", +] + [[package]] name = "rustc-demangle" version = "0.1.18" @@ -7916,9 +7999,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.31.3" +version = "0.33.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2dcfc2778a90e38f56a708bfc90572422e11d6c7ee233d053d1f782cf9df6d2" +checksum = "938a344304321a9da4973b9ff4f9f8db9caf4597dfd9dda6a60b523340a0fff0" dependencies = [ "bitflags", "errno", @@ -8502,7 +8585,7 @@ dependencies = [ "env_logger 0.9.0", "hex-literal", "lazy_static", - "lru 0.7.3", + "lru 0.7.8", "parity-scale-codec", "parking_lot 0.12.0", "paste", @@ -8712,7 +8795,7 @@ dependencies = [ "linked-hash-map", "linked_hash_set", "log 0.4.17", - "lru 0.7.3", + "lru 0.7.8", "parity-scale-codec", "parking_lot 0.12.0", "pin-project 1.0.10", @@ -8756,7 +8839,7 @@ dependencies = [ "futures-timer", "libp2p", "log 0.4.17", - "lru 0.7.3", + "lru 0.7.8", "quickcheck", "sc-network", "sp-runtime", @@ -8972,6 +9055,7 @@ dependencies = [ "sc-offchain", "sc-rpc", "sc-rpc-server", + "sc-sysinfo", "sc-telemetry", "sc-tracing", "sc-transaction-pool", @@ -9076,6 +9160,24 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sc-sysinfo" +version = "6.0.0-dev" +dependencies = [ + "futures 0.3.21", + "libc", + "log 0.4.17", + "rand 0.7.3", + "rand_pcg 0.2.1", + "regex", + "sc-telemetry", + "serde", + "serde_json 1.0.79", + "sp-core", + "sp-io", + "sp-std", +] + [[package]] name = "sc-telemetry" version = "4.0.0-dev" @@ -9669,21 +9771,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "soketto" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c71ed3d54db0a699f4948e1bb3e45b450fa31fe602621dee6680361d569c88" -dependencies = [ - "base64 0.12.3", - "bytes 0.5.6", - "futures 0.3.21", - "httparse", - "log 0.4.17", - "rand 0.7.3", - "sha-1 0.9.4", -] - [[package]] name = "soketto" version = "0.7.1" @@ -9840,7 +9927,7 @@ version = "4.0.0-dev" dependencies = [ "futures 0.3.21", "log 0.4.17", - "lru 0.7.3", + "lru 0.7.8", "parity-scale-codec", "parking_lot 0.12.0", "sp-api", @@ -10135,6 +10222,21 @@ dependencies = [ "zstd", ] +[[package]] +name = "sp-mmr-primitives" +version = "4.0.0-dev" +dependencies = [ + "hex-literal", + "log 0.4.17", + "parity-scale-codec", + "serde", + "sp-api", + "sp-core", + "sp-debug-derive", + "sp-runtime", + "sp-std", +] + [[package]] name = "sp-npos-elections" version = "4.0.0-dev" @@ -10912,6 +11014,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -12119,34 +12232,33 @@ checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" [[package]] name = "wasmparser" -version = "0.81.0" +version = "0.83.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98930446519f63d00a836efdc22f67766ceae8dbcc1571379f2bcabc6b2b9abc" +checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a" [[package]] name = "wasmtime" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414be1bc5ca12e755ffd3ff7acc3a6d1979922f8237fc34068b2156cebcc3270" +checksum = "21ffb4705016d5ca91e18a72ed6822dab50e6d5ddd7045461b17ef19071cdef1" dependencies = [ "anyhow", "backtrace", "bincode", "cfg-if 1.0.0", - "cpp_demangle", "indexmap", "lazy_static", "libc", "log 0.4.17", "object 0.27.1", + "once_cell", "paste", "psm", "rayon", "region 2.2.0", - "rustc-demangle", "serde", "target-lexicon", - "wasmparser 0.81.0", + "wasmparser 0.83.0", "wasmtime-cache", "wasmtime-cranelift", "wasmtime-environ", @@ -12157,9 +12269,9 @@ dependencies = [ [[package]] name = "wasmtime-cache" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9b4cd1949206fda9241faf8c460a7d797aa1692594d3dd6bc1cbfa57ee20d0" +checksum = "85c6ab24291fa7cb3a181f5669f6c72599b7ef781669759b45c7828c5999d0c0" dependencies = [ "anyhow", "base64 0.13.0", @@ -12177,14 +12289,14 @@ dependencies = [ [[package]] name = "wasmtime-cranelift" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4693d33725773615a4c9957e4aa731af57b27dca579702d1d8ed5750760f1a9" +checksum = "f04c810078a491b7bc4866ebe045f714d2b95e6b539e1f64009a4a7606be11de" dependencies = [ "anyhow", - "cranelift-codegen 0.80.0", - "cranelift-entity 0.80.0", - "cranelift-frontend 0.80.0", + "cranelift-codegen 0.82.3", + "cranelift-entity 0.82.3", + "cranelift-frontend 0.82.3", "cranelift-native", "cranelift-wasm", "gimli 0.26.1", @@ -12193,18 +12305,18 @@ dependencies = [ "object 0.27.1", "target-lexicon", "thiserror", - "wasmparser 0.81.0", + "wasmparser 0.83.0", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b17e47116a078b9770e6fb86cff8b9a660826623cebcfff251b047c8d8993ef" +checksum = "61448266ea164b1ac406363cdcfac81c7c44db4d94c7a81c8620ac6c5c6cdf59" dependencies = [ "anyhow", - "cranelift-entity 0.80.0", + "cranelift-entity 0.82.3", "gimli 0.26.1", "indexmap", "log 0.4.17", @@ -12213,44 +12325,58 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasmparser 0.81.0", + "wasmparser 0.83.0", "wasmtime-types", ] [[package]] name = "wasmtime-jit" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60ea5b380bdf92e32911400375aeefb900ac9d3f8e350bb6ba555a39315f2ee7" +checksum = "156b4623c6b0d4b8c24afb846c20525922f538ef464cc024abab7ea8de2109a2" dependencies = [ "addr2line", "anyhow", "bincode", "cfg-if 1.0.0", + "cpp_demangle", "gimli 0.26.1", + "log 0.4.17", "object 0.27.1", "region 2.2.0", + "rustc-demangle", "rustix", "serde", "target-lexicon", "thiserror", "wasmtime-environ", + "wasmtime-jit-debug", "wasmtime-runtime", "winapi 0.3.9", ] +[[package]] +name = "wasmtime-jit-debug" +version = "0.35.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5dc31f811760a6c76b2672c404866fd19b75e5fb3b0075a3e377a6846490654" +dependencies = [ + "lazy_static", + "object 0.27.1", + "rustix", +] + [[package]] name = "wasmtime-runtime" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc7cd79937edd6e238b337608ebbcaf9c086a8457f01dfd598324f7fa56d81a" +checksum = "f907beaff69d4d920fa4688411ee4cc75c0f01859e424677f9e426e2ef749864" dependencies = [ "anyhow", "backtrace", "cc", "cfg-if 1.0.0", "indexmap", - "lazy_static", "libc", "log 0.4.17", "mach", @@ -12261,19 +12387,20 @@ dependencies = [ "rustix", "thiserror", "wasmtime-environ", + "wasmtime-jit-debug", "winapi 0.3.9", ] [[package]] name = "wasmtime-types" -version = "0.33.0" +version = "0.35.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e5e51a461a2cf2b69e1fc48f325b17d78a8582816e18479e8ead58844b23f8" +checksum = "514ef0e5fd197b9609dc9eb74beba0c84d5a12b2417cbae55534633329ba4852" dependencies = [ - "cranelift-entity 0.80.0", + "cranelift-entity 0.82.3", "serde", "thiserror", - "wasmparser 0.81.0", + "wasmparser 0.83.0", ] [[package]] @@ -12570,18 +12697,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.9.0+zstd.1.5.0" +version = "0.10.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd" +checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.1+zstd.1.5.0" +version = "4.1.6+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079" +checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb" dependencies = [ "libc", "zstd-sys", @@ -12589,9 +12716,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.1+zstd.1.5.0" +version = "1.6.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33" +checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" dependencies = [ "cc", "libc", From ae637698061e54bea0b3e5cf3b45bb9f0ee16166 Mon Sep 17 00:00:00 2001 From: yahortsaryk Date: Sat, 15 Apr 2023 02:34:51 +0200 Subject: [PATCH 096/192] fix(ddc-validator): compilation error is fixed --- Cargo.lock | 1 + frame/ddc-validator/Cargo.toml | 3 ++ frame/ddc-validator/src/mock.rs | 79 ++++++++++++++++++++++++++++----- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07fb5329e9203..2e65369fae049 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5857,6 +5857,7 @@ version = "0.1.0" dependencies = [ "alt_serde", "array-bytes", + "frame-election-provider-support", "frame-support", "frame-system", "log 0.4.17", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 203e032e1e4d4..eeaa054367980 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -9,6 +9,8 @@ alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } + log = { version = "0.4.17", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } @@ -30,6 +32,7 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", + "frame-election-provider-support/std", "pallet-contracts/std", "pallet-ddc-staking/std", "pallet-session/std", diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index b40c3a1c05ee7..76cde6fffc1ee 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -10,6 +10,22 @@ use sp_core::H256; use sp_runtime::{ curve::PiecewiseLinear, generic, testing::{Header, TestXt}, traits::{BlakeTwo256, IdentityLookup, Verify, Extrinsic as ExtrinsicT, IdentifyAccount}, Perbill, MultiSignature, curve}; use pallet_contracts as contracts; +use sp_runtime::traits::Convert; +use pallet_session::ShouldEndSession; +use sp_runtime::{ + impl_opaque_keys, + testing::{UintAuthorityId}, +}; +use sp_staking::SessionIndex; +use frame_support::{ + traits::{U128CurrencyToVote} +}; + +use frame_election_provider_support::{ + onchain, SequentialPhragmen +}; + + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; type Balance = u128; @@ -32,6 +48,7 @@ frame_support::construct_runtime!( Balances: pallet_balances, Contracts: contracts, Session: pallet_session, + Staking: pallet_staking, Timestamp: pallet_timestamp, RandomnessCollectiveFlip: pallet_randomness_collective_flip, DdcStaking: pallet_ddc_staking, @@ -90,13 +107,20 @@ use contracts::{Config as contractsConfig}; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +pub struct TestWeightToFee; +impl Convert for TestWeightToFee { + fn convert(weight: u64) -> u128 { + weight as u128 + } +} + impl contracts::Config for Test { type Time = Timestamp; type Randomness = RandomnessCollectiveFlip; type Currency = Balances; type Event = Event; type CallStack = [pallet_contracts::Frame; 31]; - type WeightPrice = Self; //pallet_transaction_payment::Module; + type WeightPrice = TestWeightToFee; //pallet_transaction_payment::Module; type WeightInfo = (); type ChainExtension = (); type DeletionQueueDepth = (); @@ -132,15 +156,34 @@ parameter_types! { pub const DdcValidatorsQuorumSize: u32 = 3; } +pub struct TestShouldEndSession; +impl ShouldEndSession for TestShouldEndSession { + fn should_end_session(now: u32) -> bool { + now % 10 == 0 // every 10 blocks + } +} + +impl_opaque_keys! { + pub struct MockSessionKeys { + pub dummy: UintAuthorityId, + } +} + +impl From for MockSessionKeys { + fn from(dummy: UintAuthorityId) -> Self { + Self { dummy } + } +} + impl pallet_session::Config for Test { - type Event = (); + type Event = Event; type ValidatorId = AccountId; type ValidatorIdOf = (); - type ShouldEndSession = (); + type ShouldEndSession = TestShouldEndSession; type NextSessionRotation = (); type SessionManager = (); - type SessionHandler = (); - type Keys = (); + type SessionHandler = pallet_session::TestSessionHandler; + type Keys = MockSessionKeys; type WeightInfo = (); } @@ -165,11 +208,24 @@ parameter_types! { pub OffchainRepeat: BlockNumber = 5; } + +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; + type DataProvider = Staking; +} + +impl pallet_session::historical::Config for Test { + type FullIdentification = pallet_staking::Exposure; + type FullIdentificationOf = pallet_staking::ExposureOf; +} + impl pallet_staking::Config for Test { type MaxNominations = (); type Currency = Balances; type UnixTime = Timestamp; - type CurrencyToVote = (); + type CurrencyToVote = U128CurrencyToVote; type RewardRemainder = (); type Event = Event; type Slash = (); // send the slashed funds to the treasury. @@ -177,19 +233,18 @@ impl pallet_staking::Config for Test { type SessionsPerEra = SessionsPerEra; type BondingDuration = BondingDuration; type SlashDeferDuration = SlashDeferDuration; - /// A super-majority of the council can cancel the slash. - type SlashCancelOrigin = (); + type SlashCancelOrigin = frame_system::EnsureRoot; type SessionInterface = Self; type EraPayout = pallet_staking::ConvertCurve; type NextNewSession = Session; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type OffendingValidatorsThreshold = OffendingValidatorsThreshold; - type ElectionProvider = (); - type GenesisElectionProvider = (); - type VoterList = (); + type ElectionProvider = onchain::UnboundedExecution; + type GenesisElectionProvider = Self::ElectionProvider; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; type MaxUnlockingChunks = ConstU32<32>; type WeightInfo = pallet_staking::weights::SubstrateWeight; - type BenchmarkingConfig = (); + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; } impl pallet_ddc_staking::Config for Test { From 959e66c88d97e7e6a03e9f9cf67d2befe7a6a80a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 17 Apr 2023 11:50:38 +0200 Subject: [PATCH 097/192] Fix MaxNominations config --- frame/ddc-validator/src/mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index 76cde6fffc1ee..d897eca04d668 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -222,7 +222,7 @@ impl pallet_session::historical::Config for Test { } impl pallet_staking::Config for Test { - type MaxNominations = (); + type MaxNominations = ConstU32<16>; type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = U128CurrencyToVote; From 2362a51b09dc998bba18f7658e92adbee5d49708 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 17 Apr 2023 11:55:47 +0200 Subject: [PATCH 098/192] Reorder imps block according to construct_runtime! --- frame/ddc-validator/src/mock.rs | 53 +++++++++------------------------ 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index d897eca04d668..2180867f86f8f 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -9,7 +9,6 @@ use frame_system::offchain::SendTransactionTypes; use sp_core::H256; use sp_runtime::{ curve::PiecewiseLinear, generic, testing::{Header, TestXt}, traits::{BlakeTwo256, IdentityLookup, Verify, Extrinsic as ExtrinsicT, IdentifyAccount}, Perbill, MultiSignature, curve}; use pallet_contracts as contracts; - use sp_runtime::traits::Convert; use pallet_session::ShouldEndSession; use sp_runtime::{ @@ -20,24 +19,17 @@ use sp_staking::SessionIndex; use frame_support::{ traits::{U128CurrencyToVote} }; - use frame_election_provider_support::{ onchain, SequentialPhragmen }; - - type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; type Balance = u128; -// type AccountId = u64; - pub type Signature = MultiSignature; pub type AccountId = <::Signer as IdentifyAccount>::AccountId; -// pub type Balance = u128; pub type BlockNumber = u32; pub type Moment = u64; -// Configure a mock runtime to test the pallet. frame_support::construct_runtime!( pub enum Test where Block = Block, @@ -47,11 +39,11 @@ frame_support::construct_runtime!( System: frame_system, Balances: pallet_balances, Contracts: contracts, + Timestamp: pallet_timestamp, Session: pallet_session, Staking: pallet_staking, - Timestamp: pallet_timestamp, - RandomnessCollectiveFlip: pallet_randomness_collective_flip, DdcStaking: pallet_ddc_staking, + RandomnessCollectiveFlip: pallet_randomness_collective_flip, DdcValidator: pallet_ddc_validator, } ); @@ -187,6 +179,11 @@ impl pallet_session::Config for Test { type WeightInfo = (); } +impl pallet_session::historical::Config for Test { + type FullIdentification = pallet_staking::Exposure; + type FullIdentificationOf = pallet_staking::ExposureOf; +} + pallet_staking_reward_curve::build! { const REWARD_CURVE: PiecewiseLinear<'static> = curve!( min_inflation: 0_000_100, @@ -198,6 +195,13 @@ pallet_staking_reward_curve::build! { ); } +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; + type DataProvider = Staking; +} + parameter_types! { pub const SessionsPerEra: sp_staking::SessionIndex = 6; pub const BondingDuration: sp_staking::EraIndex = 3; @@ -208,19 +212,6 @@ parameter_types! { pub OffchainRepeat: BlockNumber = 5; } - -pub struct OnChainSeqPhragmen; -impl onchain::ExecutionConfig for OnChainSeqPhragmen { - type System = Test; - type Solver = SequentialPhragmen; - type DataProvider = Staking; -} - -impl pallet_session::historical::Config for Test { - type FullIdentification = pallet_staking::Exposure; - type FullIdentificationOf = pallet_staking::ExposureOf; -} - impl pallet_staking::Config for Test { type MaxNominations = ConstU32<16>; type Currency = Balances; @@ -300,22 +291,6 @@ impl SigningTypes for Test { type Signature = Signature; } -// impl frame_system::offchain::SendTransactionTypes for Test -// where -// Call: From, -// { -// type OverarchingCall = Call; -// type Extrinsic = TestXt; -// } - -// impl SendTransactionTypes for Test -// where -// Call: From, -// { -// type OverarchingCall = Call; -// type Extrinsic = Extrinsic; -// } - impl CreateSignedTransaction for Test where Call: From, From 78fd78dcbd8261a9110f87cbf527cdaffe9a1caf Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 17 Apr 2023 12:41:49 +0200 Subject: [PATCH 099/192] Fix compilation errors after merge --- frame/ddc-validator/src/mock.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index 2180867f86f8f..5e4ada7f3c149 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -144,10 +144,6 @@ impl pallet_timestamp::Config for Test { impl pallet_randomness_collective_flip::Config for Test {} -parameter_types! { - pub const DdcValidatorsQuorumSize: u32 = 3; -} - pub struct TestShouldEndSession; impl ShouldEndSession for TestShouldEndSession { fn should_end_session(now: u32) -> bool { @@ -196,10 +192,12 @@ pallet_staking_reward_curve::build! { } pub struct OnChainSeqPhragmen; -impl onchain::ExecutionConfig for OnChainSeqPhragmen { +impl onchain::Config for OnChainSeqPhragmen { type System = Test; type Solver = SequentialPhragmen; type DataProvider = Staking; + type WeightInfo = frame_election_provider_support::weights::SubstrateWeight; + } parameter_types! { @@ -236,6 +234,8 @@ impl pallet_staking::Config for Test { type MaxUnlockingChunks = ConstU32<32>; type WeightInfo = pallet_staking::weights::SubstrateWeight; type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; + type CurrencyBalance = Balance; + type OnStakerSlash = (); } impl pallet_ddc_staking::Config for Test { @@ -245,6 +245,11 @@ impl pallet_ddc_staking::Config for Test { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +parameter_types! { + pub const DdcValidatorsQuorumSize: u32 = 3; + pub const ValidationThreshold: u32 = 5; +} + impl pallet_ddc_validator::Config for Test { type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; type Event = Event; @@ -252,6 +257,7 @@ impl pallet_ddc_validator::Config for Test { type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; + type ValidationThreshold = ValidationThreshold; } impl SendTransactionTypes for Test From 0cb52b501530813de896df8f4a0fb1633a61163f Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 17 Apr 2023 14:58:29 +0200 Subject: [PATCH 100/192] Add unit test example --- frame/ddc-validator/src/tests.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/frame/ddc-validator/src/tests.rs b/frame/ddc-validator/src/tests.rs index e69de29bb2d1d..cc0df5512f593 100644 --- a/frame/ddc-validator/src/tests.rs +++ b/frame/ddc-validator/src/tests.rs @@ -0,0 +1,15 @@ +use crate::{mock::*}; +use frame_support::{assert_ok}; +use sp_core::crypto::AccountId32; + +#[test] +fn save_validated_data_works() { + new_test_ext().execute_with(|| { + assert_ok!(DdcValidator::save_validated_data( + Origin::signed(AccountId32::from([1; 32])), + true, + String::from("0xab1"), + 1, + )); + }); +} \ No newline at end of file From 71d0d266d01ae5bf4384d4e72efe20ae1d16bea6 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 18 Apr 2023 10:30:22 +0200 Subject: [PATCH 101/192] Fix build --- frame/ddc-validator/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index eeaa054367980..2c9fe6bc22838 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -9,7 +9,7 @@ alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } +frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } log = { version = "0.4.17", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } From f38a1d2b81e1087ba0ab5846ac3175d87866ae16 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 19 Apr 2023 11:46:03 +0200 Subject: [PATCH 102/192] Make data provider url configurable via rpc --- frame/ddc-validator/src/lib.rs | 53 +++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index fef99bd08fda4..2d565566cf824 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -53,6 +53,7 @@ pub use frame_support::{ traits::{Currency, Randomness, UnixTime}, weights::Weight, BoundedVec, RuntimeDebug, + storage, }; pub use frame_system::{ ensure_signed, @@ -66,7 +67,7 @@ pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_io::crypto::sr25519_public_keys; -pub use sp_runtime::offchain::{http, Duration, Timestamp}; +pub use sp_runtime::offchain::{http, Duration, Timestamp, storage::StorageValueRef}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; @@ -92,7 +93,9 @@ const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. -const DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379/"; +// const DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379/"; +const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379/"; +const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; /// DAC Validation methods. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] @@ -425,6 +428,10 @@ pub mod pallet { /// 2. Run a process at the same time, /// 3. Read data from DAC. fn offchain_worker(block_number: T::BlockNumber) { + let data_provider_url = Self::get_data_provider_url(); + + info!("data_provider_url: {:?}", data_provider_url.unwrap_or(String::from("no url"))); + // Skip if not a validator. if !sp_io::offchain::is_validator() { return @@ -628,6 +635,26 @@ pub mod pallet { Ok(()) } + fn get_data_provider_url() -> Option { + let url_ref = StorageValueRef::persistent(DATA_PROVIDER_URL_KEY).get::>(); + info!("url_ref: {:?}", url_ref); + match url_ref { + Ok(None) => { + let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); + let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}", url_key); + warn!("{}", msg); + None + } + Ok(Some(url)) => { + Some(String::from_utf8(url).unwrap()) + }, + Err(_) => { + error!("[OCW] Data provider URL is configured but the value could not be decoded"); + None + } + } + } + fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { @@ -714,11 +741,29 @@ pub mod pallet { } fn get_bytes_sent_query_url(era: EraIndex) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DATA_PROVIDER_URL, era, era) + let data_provider_url = Self::get_data_provider_url(); + + match data_provider_url { + Some(url) => { + return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); + } + None => { + return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); + } + } } fn get_bytes_received_query_url(era: EraIndex) -> String { - format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DATA_PROVIDER_URL, era, era) + let data_provider_url = Self::get_data_provider_url(); + + match data_provider_url { + Some(url) => { + return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); + } + None => { + return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); + } + } } fn http_get_json(url: &str) -> ResultStr { From 2d2ff2e1a2b0ddc98c1b108009766a9e34cdd7ec Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 19 Apr 2023 20:24:41 +0600 Subject: [PATCH 103/192] Fix local storage reading --- frame/ddc-validator/src/lib.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 2d565566cf824..4d4d88f326f49 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -636,22 +636,19 @@ pub mod pallet { } fn get_data_provider_url() -> Option { - let url_ref = StorageValueRef::persistent(DATA_PROVIDER_URL_KEY).get::>(); + let url_ref = sp_io::offchain::local_storage_get( + sp_core::offchain::StorageKind::PERSISTENT, + DATA_PROVIDER_URL_KEY, + ); info!("url_ref: {:?}", url_ref); match url_ref { - Ok(None) => { + None => { let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}", url_key); warn!("{}", msg); None - } - Ok(Some(url)) => { - Some(String::from_utf8(url).unwrap()) }, - Err(_) => { - error!("[OCW] Data provider URL is configured but the value could not be decoded"); - None - } + Some(url) => Some(String::from_utf8(url).unwrap()), } } From 16c8d244b08397eb835e4c815ba4d86707fc4755 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 19 Apr 2023 16:53:00 +0200 Subject: [PATCH 104/192] Clean up --- frame/ddc-validator/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 4d4d88f326f49..ec2769b01c15b 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -93,7 +93,6 @@ const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. -// const DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379/"; const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379/"; const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; @@ -640,7 +639,7 @@ pub mod pallet { sp_core::offchain::StorageKind::PERSISTENT, DATA_PROVIDER_URL_KEY, ); - info!("url_ref: {:?}", url_ref); + match url_ref { None => { let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); From 0cb079505ac0c85e1ad50147370f37ef9c4e1ba3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 20 Apr 2023 13:00:01 +0600 Subject: [PATCH 105/192] Remove duplicate parameter --- frame/ddc-validator/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index ec2769b01c15b..f1c58a4ba43f2 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -73,10 +73,6 @@ pub use sp_std::prelude::*; extern crate alloc; -parameter_types! { - pub const ValidationThreshold: f32 = 5.0; -} - /// The balance type of this pallet. type BalanceOf = <::Currency as Currency< ::AccountId, From 2a488bc0b568a39c917bc2b983d9ba5063c64e3f Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 20 Apr 2023 13:10:24 +0600 Subject: [PATCH 106/192] Autoformat DAC Validator files --- frame/ddc-validator/src/lib.rs | 13 +- frame/ddc-validator/src/mock.rs | 340 +++++++++++++++---------------- frame/ddc-validator/src/tests.rs | 22 +- 3 files changed, 186 insertions(+), 189 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index f1c58a4ba43f2..f64bc4ae0ae84 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -49,11 +49,10 @@ pub use frame_support::{ dispatch::DispatchResult, log::{error, info, warn}, pallet_prelude::*, - parameter_types, + parameter_types, storage, traits::{Currency, Randomness, UnixTime}, weights::Weight, BoundedVec, RuntimeDebug, - storage, }; pub use frame_system::{ ensure_signed, @@ -67,7 +66,7 @@ pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; pub use sp_io::crypto::sr25519_public_keys; -pub use sp_runtime::offchain::{http, Duration, Timestamp, storage::StorageValueRef}; +pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timestamp}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; @@ -738,10 +737,10 @@ pub mod pallet { match data_provider_url { Some(url) => { return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); - } + }, None => { return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); - } + }, } } @@ -751,10 +750,10 @@ pub mod pallet { match data_provider_url { Some(url) => { return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); - } + }, None => { return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); - } + }, } } diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index 5e4ada7f3c149..bc057705cbab9 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -1,27 +1,25 @@ -use crate::{*, self as pallet_ddc_validator}; +use crate::{self as pallet_ddc_validator, *}; +use frame_election_provider_support::{onchain, SequentialPhragmen}; use frame_support::{ - parameter_types, - weights::Weight, - traits::{ConstU16, ConstU64, Currency, Everything, Nothing} + parameter_types, + traits::{ConstU16, ConstU64, Currency, Everything, Nothing, U128CurrencyToVote}, + weights::Weight, }; -use frame_system::EnsureRoot; -use frame_system::offchain::SendTransactionTypes; -use sp_core::H256; -use sp_runtime::{ curve::PiecewiseLinear, generic, testing::{Header, TestXt}, traits::{BlakeTwo256, IdentityLookup, Verify, Extrinsic as ExtrinsicT, IdentifyAccount}, Perbill, MultiSignature, curve}; +use frame_system::{offchain::SendTransactionTypes, EnsureRoot}; use pallet_contracts as contracts; -use sp_runtime::traits::Convert; use pallet_session::ShouldEndSession; +use sp_core::H256; use sp_runtime::{ - impl_opaque_keys, - testing::{UintAuthorityId}, + curve, + curve::PiecewiseLinear, + generic, impl_opaque_keys, + testing::{Header, TestXt, UintAuthorityId}, + traits::{ + BlakeTwo256, Convert, Extrinsic as ExtrinsicT, IdentifyAccount, IdentityLookup, Verify, + }, + MultiSignature, Perbill, }; use sp_staking::SessionIndex; -use frame_support::{ - traits::{U128CurrencyToVote} -}; -use frame_election_provider_support::{ - onchain, SequentialPhragmen -}; type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; type Balance = u128; @@ -38,117 +36,118 @@ frame_support::construct_runtime!( { System: frame_system, Balances: pallet_balances, - Contracts: contracts, - Timestamp: pallet_timestamp, - Session: pallet_session, - Staking: pallet_staking, - DdcStaking: pallet_ddc_staking, - RandomnessCollectiveFlip: pallet_randomness_collective_flip, - DdcValidator: pallet_ddc_validator, + Contracts: contracts, + Timestamp: pallet_timestamp, + Session: pallet_session, + Staking: pallet_staking, + DdcStaking: pallet_ddc_staking, + RandomnessCollectiveFlip: pallet_randomness_collective_flip, + DdcValidator: pallet_ddc_validator, } ); parameter_types! { - pub const BlockHashCount: BlockNumber = 250; - pub const MaximumBlockWeight: Weight = 1024; - pub const MaximumBlockLength: u32 = 2 * 1024; - pub const AvailableBlockRatio: Perbill = Perbill::one(); + pub const BlockHashCount: BlockNumber = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Config for Test { - type BaseCallFilter = Everything; - type BlockWeights = (); - type BlockLength = (); - type Origin = Origin; - type Index = u64; - type BlockNumber = BlockNumber; - type Hash = H256; - type Call = Call; - type Hashing = BlakeTwo256; - type AccountId = AccountId; - // u64; // sp_core::sr25519::Public; - type Lookup = IdentityLookup; - type Header = generic::Header; - type Event = Event; - type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; + type BaseCallFilter = Everything; + type BlockWeights = (); + type BlockLength = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = BlockNumber; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = AccountId; + // u64; // sp_core::sr25519::Public; + type Lookup = IdentityLookup; + type Header = generic::Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } parameter_types! { - pub const SignedClaimHandicap: BlockNumber = 2; - pub const TombstoneDeposit: Balance = 16; - pub const StorageSizeOffset: u32 = 8; - pub const RentByteFee: Balance = 4; - pub const RentDepositOffset: Balance = 10_000; - pub const SurchargeReward: Balance = 150; - pub const MaxDepth: u32 = 100; - pub const MaxValueSize: u32 = 16_384; - pub Schedule: pallet_contracts::Schedule = Default::default(); + pub const SignedClaimHandicap: BlockNumber = 2; + pub const TombstoneDeposit: Balance = 16; + pub const StorageSizeOffset: u32 = 8; + pub const RentByteFee: Balance = 4; + pub const RentDepositOffset: Balance = 10_000; + pub const SurchargeReward: Balance = 150; + pub const MaxDepth: u32 = 100; + pub const MaxValueSize: u32 = 16_384; + pub Schedule: pallet_contracts::Schedule = Default::default(); } -use contracts::{Config as contractsConfig}; +use contracts::Config as contractsConfig; -type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; pub struct TestWeightToFee; impl Convert for TestWeightToFee { - fn convert(weight: u64) -> u128 { - weight as u128 - } + fn convert(weight: u64) -> u128 { + weight as u128 + } } impl contracts::Config for Test { - type Time = Timestamp; - type Randomness = RandomnessCollectiveFlip; - type Currency = Balances; - type Event = Event; - type CallStack = [pallet_contracts::Frame; 31]; - type WeightPrice = TestWeightToFee; //pallet_transaction_payment::Module; - type WeightInfo = (); - type ChainExtension = (); - type DeletionQueueDepth = (); - type DeletionWeightLimit = (); - type Schedule = Schedule; - type Call = Call; - type CallFilter = Nothing; - type DepositPerByte = DepositPerByte; - type DepositPerItem = DepositPerItem; - type AddressGenerator = pallet_contracts::DefaultAddressGenerator; + type Time = Timestamp; + type Randomness = RandomnessCollectiveFlip; + type Currency = Balances; + type Event = Event; + type CallStack = [pallet_contracts::Frame; 31]; + type WeightPrice = TestWeightToFee; //pallet_transaction_payment::Module; + type WeightInfo = (); + type ChainExtension = (); + type DeletionQueueDepth = (); + type DeletionWeightLimit = (); + type Schedule = Schedule; + type Call = Call; + type CallFilter = Nothing; + type DepositPerByte = DepositPerByte; + type DepositPerItem = DepositPerItem; + type AddressGenerator = pallet_contracts::DefaultAddressGenerator; } parameter_types! { - pub const TransactionByteFee: u64 = 0; - pub const DepositPerItem: Balance = 0; + pub const TransactionByteFee: u64 = 0; + pub const DepositPerItem: Balance = 0; pub const DepositPerByte: Balance = 0; } parameter_types! { - pub const MinimumPeriod: u64 = 1; + pub const MinimumPeriod: u64 = 1; } impl pallet_timestamp::Config for Test { - type Moment = Moment; - type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; - type WeightInfo = (); + type Moment = Moment; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; + type WeightInfo = (); } impl pallet_randomness_collective_flip::Config for Test {} pub struct TestShouldEndSession; impl ShouldEndSession for TestShouldEndSession { - fn should_end_session(now: u32) -> bool { - now % 10 == 0 // every 10 blocks - } + fn should_end_session(now: u32) -> bool { + now % 10 == 0 // every 10 blocks + } } impl_opaque_keys! { @@ -164,15 +163,15 @@ impl From for MockSessionKeys { } impl pallet_session::Config for Test { - type Event = Event; - type ValidatorId = AccountId; - type ValidatorIdOf = (); - type ShouldEndSession = TestShouldEndSession; - type NextSessionRotation = (); - type SessionManager = (); - type SessionHandler = pallet_session::TestSessionHandler; - type Keys = MockSessionKeys; - type WeightInfo = (); + type Event = Event; + type ValidatorId = AccountId; + type ValidatorIdOf = (); + type ShouldEndSession = TestShouldEndSession; + type NextSessionRotation = (); + type SessionManager = (); + type SessionHandler = pallet_session::TestSessionHandler; + type Keys = MockSessionKeys; + type WeightInfo = (); } impl pallet_session::historical::Config for Test { @@ -193,11 +192,10 @@ pallet_staking_reward_curve::build! { pub struct OnChainSeqPhragmen; impl onchain::Config for OnChainSeqPhragmen { - type System = Test; - type Solver = SequentialPhragmen; - type DataProvider = Staking; - type WeightInfo = frame_election_provider_support::weights::SubstrateWeight; - + type System = Test; + type Solver = SequentialPhragmen; + type DataProvider = Staking; + type WeightInfo = frame_election_provider_support::weights::SubstrateWeight; } parameter_types! { @@ -211,61 +209,61 @@ parameter_types! { } impl pallet_staking::Config for Test { - type MaxNominations = ConstU32<16>; - type Currency = Balances; - type UnixTime = Timestamp; - type CurrencyToVote = U128CurrencyToVote; - type RewardRemainder = (); - type Event = Event; - type Slash = (); // send the slashed funds to the treasury. - type Reward = (); // rewards are minted from the void - type SessionsPerEra = SessionsPerEra; - type BondingDuration = BondingDuration; - type SlashDeferDuration = SlashDeferDuration; - type SlashCancelOrigin = frame_system::EnsureRoot; - type SessionInterface = Self; - type EraPayout = pallet_staking::ConvertCurve; - type NextNewSession = Session; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type OffendingValidatorsThreshold = OffendingValidatorsThreshold; - type ElectionProvider = onchain::UnboundedExecution; - type GenesisElectionProvider = Self::ElectionProvider; - type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; - type MaxUnlockingChunks = ConstU32<32>; - type WeightInfo = pallet_staking::weights::SubstrateWeight; - type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; - type CurrencyBalance = Balance; - type OnStakerSlash = (); + type MaxNominations = ConstU32<16>; + type Currency = Balances; + type UnixTime = Timestamp; + type CurrencyToVote = U128CurrencyToVote; + type RewardRemainder = (); + type Event = Event; + type Slash = (); // send the slashed funds to the treasury. + type Reward = (); // rewards are minted from the void + type SessionsPerEra = SessionsPerEra; + type BondingDuration = BondingDuration; + type SlashDeferDuration = SlashDeferDuration; + type SlashCancelOrigin = frame_system::EnsureRoot; + type SessionInterface = Self; + type EraPayout = pallet_staking::ConvertCurve; + type NextNewSession = Session; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; + type ElectionProvider = onchain::UnboundedExecution; + type GenesisElectionProvider = Self::ElectionProvider; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; + type MaxUnlockingChunks = ConstU32<32>; + type WeightInfo = pallet_staking::weights::SubstrateWeight; + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; + type CurrencyBalance = Balance; + type OnStakerSlash = (); } impl pallet_ddc_staking::Config for Test { - type BondingDuration = BondingDuration; - type Currency = Balances; - type Event = Event; - type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; + type BondingDuration = BondingDuration; + type Currency = Balances; + type Event = Event; + type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } parameter_types! { pub const DdcValidatorsQuorumSize: u32 = 3; - pub const ValidationThreshold: u32 = 5; + pub const ValidationThreshold: u32 = 5; } impl pallet_ddc_validator::Config for Test { - type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; - type Event = Event; - type Randomness = RandomnessCollectiveFlip; - type Call = Call; - type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; - type TimeProvider = pallet_timestamp::Pallet; - type ValidationThreshold = ValidationThreshold; + type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; + type Event = Event; + type Randomness = RandomnessCollectiveFlip; + type Call = Call; + type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; + type TimeProvider = pallet_timestamp::Pallet; + type ValidationThreshold = ValidationThreshold; } impl SendTransactionTypes for Test - where - Call: From, +where + Call: From, { - type OverarchingCall = Call; - type Extrinsic = Extrinsic; + type OverarchingCall = Call; + type Extrinsic = Extrinsic; } parameter_types! { @@ -274,39 +272,39 @@ parameter_types! { } impl pallet_balances::Config for Test { - type Balance = Balance; - type DustRemoval = (); - type Event = Event; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type WeightInfo = (); - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = (); + type Balance = Balance; + type DustRemoval = (); + type Event = Event; + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = (); } // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { - frame_system::GenesisConfig::default().build_storage::().unwrap().into() + frame_system::GenesisConfig::default().build_storage::().unwrap().into() } pub type Extrinsic = TestXt; impl SigningTypes for Test { - type Public = ::Signer; - type Signature = Signature; + type Public = ::Signer; + type Signature = Signature; } impl CreateSignedTransaction for Test - where - Call: From, +where + Call: From, { - fn create_transaction>( - call: Call, - _public: ::Signer, - _account: AccountId, - nonce: u64, - ) -> Option<(Call, ::SignaturePayload)> { - Some((call, (nonce, ()))) - } -} \ No newline at end of file + fn create_transaction>( + call: Call, + _public: ::Signer, + _account: AccountId, + nonce: u64, + ) -> Option<(Call, ::SignaturePayload)> { + Some((call, (nonce, ()))) + } +} diff --git a/frame/ddc-validator/src/tests.rs b/frame/ddc-validator/src/tests.rs index cc0df5512f593..2baeb76c0b127 100644 --- a/frame/ddc-validator/src/tests.rs +++ b/frame/ddc-validator/src/tests.rs @@ -1,15 +1,15 @@ -use crate::{mock::*}; -use frame_support::{assert_ok}; +use crate::mock::*; +use frame_support::assert_ok; use sp_core::crypto::AccountId32; #[test] fn save_validated_data_works() { - new_test_ext().execute_with(|| { - assert_ok!(DdcValidator::save_validated_data( - Origin::signed(AccountId32::from([1; 32])), - true, - String::from("0xab1"), - 1, - )); - }); -} \ No newline at end of file + new_test_ext().execute_with(|| { + assert_ok!(DdcValidator::save_validated_data( + Origin::signed(AccountId32::from([1; 32])), + true, + String::from("0xab1"), + 1, + )); + }); +} From 647e62fb3d9f3809075a1a8d15f107f2317cd765 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 20 Apr 2023 14:13:45 +0600 Subject: [PATCH 107/192] Separate modules for validation and DAC client --- frame/ddc-validator/src/dac.rs | 1 + frame/ddc-validator/src/lib.rs | 3 +++ frame/ddc-validator/src/validation.rs | 1 + 3 files changed, 5 insertions(+) create mode 100644 frame/ddc-validator/src/dac.rs create mode 100644 frame/ddc-validator/src/validation.rs diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs new file mode 100644 index 0000000000000..b24eb0cc5a467 --- /dev/null +++ b/frame/ddc-validator/src/dac.rs @@ -0,0 +1 @@ +//! A module with Data Activity Capture (DAC) interaction. diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index f64bc4ae0ae84..fec14bdb0184a 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -34,6 +34,9 @@ #![cfg_attr(not(feature = "std"), no_std)] +mod dac; +mod validation; + #[cfg(test)] mod mock; diff --git a/frame/ddc-validator/src/validation.rs b/frame/ddc-validator/src/validation.rs new file mode 100644 index 0000000000000..017d90782db54 --- /dev/null +++ b/frame/ddc-validator/src/validation.rs @@ -0,0 +1 @@ +//! DAC Validation implementation. From 1f52c807aa5dde22e69be882b41aac29cbddd278 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 20 Apr 2023 16:04:01 +0200 Subject: [PATCH 108/192] Implement job assignment --- Cargo.lock | 1 + frame/ddc-validator/Cargo.toml | 2 +- frame/ddc-validator/src/lib.rs | 110 +++++++++++++++++++++++++-------- 3 files changed, 85 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 198afca1784e3..e4c1fd053e2a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5879,6 +5879,7 @@ dependencies = [ "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", + "rand 0.8.4", "scale-info", "serde", "serde_json 1.0.44", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 2c9fe6bc22838..26477e7c6d0b0 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -10,7 +10,6 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } - log = { version = "0.4.17", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } @@ -25,6 +24,7 @@ sp-keystore = { version = "0.12.0", default-features = false, path = "../../prim sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +rand = { version = "0.8", default-features = false } [features] default = ["std"] diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index ec2769b01c15b..3d58f37d1482a 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -40,6 +40,7 @@ mod mock; #[cfg(test)] mod tests; +use std::collections::{HashMap}; pub use alloc::{format, string::String}; pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; pub use codec::{Decode, Encode, HasCompact, MaxEncodedLen}; @@ -70,6 +71,7 @@ pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, Duration, Timestamp, storage::StorageValueRef}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; +use rand::seq::SliceRandom; extern crate alloc; @@ -304,6 +306,8 @@ pub mod pallet { #[pallet::constant] type DdcValidatorsQuorumSize: Get; + type ValidatorsMax: Get; + /// Proof-of-Delivery parameter specifies an allowed deviation between bytes sent and bytes /// received. The deviation is expressed as a percentage. For example, if the value is 10, /// then the difference between bytes sent and bytes received is allowed to be up to 10%. @@ -312,6 +316,17 @@ pub mod pallet { type ValidationThreshold: Get; } + #[pallet::storage] + #[pallet::getter(fn assignments)] + pub(super) type Assignments = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + String, + Vec, + >; + /// A signal to start a process on all the validators. #[pallet::storage] #[pallet::getter(fn signal)] @@ -370,36 +385,23 @@ pub mod pallet { fn on_initialize(block_number: T::BlockNumber) -> Weight { // Reset the signal in the beginning of the block to keep it reset until an incoming // transaction sets it to true. - if Signal::::get().unwrap_or(false) { - Signal::::set(Some(false)); - } + // if Signal::::get().unwrap_or(false) { + // Signal::::set(Some(false)); + // } // Old task manager. - // - // if block_number <= 1u32.into() { - // return 0 - // } - // let era = Self::get_current_era(); - // if let Some(last_managed_era) = >::get() { - // if last_managed_era >= era { - // return 0 - // } - // } - // >::put(era); - - // let validators: Vec = >::iter_keys().collect(); - // let validators_count = validators.len() as u32; - // let edges: Vec = - // >::iter_keys().collect(); log::info!( - // "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, - // validators: {:?}, edges: {:?}", block_number, - // era, - // >::get(), - // validators_count, - // validators, - // edges, - // ); + if block_number <= 1u32.into() { + return 0 + } + + let era = Self::get_current_era(); + if let Some(last_managed_era) = >::get() { + if last_managed_era >= era { + return 0 + } + } + >::put(era); // // A naive approach assigns random validators for each edge. // for edge in edges { @@ -811,6 +813,60 @@ pub mod pallet { } } + fn shuffle(mut list: Vec) -> Vec { + list.shuffle(&mut rand::thread_rng()); + + list + } + + fn split(list: Vec, segment_len: usize) -> Vec> { + list.chunks(segment_len).map(|chunk| chunk.to_vec()).collect() + } + + fn assign(quorum_size: usize) -> HashMap> { + let validators: Vec = >::iter_keys().collect(); + let edges: Vec = >::iter_keys().collect(); + + let shuffled_validators = Self::shuffle(validators); + let shuffled_edges = Self::shuffle(edges); + + let validators_keys: Vec = shuffled_validators.iter().map( |v| { + Self::account_to_string(v.clone()) + }).collect(); + + let edges_keys: Vec = shuffled_edges.iter().map( |e| { + Self::account_to_string(e.clone()) + }).collect(); + + let quorums = Self::split(validators_keys, quorum_size); + let edges_groups = Self::split(edges_keys, quorums.len()); + + Self::map_validators_to_edges(quorums, edges_groups) + } + + fn map_validators_to_edges(quorums: Vec>, edges_groups: Vec>) -> HashMap> { + let mut validators_to_edges: HashMap> = HashMap::new(); + + for (i, quorum) in quorums.iter().enumerate() { + let edges_group = &edges_groups[i]; + for validator in quorum { + validators_to_edges.insert(validator.clone(), edges_group.clone()); + } + } + + validators_to_edges + } + + fn save_assignments(era: EraIndex, assignments: HashMap>) -> DispatchResult { + let era = Self::get_current_era(); + + for (validator, edges) in assignments { + Assignments::::insert(era, validator, edges); + } + + Ok(()) + } + /// Fetch the tasks related to current validator fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { let mut cdn_nodes: Vec = vec![]; From 604441ff766f79201afe9661f4252d3ae1f23e61 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 20 Apr 2023 18:57:39 +0200 Subject: [PATCH 109/192] Compilation fixes --- Cargo.lock | 1 - bin/node/runtime/src/lib.rs | 4 +++- frame/ddc-validator/Cargo.toml | 1 - frame/ddc-validator/src/lib.rs | 39 +++++++++++++++------------------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4c1fd053e2a8..198afca1784e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5879,7 +5879,6 @@ dependencies = [ "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", - "rand 0.8.4", "scale-info", "serde", "serde_json 1.0.44", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index e03d634037f9d..d423ca254762a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -130,7 +130,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 30401, + spec_version: 30403, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, @@ -1236,6 +1236,7 @@ impl pallet_ddc_staking::Config for Runtime { parameter_types! { pub const DdcValidatorsQuorumSize: u32 = 3; pub const ValidationThreshold: u32 = 5; + pub const ValidatorsMax: u32 = 64; } impl pallet_ddc_validator::Config for Runtime { @@ -1246,6 +1247,7 @@ impl pallet_ddc_validator::Config for Runtime { type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; type ValidationThreshold = ValidationThreshold; + type ValidatorsMax = ValidatorsMax; } construct_runtime!( diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 26477e7c6d0b0..ff6dbd53b3fd6 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -24,7 +24,6 @@ sp-keystore = { version = "0.12.0", default-features = false, path = "../../prim sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -rand = { version = "0.8", default-features = false } [features] default = ["std"] diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 3d58f37d1482a..050aa72678292 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -40,7 +40,6 @@ mod mock; #[cfg(test)] mod tests; -use std::collections::{HashMap}; pub use alloc::{format, string::String}; pub use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; pub use codec::{Decode, Encode, HasCompact, MaxEncodedLen}; @@ -71,7 +70,6 @@ pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, Duration, Timestamp, storage::StorageValueRef}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; -use rand::seq::SliceRandom; extern crate alloc; @@ -396,13 +394,18 @@ pub mod pallet { } let era = Self::get_current_era(); + info!("current era: {:?}", era); + if let Some(last_managed_era) = >::get() { + info!("last_managed_era: {:?}", last_managed_era); if last_managed_era >= era { return 0 } } >::put(era); + Self::assign(3usize); + // // A naive approach assigns random validators for each edge. // for edge in edges { // let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = @@ -814,7 +817,11 @@ pub mod pallet { } fn shuffle(mut list: Vec) -> Vec { - list.shuffle(&mut rand::thread_rng()); + let len = list.len(); + for i in 1..len { + let random_index = Self::choose(len as u32).unwrap() as usize; + list.swap(i, random_index) + } list } @@ -823,10 +830,14 @@ pub mod pallet { list.chunks(segment_len).map(|chunk| chunk.to_vec()).collect() } - fn assign(quorum_size: usize) -> HashMap> { + fn assign(quorum_size: usize) { let validators: Vec = >::iter_keys().collect(); let edges: Vec = >::iter_keys().collect(); + if edges.len() == 0 { + return; + } + let shuffled_validators = Self::shuffle(validators); let shuffled_edges = Self::shuffle(edges); @@ -841,30 +852,14 @@ pub mod pallet { let quorums = Self::split(validators_keys, quorum_size); let edges_groups = Self::split(edges_keys, quorums.len()); - Self::map_validators_to_edges(quorums, edges_groups) - } - - fn map_validators_to_edges(quorums: Vec>, edges_groups: Vec>) -> HashMap> { - let mut validators_to_edges: HashMap> = HashMap::new(); + let era = Self::get_current_era(); for (i, quorum) in quorums.iter().enumerate() { let edges_group = &edges_groups[i]; for validator in quorum { - validators_to_edges.insert(validator.clone(), edges_group.clone()); + Assignments::::insert(era, validator, edges_group); } } - - validators_to_edges - } - - fn save_assignments(era: EraIndex, assignments: HashMap>) -> DispatchResult { - let era = Self::get_current_era(); - - for (validator, edges) in assignments { - Assignments::::insert(era, validator, edges); - } - - Ok(()) } /// Fetch the tasks related to current validator From a00acc6993b3c99e7684192118ba7eb349a0f77d Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 21 Apr 2023 13:30:15 +0600 Subject: [PATCH 110/192] Validation results with aggregates from DAC --- frame/ddc-validator/src/lib.rs | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index fec14bdb0184a..b8b296e7e4430 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -94,6 +94,30 @@ const ERA_IN_BLOCKS: u8 = 20; const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379/"; const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; +/// Aggregated values from DAC that describe CDN node's activity during a certain era. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct DacTotalAggregates { + /// Total bytes received by the client. + pub received: u64, + /// Total bytes sent by the CDN node. + pub sent: u64, + /// Total bytes sent by the CDN node to the client which interrupts the connection. + pub failed_by_client: u64, + /// ToDo: explain. + pub failure_rate: u64, +} + +/// Final DAC Validation decision. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct ValidationDecision { + /// Validation result. + pub result: bool, + /// A hash of the data used to produce validation result. + pub payload: [u8; 256], + /// Values aggregated from the payload. + pub totals: DacTotalAggregates, +} + /// DAC Validation methods. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum ValidationMethodKind { @@ -315,6 +339,12 @@ pub mod pallet { #[pallet::getter(fn signal)] pub(super) type Signal = StorageValue<_, bool>; + /// The map from the era and CDN participant stash key to the validation decision related. + #[pallet::storage] + #[pallet::getter(fn validation_decisions)] + pub type ValidationDecisions = + StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; + /// The map from the era and CDN participant stash key to the validation decisions related. #[pallet::storage] #[pallet::getter(fn tasks)] @@ -591,6 +621,28 @@ pub mod pallet { Ok(()) } + + /// Set validation decision for a given CDN node in an era. + #[pallet::weight(10_000)] + pub fn set_validation_decision( + origin: OriginFor, + era: EraIndex, + cdn_node: T::AccountId, + validation_decision: ValidationDecision, + ) -> DispatchResult { + ensure_signed(origin)?; + + // ToDo: check if origin is a validator. + // ToDo: check if the era is current - 1. + // ToDo: check if the validation decision is not set yet. + // ToDo: check cdn_node is known to ddc-staking. + + ValidationDecisions::::insert(era, cdn_node, validation_decision); + + // ToDo: emit event. + + Ok(()) + } } impl Pallet From fe6b5f41d836f4016e7c80c444ba6366b874f3dc Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 21 Apr 2023 14:04:57 +0600 Subject: [PATCH 111/192] Remove unnecessary validators signal reset call --- frame/ddc-validator/src/lib.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b8b296e7e4430..3d7f4d229e55f 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -521,15 +521,6 @@ pub mod pallet { Ok(()) } - #[pallet::weight(10_000)] - pub fn reset_signal(origin: OriginFor) -> DispatchResult { - ensure_signed(origin)?; - - Signal::::set(Some(false)); - - Ok(()) - } - #[pallet::weight(10000)] pub fn save_validated_data( origin: OriginFor, From f92657c816f37090a4d805c6a4ed464aa5dda4a0 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 21 Apr 2023 15:34:16 +0600 Subject: [PATCH 112/192] Remove an obsolete task manager related code --- bin/node/runtime/src/lib.rs | 3 - frame/ddc-validator/src/lib.rs | 317 +-------------------------------- 2 files changed, 6 insertions(+), 314 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index e03d634037f9d..71ab00de48b93 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1234,14 +1234,11 @@ impl pallet_ddc_staking::Config for Runtime { } parameter_types! { - pub const DdcValidatorsQuorumSize: u32 = 3; pub const ValidationThreshold: u32 = 5; } impl pallet_ddc_validator::Config for Runtime { - type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; type Event = Event; - type Randomness = RandomnessCollectiveFlip; type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 3d7f4d229e55f..350fd204bed95 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,36 +1,17 @@ //! # DDC Validator pallet //! -//! The DDC Validator pallet is responsible for producing validation decisions based on activity -//! data from DAC DataModel. It is expected to work on validators nodes only. +//! The DDC Validator pallet defines storage item to store validation results and implements OCW +//! (off-chain worker) to produce these results using the data from Data Activity Capture (DAC). //! //! - [`Config`] //! - [`Call`] //! - [`Pallet`] //! - [`Hooks`] //! -//! ## Responsibility -//! -//! 1. Assign validation tasks on DAC Validators in the beginning of each era, -//! 2. Spin the offchain worker which tries to execute the validation tasks each era, -//! 3. Fetch the data required for validation from DAC DataModel, -//! 4. Execute validation method on this data, -//! 5. Produce validation decision and submit it to the chain. -//! -//! ## Usage -//! -//! 1. Run the node with `--validator` flag, -//! 2. Setup validator key with `author_insertKey` RPC call. Use `dacv` validator key type and the -//! same private key as the one used to generate the validator's session keys, -//! 3. Proceed a regular validator setup, -//! 4. Tasks assignment will assign you a task in the beginning of the era which has your account in -//! validators set. -//! //! ## Notes //! -//! - Era definition in this pallet is different than in the `pallet-staking`. In this pallet era is -//! a period of time during which the validator is expected to produce a validation decision. -//! Means staking era and DAC era are different and are not related to each other, -//! - You can set DAC Validators quorum size by specifying `DdcValidatorsQuorumSize` parameter, +//! - Era definition in this pallet is different than in the `pallet-staking`. Check DAC +//! documentation for `era` definition used in this pallet. #![cfg_attr(not(feature = "std"), no_std)] @@ -118,25 +99,6 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } -/// DAC Validation methods. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] -pub enum ValidationMethodKind { - /// Compare amount of served content with amount of content consumed. - ProofOfDelivery, -} - -/// Associates validation decision with the validator and the method used to produce it. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] -pub struct Decision { - /// Individual validator's decision. Can be `None` if the validator did not produce a decision - /// (yet). - pub decision: Option, - /// The method used to produce the decision. - pub method: ValidationMethodKind, - /// The validator who produced the decision. - pub validator: AccountId, -} - #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { era: EraIndex, @@ -310,22 +272,12 @@ pub mod pallet { /// The overarching event type. type Event: From> + IsType<::Event>; - /// Something that provides randomness in the runtime. Required by the tasks assignment - /// procedure. - type Randomness: Randomness; - /// A dispatchable call. type Call: From>; type AuthorityId: AppCrypto; type TimeProvider: UnixTime; - /// Number of validators expected to produce an individual validation decision to form a - /// consensus. Tasks assignment procedure use this value to determine the number of - /// validators are getting the same task. Must be an odd number. - #[pallet::constant] - type DdcValidatorsQuorumSize: Get; - /// Proof-of-Delivery parameter specifies an allowed deviation between bytes sent and bytes /// received. The deviation is expressed as a percentage. For example, if the value is 10, /// then the difference between bytes sent and bytes received is allowed to be up to 10%. @@ -345,23 +297,6 @@ pub mod pallet { pub type ValidationDecisions = StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; - /// The map from the era and CDN participant stash key to the validation decisions related. - #[pallet::storage] - #[pallet::getter(fn tasks)] - pub type Tasks = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - BoundedVec, T::DdcValidatorsQuorumSize>, - >; - - /// The last era for which the tasks assignment produced. - #[pallet::storage] - #[pallet::getter(fn last_managed_era)] - pub type LastManagedEra = StorageValue<_, EraIndex>; - #[pallet::storage] #[pallet::getter(fn validation_results)] pub(super) type ValidationResults = @@ -372,22 +307,10 @@ pub mod pallet { pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - /// DAC Validator successfully published the validation decision. - ValidationDecisionSubmitted, - } + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} #[pallet::error] - pub enum Error { - /// Validation decision attempts to submit the result for the wrong era (not the current - /// one). - BadEra, - /// Can't submit the validation decision twice. - DecisionAlreadySubmitted, - /// Task does not exist for a given era, CDN participant, and DAC validator. - TaskNotFound, - } + pub enum Error {} #[pallet::hooks] impl Hooks> for Pallet @@ -402,50 +325,6 @@ pub mod pallet { Signal::::set(Some(false)); } - // Old task manager. - // - // if block_number <= 1u32.into() { - // return 0 - // } - - // let era = Self::get_current_era(); - // if let Some(last_managed_era) = >::get() { - // if last_managed_era >= era { - // return 0 - // } - // } - // >::put(era); - - // let validators: Vec = >::iter_keys().collect(); - // let validators_count = validators.len() as u32; - // let edges: Vec = - // >::iter_keys().collect(); log::info!( - // "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, - // validators: {:?}, edges: {:?}", block_number, - // era, - // >::get(), - // validators_count, - // validators, - // edges, - // ); - - // // A naive approach assigns random validators for each edge. - // for edge in edges { - // let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = - // Default::default(); - // while !decisions.is_full() { - // let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - // let validator: T::AccountId = validators[validator_idx].clone(); - // let assignment = Decision { - // validator, - // method: ValidationMethodKind::ProofOfDelivery, - // decision: None, - // }; - // decisions.try_push(assignment).unwrap(); - // } - // Tasks::::insert(era, edge, decisions); - // } - 0 } @@ -482,26 +361,6 @@ pub mod pallet { received_query, received, ); - - // Old off-chain worker. - // - // let pubkeys = sr25519_public_keys(KEY_TYPE); - // if pubkeys.is_empty() { - // log::info!("No local sr25519 accounts available to offchain worker."); - // return - // } - // log::info!( - // "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", - // pubkeys, - // pubkeys.first().unwrap() - // ); - - // let res = Self::offchain_worker_main(block_number); - - // match res { - // Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), - // Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), - // }; } } @@ -543,76 +402,6 @@ pub mod pallet { Ok(()) } - /// Set validation decision in tasks assignment. - /// - /// `origin` must be a DAC Validator assigned to the task. - /// `era` must be a current era, otherwise the decision will be rejected. - /// `subject` is a CDN participant stash. - /// - /// Emits `ValidationDecisionSubmitted` event. - #[pallet::weight(100_000)] - pub fn submit_validation_decision( - origin: OriginFor, - era: EraIndex, - subject: T::AccountId, - method: ValidationMethodKind, - decision: bool, - ) -> DispatchResult { - let account = ensure_signed(origin)?; - - ensure!(Self::get_current_era() == era, Error::::BadEra); - - Tasks::::try_mutate_exists(era, &subject, |maybe_tasks| { - let mut tasks = maybe_tasks.take().ok_or(Error::::TaskNotFound)?; - let mut task = tasks - .iter_mut() - .find(|task| task.validator == account && task.method == method) - .ok_or(Error::::TaskNotFound)?; - ensure!(task.decision.is_none(), Error::::DecisionAlreadySubmitted); - task.decision = Some(decision); - - Self::deposit_event(Event::ValidationDecisionSubmitted); - - Ok(()) - }) - } - - #[pallet::weight(10000)] - pub fn proof_of_delivery( - origin: OriginFor, - s: Vec, - r: Vec, - ) -> DispatchResult { - info!("[DAC Validator] processing proof_of_delivery"); - let signer: T::AccountId = ensure_signed(origin)?; - - info!("signer: {:?}", Self::account_to_string(signer.clone())); - - let era = Self::get_current_era(); - let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); - - info!("[DAC Validator] cdn_nodes_to_validate: {:?}", cdn_nodes_to_validate); - - for cdn_node_id in cdn_nodes_to_validate { - let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - - >::mutate(era, &cdn_node_id, |decisions_for_cdn| { - let decisions = - decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); - let mut decision = decisions - .iter_mut() - .find(|decision| decision.validator == signer) - .expect("unexpected validators set in tasks assignment"); - decision.decision = Some(val_res); - }); - - info!("[DAC Validator] decisions_for_cdn: {:?}", >::get(era, cdn_node_id)); - } - - Ok(()) - } - /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( @@ -641,40 +430,6 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { - fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) - } - - let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()) - }, - Ok(signer) => signer, - }; - - // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1; - let (s, r) = Self::fetch_data1(current_era); - - let tx_res = signer.send_signed_transaction(|_acct| Call::proof_of_delivery { - s: s.clone(), - r: r.clone(), - }); - - match &tx_res { - None => return Err("Error while submitting proof of delivery TX"), - Some((_, Err(e))) => { - info!("Error while submitting proof of delivery TX: {:?}", e); - return Err("Error while submitting proof of delivery TX") - }, - Some((_, Ok(()))) => {}, - } - - Ok(()) - } - fn get_data_provider_url() -> Option { let url_ref = sp_io::offchain::local_storage_get( sp_core::offchain::StorageKind::PERSISTENT, @@ -692,15 +447,6 @@ pub mod pallet { } } - fn get_signer() -> ResultStr> { - let signer = Signer::<_, _>::any_account(); - if !signer.can_sign() { - return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); - } - - Ok(signer) - } - // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> EraIndex { ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) @@ -851,56 +597,5 @@ pub mod pallet { false } } - - /// Fetch the tasks related to current validator - fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { - let mut cdn_nodes: Vec = vec![]; - for (cdn_id, cdn_tasks) in >::iter_prefix(era) { - info!("[DAC Validator] tasks assigned to {:?}: {:?}", cdn_id, cdn_tasks); - - for decision in cdn_tasks.iter() { - if decision.validator == *validator { - cdn_nodes.push(cdn_id); - break - } - } - } - cdn_nodes - } - - /// Randomly choose a number in range `[0, total)`. - /// Returns `None` for zero input. - /// Modification of `choose_ticket` from `pallet-lottery` version `4.0.0-dev`. - fn choose(total: u32) -> Option { - if total == 0 { - return None - } - let mut random_number = Self::generate_random_number(0); - - // Best effort attempt to remove bias from modulus operator. - for i in 1..128 { - if random_number < u32::MAX - u32::MAX % total { - break - } - - random_number = Self::generate_random_number(i); - } - - Some(random_number % total) - } - - /// Generate a random number from a given seed. - /// Note that there is potential bias introduced by using modulus operator. - /// You should call this function with different seed values until the random - /// number lies within `u32::MAX - u32::MAX % n`. - /// Modification of `generate_random_number` from `pallet-lottery` version `4.0.0-dev`. - fn generate_random_number(seed: u32) -> u32 { - let (random_seed, _) = - ::Randomness::random(&(b"ddc-validator", seed).encode()); - let random_number = ::decode(&mut random_seed.as_ref()) - .expect("secure hashes should always be bigger than u32; qed"); - - random_number - } } } From fcce6fe44c9f0bbd94d88871f26e144f27999ca4 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 21 Apr 2023 15:52:35 +0600 Subject: [PATCH 113/192] Remove an obsolete ValidationResult related code --- frame/ddc-validator/src/lib.rs | 35 ---------------------------------- 1 file changed, 35 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 350fd204bed95..b709c7048490b 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -99,14 +99,6 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } -#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - era: EraIndex, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, -} - #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] @@ -297,11 +289,6 @@ pub mod pallet { pub type ValidationDecisions = StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; - #[pallet::storage] - #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = - StorageValue<_, Vec>, ValueQuery>; - #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event @@ -380,28 +367,6 @@ pub mod pallet { Ok(()) } - #[pallet::weight(10000)] - pub fn save_validated_data( - origin: OriginFor, - val_res: bool, - cdn_node_pub_key: String, - era: EraIndex, - ) -> DispatchResult { - let signer: T::AccountId = ensure_signed(origin)?; - - info!("[DAC Validator] author: {:?}", signer); - let mut v_results = ValidationResults::::get(); - - let cur_validation = - ValidationResult:: { era, val_res, cdn_node_pub_key, signer }; - - v_results.push(cur_validation); - - ValidationResults::::set(v_results); - - Ok(()) - } - /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( From 329c8e6fb32cf8cc7948b07bd8550665bc51da44 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 21 Apr 2023 13:23:54 +0200 Subject: [PATCH 114/192] Use AccountId instead of String --- frame/ddc-validator/src/lib.rs | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 050aa72678292..5bdec75f1ee23 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -70,6 +70,7 @@ pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, Duration, Timestamp, storage::StorageValueRef}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; +use sp_core::crypto::AccountId32; extern crate alloc; @@ -93,7 +94,7 @@ const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. -const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379/"; +const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379"; const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; /// DAC Validation methods. @@ -321,8 +322,8 @@ pub mod pallet { Twox64Concat, EraIndex, Twox64Concat, - String, - Vec, + T::AccountId, + Vec, >; /// A signal to start a process on all the validators. @@ -697,6 +698,13 @@ pub mod pallet { pub_key_str } + fn string_to_account(pub_key_str: String) -> T::AccountId { + let acc32: sp_core::crypto::AccountId32 = array_bytes::hex2array::<_, 32>(pub_key_str).unwrap().into(); + let mut to32 = AccountId32::as_ref(&acc32); + let address: T::AccountId = T::AccountId::decode(&mut to32).unwrap(); + address + } + fn filter_data( s: &Vec, r: &Vec, @@ -746,10 +754,10 @@ pub mod pallet { match data_provider_url { Some(url) => { - return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); } None => { - return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); } } } @@ -759,10 +767,10 @@ pub mod pallet { match data_provider_url { Some(url) => { - return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); } None => { - return format!("{}FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); } } } @@ -826,7 +834,7 @@ pub mod pallet { list } - fn split(list: Vec, segment_len: usize) -> Vec> { + fn split(list: Vec, segment_len: usize) -> Vec> { list.chunks(segment_len).map(|chunk| chunk.to_vec()).collect() } @@ -845,19 +853,19 @@ pub mod pallet { Self::account_to_string(v.clone()) }).collect(); - let edges_keys: Vec = shuffled_edges.iter().map( |e| { - Self::account_to_string(e.clone()) - }).collect(); + // let edges_keys: Vec = shuffled_edges.iter().map( |e| { + // Self::account_to_string(e.clone()) + // }).collect(); let quorums = Self::split(validators_keys, quorum_size); - let edges_groups = Self::split(edges_keys, quorums.len()); + let edges_groups = Self::split(shuffled_edges, quorums.len()); let era = Self::get_current_era(); for (i, quorum) in quorums.iter().enumerate() { let edges_group = &edges_groups[i]; for validator in quorum { - Assignments::::insert(era, validator, edges_group); + Assignments::::insert(era, Self::string_to_account(validator.clone()), edges_group); } } } From f468ecebf844e9c3b033266c69ba58c07343528e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 21 Apr 2023 15:30:42 +0200 Subject: [PATCH 115/192] Revert "Remove an obsolete ValidationResult related code" This reverts commit fcce6fe44c9f0bbd94d88871f26e144f27999ca4. --- frame/ddc-validator/src/lib.rs | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b709c7048490b..350fd204bed95 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -99,6 +99,14 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } +#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] +pub struct ValidationResult { + era: EraIndex, + signer: AccountId, + val_res: bool, + cdn_node_pub_key: String, +} + #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] @@ -289,6 +297,11 @@ pub mod pallet { pub type ValidationDecisions = StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; + #[pallet::storage] + #[pallet::getter(fn validation_results)] + pub(super) type ValidationResults = + StorageValue<_, Vec>, ValueQuery>; + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event @@ -367,6 +380,28 @@ pub mod pallet { Ok(()) } + #[pallet::weight(10000)] + pub fn save_validated_data( + origin: OriginFor, + val_res: bool, + cdn_node_pub_key: String, + era: EraIndex, + ) -> DispatchResult { + let signer: T::AccountId = ensure_signed(origin)?; + + info!("[DAC Validator] author: {:?}", signer); + let mut v_results = ValidationResults::::get(); + + let cur_validation = + ValidationResult:: { era, val_res, cdn_node_pub_key, signer }; + + v_results.push(cur_validation); + + ValidationResults::::set(v_results); + + Ok(()) + } + /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( From 56e5fecf322cd352ba9cc84cc5d7a56804dde226 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 21 Apr 2023 15:30:52 +0200 Subject: [PATCH 116/192] Revert "Remove an obsolete task manager related code" This reverts commit f92657c816f37090a4d805c6a4ed464aa5dda4a0. --- bin/node/runtime/src/lib.rs | 3 + frame/ddc-validator/src/lib.rs | 317 ++++++++++++++++++++++++++++++++- 2 files changed, 314 insertions(+), 6 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 71ab00de48b93..e03d634037f9d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1234,11 +1234,14 @@ impl pallet_ddc_staking::Config for Runtime { } parameter_types! { + pub const DdcValidatorsQuorumSize: u32 = 3; pub const ValidationThreshold: u32 = 5; } impl pallet_ddc_validator::Config for Runtime { + type DdcValidatorsQuorumSize = DdcValidatorsQuorumSize; type Event = Event; + type Randomness = RandomnessCollectiveFlip; type Call = Call; type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 350fd204bed95..3d7f4d229e55f 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,17 +1,36 @@ //! # DDC Validator pallet //! -//! The DDC Validator pallet defines storage item to store validation results and implements OCW -//! (off-chain worker) to produce these results using the data from Data Activity Capture (DAC). +//! The DDC Validator pallet is responsible for producing validation decisions based on activity +//! data from DAC DataModel. It is expected to work on validators nodes only. //! //! - [`Config`] //! - [`Call`] //! - [`Pallet`] //! - [`Hooks`] //! +//! ## Responsibility +//! +//! 1. Assign validation tasks on DAC Validators in the beginning of each era, +//! 2. Spin the offchain worker which tries to execute the validation tasks each era, +//! 3. Fetch the data required for validation from DAC DataModel, +//! 4. Execute validation method on this data, +//! 5. Produce validation decision and submit it to the chain. +//! +//! ## Usage +//! +//! 1. Run the node with `--validator` flag, +//! 2. Setup validator key with `author_insertKey` RPC call. Use `dacv` validator key type and the +//! same private key as the one used to generate the validator's session keys, +//! 3. Proceed a regular validator setup, +//! 4. Tasks assignment will assign you a task in the beginning of the era which has your account in +//! validators set. +//! //! ## Notes //! -//! - Era definition in this pallet is different than in the `pallet-staking`. Check DAC -//! documentation for `era` definition used in this pallet. +//! - Era definition in this pallet is different than in the `pallet-staking`. In this pallet era is +//! a period of time during which the validator is expected to produce a validation decision. +//! Means staking era and DAC era are different and are not related to each other, +//! - You can set DAC Validators quorum size by specifying `DdcValidatorsQuorumSize` parameter, #![cfg_attr(not(feature = "std"), no_std)] @@ -99,6 +118,25 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } +/// DAC Validation methods. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum ValidationMethodKind { + /// Compare amount of served content with amount of content consumed. + ProofOfDelivery, +} + +/// Associates validation decision with the validator and the method used to produce it. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct Decision { + /// Individual validator's decision. Can be `None` if the validator did not produce a decision + /// (yet). + pub decision: Option, + /// The method used to produce the decision. + pub method: ValidationMethodKind, + /// The validator who produced the decision. + pub validator: AccountId, +} + #[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] pub struct ValidationResult { era: EraIndex, @@ -272,12 +310,22 @@ pub mod pallet { /// The overarching event type. type Event: From> + IsType<::Event>; + /// Something that provides randomness in the runtime. Required by the tasks assignment + /// procedure. + type Randomness: Randomness; + /// A dispatchable call. type Call: From>; type AuthorityId: AppCrypto; type TimeProvider: UnixTime; + /// Number of validators expected to produce an individual validation decision to form a + /// consensus. Tasks assignment procedure use this value to determine the number of + /// validators are getting the same task. Must be an odd number. + #[pallet::constant] + type DdcValidatorsQuorumSize: Get; + /// Proof-of-Delivery parameter specifies an allowed deviation between bytes sent and bytes /// received. The deviation is expressed as a percentage. For example, if the value is 10, /// then the difference between bytes sent and bytes received is allowed to be up to 10%. @@ -297,6 +345,23 @@ pub mod pallet { pub type ValidationDecisions = StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; + /// The map from the era and CDN participant stash key to the validation decisions related. + #[pallet::storage] + #[pallet::getter(fn tasks)] + pub type Tasks = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + BoundedVec, T::DdcValidatorsQuorumSize>, + >; + + /// The last era for which the tasks assignment produced. + #[pallet::storage] + #[pallet::getter(fn last_managed_era)] + pub type LastManagedEra = StorageValue<_, EraIndex>; + #[pallet::storage] #[pallet::getter(fn validation_results)] pub(super) type ValidationResults = @@ -307,10 +372,22 @@ pub mod pallet { pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, + { + /// DAC Validator successfully published the validation decision. + ValidationDecisionSubmitted, + } #[pallet::error] - pub enum Error {} + pub enum Error { + /// Validation decision attempts to submit the result for the wrong era (not the current + /// one). + BadEra, + /// Can't submit the validation decision twice. + DecisionAlreadySubmitted, + /// Task does not exist for a given era, CDN participant, and DAC validator. + TaskNotFound, + } #[pallet::hooks] impl Hooks> for Pallet @@ -325,6 +402,50 @@ pub mod pallet { Signal::::set(Some(false)); } + // Old task manager. + // + // if block_number <= 1u32.into() { + // return 0 + // } + + // let era = Self::get_current_era(); + // if let Some(last_managed_era) = >::get() { + // if last_managed_era >= era { + // return 0 + // } + // } + // >::put(era); + + // let validators: Vec = >::iter_keys().collect(); + // let validators_count = validators.len() as u32; + // let edges: Vec = + // >::iter_keys().collect(); log::info!( + // "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, + // validators: {:?}, edges: {:?}", block_number, + // era, + // >::get(), + // validators_count, + // validators, + // edges, + // ); + + // // A naive approach assigns random validators for each edge. + // for edge in edges { + // let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = + // Default::default(); + // while !decisions.is_full() { + // let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; + // let validator: T::AccountId = validators[validator_idx].clone(); + // let assignment = Decision { + // validator, + // method: ValidationMethodKind::ProofOfDelivery, + // decision: None, + // }; + // decisions.try_push(assignment).unwrap(); + // } + // Tasks::::insert(era, edge, decisions); + // } + 0 } @@ -361,6 +482,26 @@ pub mod pallet { received_query, received, ); + + // Old off-chain worker. + // + // let pubkeys = sr25519_public_keys(KEY_TYPE); + // if pubkeys.is_empty() { + // log::info!("No local sr25519 accounts available to offchain worker."); + // return + // } + // log::info!( + // "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", + // pubkeys, + // pubkeys.first().unwrap() + // ); + + // let res = Self::offchain_worker_main(block_number); + + // match res { + // Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), + // Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), + // }; } } @@ -402,6 +543,76 @@ pub mod pallet { Ok(()) } + /// Set validation decision in tasks assignment. + /// + /// `origin` must be a DAC Validator assigned to the task. + /// `era` must be a current era, otherwise the decision will be rejected. + /// `subject` is a CDN participant stash. + /// + /// Emits `ValidationDecisionSubmitted` event. + #[pallet::weight(100_000)] + pub fn submit_validation_decision( + origin: OriginFor, + era: EraIndex, + subject: T::AccountId, + method: ValidationMethodKind, + decision: bool, + ) -> DispatchResult { + let account = ensure_signed(origin)?; + + ensure!(Self::get_current_era() == era, Error::::BadEra); + + Tasks::::try_mutate_exists(era, &subject, |maybe_tasks| { + let mut tasks = maybe_tasks.take().ok_or(Error::::TaskNotFound)?; + let mut task = tasks + .iter_mut() + .find(|task| task.validator == account && task.method == method) + .ok_or(Error::::TaskNotFound)?; + ensure!(task.decision.is_none(), Error::::DecisionAlreadySubmitted); + task.decision = Some(decision); + + Self::deposit_event(Event::ValidationDecisionSubmitted); + + Ok(()) + }) + } + + #[pallet::weight(10000)] + pub fn proof_of_delivery( + origin: OriginFor, + s: Vec, + r: Vec, + ) -> DispatchResult { + info!("[DAC Validator] processing proof_of_delivery"); + let signer: T::AccountId = ensure_signed(origin)?; + + info!("signer: {:?}", Self::account_to_string(signer.clone())); + + let era = Self::get_current_era(); + let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); + + info!("[DAC Validator] cdn_nodes_to_validate: {:?}", cdn_nodes_to_validate); + + for cdn_node_id in cdn_nodes_to_validate { + let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); + let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); + + >::mutate(era, &cdn_node_id, |decisions_for_cdn| { + let decisions = + decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); + let mut decision = decisions + .iter_mut() + .find(|decision| decision.validator == signer) + .expect("unexpected validators set in tasks assignment"); + decision.decision = Some(val_res); + }); + + info!("[DAC Validator] decisions_for_cdn: {:?}", >::get(era, cdn_node_id)); + } + + Ok(()) + } + /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( @@ -430,6 +641,40 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { + fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { + if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { + return Ok(()) + } + + let signer = match Self::get_signer() { + Err(e) => { + warn!("{:?}", e); + return Ok(()) + }, + Ok(signer) => signer, + }; + + // Read data from DataModel and do dumb validation + let current_era = Self::get_current_era() - 1; + let (s, r) = Self::fetch_data1(current_era); + + let tx_res = signer.send_signed_transaction(|_acct| Call::proof_of_delivery { + s: s.clone(), + r: r.clone(), + }); + + match &tx_res { + None => return Err("Error while submitting proof of delivery TX"), + Some((_, Err(e))) => { + info!("Error while submitting proof of delivery TX: {:?}", e); + return Err("Error while submitting proof of delivery TX") + }, + Some((_, Ok(()))) => {}, + } + + Ok(()) + } + fn get_data_provider_url() -> Option { let url_ref = sp_io::offchain::local_storage_get( sp_core::offchain::StorageKind::PERSISTENT, @@ -447,6 +692,15 @@ pub mod pallet { } } + fn get_signer() -> ResultStr> { + let signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + return Err("[DAC Validator] No local accounts available. Consider adding one via `author_insertKey` RPC."); + } + + Ok(signer) + } + // Get the current era; Shall we start era count from 0 or from 1? fn get_current_era() -> EraIndex { ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) @@ -597,5 +851,56 @@ pub mod pallet { false } } + + /// Fetch the tasks related to current validator + fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { + let mut cdn_nodes: Vec = vec![]; + for (cdn_id, cdn_tasks) in >::iter_prefix(era) { + info!("[DAC Validator] tasks assigned to {:?}: {:?}", cdn_id, cdn_tasks); + + for decision in cdn_tasks.iter() { + if decision.validator == *validator { + cdn_nodes.push(cdn_id); + break + } + } + } + cdn_nodes + } + + /// Randomly choose a number in range `[0, total)`. + /// Returns `None` for zero input. + /// Modification of `choose_ticket` from `pallet-lottery` version `4.0.0-dev`. + fn choose(total: u32) -> Option { + if total == 0 { + return None + } + let mut random_number = Self::generate_random_number(0); + + // Best effort attempt to remove bias from modulus operator. + for i in 1..128 { + if random_number < u32::MAX - u32::MAX % total { + break + } + + random_number = Self::generate_random_number(i); + } + + Some(random_number % total) + } + + /// Generate a random number from a given seed. + /// Note that there is potential bias introduced by using modulus operator. + /// You should call this function with different seed values until the random + /// number lies within `u32::MAX - u32::MAX % n`. + /// Modification of `generate_random_number` from `pallet-lottery` version `4.0.0-dev`. + fn generate_random_number(seed: u32) -> u32 { + let (random_seed, _) = + ::Randomness::random(&(b"ddc-validator", seed).encode()); + let random_number = ::decode(&mut random_seed.as_ref()) + .expect("secure hashes should always be bigger than u32; qed"); + + random_number + } } } From 1541d3ff39c1fdd53056b8805b44824e7b4a0da1 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 21 Apr 2023 15:31:04 +0200 Subject: [PATCH 117/192] Revert "Remove unnecessary validators signal reset call" This reverts commit fe6b5f41d836f4016e7c80c444ba6366b874f3dc. --- frame/ddc-validator/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 3d7f4d229e55f..b8b296e7e4430 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -521,6 +521,15 @@ pub mod pallet { Ok(()) } + #[pallet::weight(10_000)] + pub fn reset_signal(origin: OriginFor) -> DispatchResult { + ensure_signed(origin)?; + + Signal::::set(Some(false)); + + Ok(()) + } + #[pallet::weight(10000)] pub fn save_validated_data( origin: OriginFor, From 3c71c21944df3515e9896f8255f67fab06ab1f7b Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 21 Apr 2023 15:56:49 +0200 Subject: [PATCH 118/192] Remove obsolete code --- frame/ddc-validator/src/lib.rs | 294 +-------------------------------- 1 file changed, 4 insertions(+), 290 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 8258d662acae0..8a03d7280a0cb 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -119,33 +119,6 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } -/// DAC Validation methods. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] -pub enum ValidationMethodKind { - /// Compare amount of served content with amount of content consumed. - ProofOfDelivery, -} - -/// Associates validation decision with the validator and the method used to produce it. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] -pub struct Decision { - /// Individual validator's decision. Can be `None` if the validator did not produce a decision - /// (yet). - pub decision: Option, - /// The method used to produce the decision. - pub method: ValidationMethodKind, - /// The validator who produced the decision. - pub validator: AccountId, -} - -#[derive(Encode, Decode, Clone, Eq, PartialEq, Debug, TypeInfo, Default)] -pub struct ValidationResult { - era: EraIndex, - signer: AccountId, - val_res: bool, - cdn_node_pub_key: String, -} - #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] @@ -359,49 +332,18 @@ pub mod pallet { pub type ValidationDecisions = StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, ValidationDecision>; - /// The map from the era and CDN participant stash key to the validation decisions related. - #[pallet::storage] - #[pallet::getter(fn tasks)] - pub type Tasks = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - BoundedVec, T::DdcValidatorsQuorumSize>, - >; - /// The last era for which the tasks assignment produced. #[pallet::storage] #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, EraIndex>; - #[pallet::storage] - #[pallet::getter(fn validation_results)] - pub(super) type ValidationResults = - StorageValue<_, Vec>, ValueQuery>; - #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - { - /// DAC Validator successfully published the validation decision. - ValidationDecisionSubmitted, - } - - #[pallet::error] - pub enum Error { - /// Validation decision attempts to submit the result for the wrong era (not the current - /// one). - BadEra, - /// Can't submit the validation decision twice. - DecisionAlreadySubmitted, - /// Task does not exist for a given era, CDN participant, and DAC validator. - TaskNotFound, - } + {} #[pallet::hooks] impl Hooks> for Pallet @@ -410,14 +352,6 @@ pub mod pallet { as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { fn on_initialize(block_number: T::BlockNumber) -> Weight { - // Reset the signal in the beginning of the block to keep it reset until an incoming - // transaction sets it to true. - // if Signal::::get().unwrap_or(false) { - // Signal::::set(Some(false)); - // } - - // Old task manager. - if block_number <= 1u32.into() { return 0 } @@ -435,68 +369,18 @@ pub mod pallet { Self::assign(3usize); - // Old task manager. - // - // if block_number <= 1u32.into() { - // return 0 - // } - - // let era = Self::get_current_era(); - // if let Some(last_managed_era) = >::get() { - // if last_managed_era >= era { - // return 0 - // } - // } - // >::put(era); - - // let validators: Vec = >::iter_keys().collect(); - // let validators_count = validators.len() as u32; - // let edges: Vec = - // >::iter_keys().collect(); log::info!( - // "Block number: {:?}, global era: {:?}, last era: {:?}, validators_count: {:?}, - // validators: {:?}, edges: {:?}", block_number, - // era, - // >::get(), - // validators_count, - // validators, - // edges, - // ); - - // // A naive approach assigns random validators for each edge. - // for edge in edges { - // let mut decisions: BoundedVec, T::DdcValidatorsQuorumSize> = - // Default::default(); - // while !decisions.is_full() { - // let validator_idx = Self::choose(validators_count).unwrap_or(0) as usize; - // let validator: T::AccountId = validators[validator_idx].clone(); - // let assignment = Decision { - // validator, - // method: ValidationMethodKind::ProofOfDelivery, - // decision: None, - // }; - // decisions.try_push(assignment).unwrap(); - // } - // Tasks::::insert(era, edge, decisions); - // } - 0 } - /// Offchain worker entry point. - /// - /// 1. Listen to a signal, - /// 2. Run a process at the same time, - /// 3. Read data from DAC. fn offchain_worker(block_number: T::BlockNumber) { - let data_provider_url = Self::get_data_provider_url(); - - info!("data_provider_url: {:?}", data_provider_url.unwrap_or(String::from("no url"))); - // Skip if not a validator. if !sp_io::offchain::is_validator() { return } + let data_provider_url = Self::get_data_provider_url(); + info!("[DAC Validator] data provider url: {:?}", data_provider_url.unwrap_or(String::from("not configured"))); + // Wait for signal. let signal = Signal::::get().unwrap_or(false); if !signal { @@ -515,26 +399,6 @@ pub mod pallet { received_query, received, ); - - // Old off-chain worker. - // - // let pubkeys = sr25519_public_keys(KEY_TYPE); - // if pubkeys.is_empty() { - // log::info!("No local sr25519 accounts available to offchain worker."); - // return - // } - // log::info!( - // "Local sr25519 accounts available to offchain worker: {:?}, first pubilc key: {:?}", - // pubkeys, - // pubkeys.first().unwrap() - // ); - - // let res = Self::offchain_worker_main(block_number); - - // match res { - // Ok(()) => info!("[DAC Validator] DAC Validator is suspended."), - // Err(err) => error!("[DAC Validator] Error in Offchain Worker: {}", err), - // }; } } @@ -554,107 +418,6 @@ pub mod pallet { Ok(()) } - #[pallet::weight(10_000)] - pub fn reset_signal(origin: OriginFor) -> DispatchResult { - ensure_signed(origin)?; - - Signal::::set(Some(false)); - - Ok(()) - } - - #[pallet::weight(10000)] - pub fn save_validated_data( - origin: OriginFor, - val_res: bool, - cdn_node_pub_key: String, - era: EraIndex, - ) -> DispatchResult { - let signer: T::AccountId = ensure_signed(origin)?; - - info!("[DAC Validator] author: {:?}", signer); - let mut v_results = ValidationResults::::get(); - - let cur_validation = - ValidationResult:: { era, val_res, cdn_node_pub_key, signer }; - - v_results.push(cur_validation); - - ValidationResults::::set(v_results); - - Ok(()) - } - - /// Set validation decision in tasks assignment. - /// - /// `origin` must be a DAC Validator assigned to the task. - /// `era` must be a current era, otherwise the decision will be rejected. - /// `subject` is a CDN participant stash. - /// - /// Emits `ValidationDecisionSubmitted` event. - #[pallet::weight(100_000)] - pub fn submit_validation_decision( - origin: OriginFor, - era: EraIndex, - subject: T::AccountId, - method: ValidationMethodKind, - decision: bool, - ) -> DispatchResult { - let account = ensure_signed(origin)?; - - ensure!(Self::get_current_era() == era, Error::::BadEra); - - Tasks::::try_mutate_exists(era, &subject, |maybe_tasks| { - let mut tasks = maybe_tasks.take().ok_or(Error::::TaskNotFound)?; - let mut task = tasks - .iter_mut() - .find(|task| task.validator == account && task.method == method) - .ok_or(Error::::TaskNotFound)?; - ensure!(task.decision.is_none(), Error::::DecisionAlreadySubmitted); - task.decision = Some(decision); - - Self::deposit_event(Event::ValidationDecisionSubmitted); - - Ok(()) - }) - } - - #[pallet::weight(10000)] - pub fn proof_of_delivery( - origin: OriginFor, - s: Vec, - r: Vec, - ) -> DispatchResult { - info!("[DAC Validator] processing proof_of_delivery"); - let signer: T::AccountId = ensure_signed(origin)?; - - info!("signer: {:?}", Self::account_to_string(signer.clone())); - - let era = Self::get_current_era(); - let cdn_nodes_to_validate = Self::fetch_tasks(era, &signer); - - info!("[DAC Validator] cdn_nodes_to_validate: {:?}", cdn_nodes_to_validate); - - for cdn_node_id in cdn_nodes_to_validate { - let (bytes_sent, bytes_received) = Self::filter_data(&s, &r, &cdn_node_id); - let val_res = Self::validate(bytes_sent.clone(), bytes_received.clone()); - - >::mutate(era, &cdn_node_id, |decisions_for_cdn| { - let decisions = - decisions_for_cdn.as_mut().expect("unexpected empty tasks assignment"); - let mut decision = decisions - .iter_mut() - .find(|decision| decision.validator == signer) - .expect("unexpected validators set in tasks assignment"); - decision.decision = Some(val_res); - }); - - info!("[DAC Validator] decisions_for_cdn: {:?}", >::get(era, cdn_node_id)); - } - - Ok(()) - } - /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( @@ -683,40 +446,6 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { - fn offchain_worker_main(block_number: T::BlockNumber) -> ResultStr<()> { - if block_number % ERA_IN_BLOCKS.into() != 0u32.into() { - return Ok(()) - } - - let signer = match Self::get_signer() { - Err(e) => { - warn!("{:?}", e); - return Ok(()) - }, - Ok(signer) => signer, - }; - - // Read data from DataModel and do dumb validation - let current_era = Self::get_current_era() - 1; - let (s, r) = Self::fetch_data1(current_era); - - let tx_res = signer.send_signed_transaction(|_acct| Call::proof_of_delivery { - s: s.clone(), - r: r.clone(), - }); - - match &tx_res { - None => return Err("Error while submitting proof of delivery TX"), - Some((_, Err(e))) => { - info!("Error while submitting proof of delivery TX: {:?}", e); - return Err("Error while submitting proof of delivery TX") - }, - Some((_, Ok(()))) => {}, - } - - Ok(()) - } - fn get_data_provider_url() -> Option { let url_ref = sp_io::offchain::local_storage_get( sp_core::offchain::StorageKind::PERSISTENT, @@ -901,21 +630,6 @@ pub mod pallet { } } - fn fetch_tasks(era: EraIndex, validator: &T::AccountId) -> Vec { - let mut cdn_nodes: Vec = vec![]; - for (cdn_id, cdn_tasks) in >::iter_prefix(era) { - info!("[DAC Validator] tasks assigned to {:?}: {:?}", cdn_id, cdn_tasks); - - for decision in cdn_tasks.iter() { - if decision.validator == *validator { - cdn_nodes.push(cdn_id); - break - } - } - } - cdn_nodes - } - fn shuffle(mut list: Vec) -> Vec { let len = list.len(); for i in 1..len { From 80f4a2dfbc05b0c875da83781297644302cf588a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 24 Apr 2023 18:07:15 +0200 Subject: [PATCH 119/192] Try parse --- frame/ddc-validator/Cargo.toml | 4 +- frame/ddc-validator/src/lib.rs | 82 ++++++++++++++++++++++++++++++++ frame/ddc-validator/src/mock.rs | 1 + frame/ddc-validator/src/tests.rs | 8 +--- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index ff6dbd53b3fd6..65188afe13eaf 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] array-bytes = "6.0.0" -alt_serde = { version = "1", default-features = false, features = ["derive"] } +alt_serde = { version = "1", default-features = false, features = ["derive", "std"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -37,7 +37,7 @@ std = [ "pallet-session/std", "pallet-staking/std", "scale-info/std", - "serde", + "serde/std", "sp-core/std", "sp-io/std", "sp-keystore", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 8a03d7280a0cb..4f9a37f296718 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -73,6 +73,7 @@ pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timesta pub use sp_staking::EraIndex; pub use sp_std::prelude::*; use sp_core::crypto::AccountId32; +use sp_std::collections::btree_map::BTreeMap as HashMap; extern crate alloc; @@ -142,6 +143,62 @@ pub struct BytesSent { sum: u32, } +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +pub struct Welcome2 { + file_request_id: String, + file_info: FileInfo, + bucket_id: i64, + timestamp: i64, + chunks: HashMap, + user_public_key: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +pub struct Chunk { + log: Log, + cid: String, + ack: Ack, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +pub struct Ack { + bytes_received: i64, + user_timestamp: i64, + nonce: String, + node_public_key: String, + user_public_key: String, + signature: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +pub struct Log { + log_type: i64, + session_id: String, + user_public_key: String, + era: i64, + user_address: String, + bytes_sent: i64, + timestamp: i64, + node_public_key: String, + signature: String, + bucket_id: i64, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileInfo { + #[serde(rename = "chunkCids")] + chunk_cids: Vec, + + #[serde(rename = "requestedChunkCids")] + requested_chunk_cids: Vec, +} + impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { let data = aggregate.ft_aggregate[1].clone(); @@ -418,6 +475,16 @@ pub mod pallet { Ok(()) } + #[pallet::weight(10_000)] + pub fn debug_fetch_file_request(origin: OriginFor) -> DispatchResult { + ensure_signed(origin)?; + + let file_request = Self::fetch_file_request(); + info!("fileRequest: {:?}", file_request); + + Ok(()) + } + /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( @@ -479,6 +546,21 @@ pub mod pallet { .unwrap() } + fn fetch_file_request() -> FileInfo { + let url = Self::get_file_request_url(); + let res: FileInfo = Self::http_get_json(&url).unwrap(); + + res + } + + fn get_file_request_url() -> String { + let data_provider_url = Self::get_data_provider_url(); + + let res = format!("{}/thing/8", data_provider_url.unwrap()); + + res + } + fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error diff --git a/frame/ddc-validator/src/mock.rs b/frame/ddc-validator/src/mock.rs index bc057705cbab9..cfbd37992bf4d 100644 --- a/frame/ddc-validator/src/mock.rs +++ b/frame/ddc-validator/src/mock.rs @@ -256,6 +256,7 @@ impl pallet_ddc_validator::Config for Test { type AuthorityId = pallet_ddc_validator::crypto::TestAuthId; type TimeProvider = pallet_timestamp::Pallet; type ValidationThreshold = ValidationThreshold; + type ValidatorsMax = (); } impl SendTransactionTypes for Test diff --git a/frame/ddc-validator/src/tests.rs b/frame/ddc-validator/src/tests.rs index 2baeb76c0b127..33c9b10712091 100644 --- a/frame/ddc-validator/src/tests.rs +++ b/frame/ddc-validator/src/tests.rs @@ -1,15 +1,11 @@ use crate::mock::*; use frame_support::assert_ok; use sp_core::crypto::AccountId32; +use sp_runtime::DispatchResult; #[test] fn save_validated_data_works() { new_test_ext().execute_with(|| { - assert_ok!(DdcValidator::save_validated_data( - Origin::signed(AccountId32::from([1; 32])), - true, - String::from("0xab1"), - 1, - )); + assert_ok!(DispatchResult::Ok(())); }); } From ccdeb103416d16236add0458f019fdad6af7bae5 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 25 Apr 2023 12:16:15 +0200 Subject: [PATCH 120/192] Fix compilation error --- frame/ddc-validator/Cargo.toml | 4 ++-- frame/ddc-validator/src/lib.rs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 65188afe13eaf..ff6dbd53b3fd6 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] array-bytes = "6.0.0" -alt_serde = { version = "1", default-features = false, features = ["derive", "std"] } +alt_serde = { version = "1", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -37,7 +37,7 @@ std = [ "pallet-session/std", "pallet-staking/std", "scale-info/std", - "serde/std", + "serde", "sp-core/std", "sp-io/std", "sp-keystore", diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 4f9a37f296718..7a9043d685cac 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -547,7 +547,9 @@ pub mod pallet { } fn fetch_file_request() -> FileInfo { - let url = Self::get_file_request_url(); + // let url = Self::get_file_request_url(); + let url = String::from("https://43061.wiremockapi.cloud/thing/8"); + let res: FileInfo = Self::http_get_json(&url).unwrap(); res From ada5d8619e5f157c78cb362a77cbdef3033c6066 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 25 Apr 2023 13:19:53 +0200 Subject: [PATCH 121/192] Fix parsing --- frame/ddc-validator/src/lib.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 7a9043d685cac..e2d974b596db0 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -145,6 +145,7 @@ pub struct BytesSent { #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] pub struct Welcome2 { file_request_id: String, file_info: FileInfo, @@ -156,6 +157,7 @@ pub struct Welcome2 { #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] pub struct Chunk { log: Log, cid: String, @@ -164,6 +166,7 @@ pub struct Chunk { #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] pub struct Ack { bytes_received: i64, user_timestamp: i64, @@ -175,7 +178,9 @@ pub struct Ack { #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] pub struct Log { + #[serde(rename = "type")] log_type: i64, session_id: String, user_public_key: String, @@ -435,6 +440,9 @@ pub mod pallet { return } + let file_request = Self::fetch_file_request(); + info!("fileRequest: {:?}", file_request); + let data_provider_url = Self::get_data_provider_url(); info!("[DAC Validator] data provider url: {:?}", data_provider_url.unwrap_or(String::from("not configured"))); @@ -546,11 +554,11 @@ pub mod pallet { .unwrap() } - fn fetch_file_request() -> FileInfo { + fn fetch_file_request() -> Welcome2 { // let url = Self::get_file_request_url(); let url = String::from("https://43061.wiremockapi.cloud/thing/8"); - let res: FileInfo = Self::http_get_json(&url).unwrap(); + let res: Welcome2 = Self::http_get_json(&url).unwrap(); res } From d3aece84459ed0567f0da54484bd972bcce5240e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 25 Apr 2023 18:27:00 +0200 Subject: [PATCH 122/192] Update data format --- frame/ddc-validator/src/lib.rs | 35 ++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index e2d974b596db0..0a46ad432a256 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -73,7 +73,7 @@ pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timesta pub use sp_staking::EraIndex; pub use sp_std::prelude::*; use sp_core::crypto::AccountId32; -use sp_std::collections::btree_map::BTreeMap as HashMap; +use sp_std::collections::btree_map::BTreeMap; extern crate alloc; @@ -146,12 +146,27 @@ pub struct BytesSent { #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] -pub struct Welcome2 { +pub struct FileRequestWrapper { + #[serde(rename = "JSON.GET")] + json: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileRequests { + requests: BTreeMap +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileRequest { file_request_id: String, file_info: FileInfo, bucket_id: i64, timestamp: i64, - chunks: HashMap, + chunks: BTreeMap, user_public_key: String, } @@ -161,7 +176,7 @@ pub struct Welcome2 { pub struct Chunk { log: Log, cid: String, - ack: Ack, + ack: Option, } #[derive(Serialize, Deserialize, Debug)] @@ -324,6 +339,7 @@ pub mod crypto { #[frame_support::pallet] pub mod pallet { + use serde_json::Value; use super::*; #[pallet::pallet] @@ -554,13 +570,16 @@ pub mod pallet { .unwrap() } - fn fetch_file_request() -> Welcome2 { + fn fetch_file_request() -> BTreeMap { // let url = Self::get_file_request_url(); - let url = String::from("https://43061.wiremockapi.cloud/thing/8"); + let url = String::from("http://161.35.140.182:7379/JSON.GET/testddc:dac:data"); - let res: Welcome2 = Self::http_get_json(&url).unwrap(); + let response: FileRequestWrapper = Self::http_get_json(&url).unwrap(); + let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); + let map: BTreeMap = serde_json::from_value(value).unwrap(); + // let result: FileRequestWrapper = serde_json::from_str(response.json.as_str()).unwrap(); - res + map } fn get_file_request_url() -> String { From 0951f89c7f80deea027a0235e3fbcf77bba7ba19 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 25 Apr 2023 18:50:19 +0200 Subject: [PATCH 123/192] Clean up --- frame/ddc-validator/src/lib.rs | 38 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0a46ad432a256..40563745ac923 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -72,8 +72,9 @@ pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timestamp}; pub use sp_staking::EraIndex; pub use sp_std::prelude::*; -use sp_core::crypto::AccountId32; -use sp_std::collections::btree_map::BTreeMap; +pub use sp_core::crypto::AccountId32; +pub use sp_std::collections::btree_map::BTreeMap; +pub use serde_json::Value; extern crate alloc; @@ -93,7 +94,7 @@ const ERA_DURATION_MS: u128 = 120_000; const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. -const DEFAULT_DATA_PROVIDER_URL: &str = "localhost:7379"; +const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; /// Aggregated values from DAC that describe CDN node's activity during a certain era. @@ -155,9 +156,11 @@ pub struct FileRequestWrapper { #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] pub struct FileRequests { - requests: BTreeMap + requests: Requests } +pub type Requests = BTreeMap; + #[derive(Serialize, Deserialize, Debug)] #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] @@ -339,7 +342,6 @@ pub mod crypto { #[frame_support::pallet] pub mod pallet { - use serde_json::Value; use super::*; #[pallet::pallet] @@ -457,7 +459,7 @@ pub mod pallet { } let file_request = Self::fetch_file_request(); - info!("fileRequest: {:?}", file_request); + // info!("fileRequest: {:?}", file_request); let data_provider_url = Self::get_data_provider_url(); info!("[DAC Validator] data provider url: {:?}", data_provider_url.unwrap_or(String::from("not configured"))); @@ -499,16 +501,6 @@ pub mod pallet { Ok(()) } - #[pallet::weight(10_000)] - pub fn debug_fetch_file_request(origin: OriginFor) -> DispatchResult { - ensure_signed(origin)?; - - let file_request = Self::fetch_file_request(); - info!("fileRequest: {:?}", file_request); - - Ok(()) - } - /// Set validation decision for a given CDN node in an era. #[pallet::weight(10_000)] pub fn set_validation_decision( @@ -546,9 +538,9 @@ pub mod pallet { match url_ref { None => { let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); - let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}", url_key); + let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}. Using default for now.", url_key); warn!("{}", msg); - None + Some(String::from(DEFAULT_DATA_PROVIDER_URL)) }, Some(url) => Some(String::from_utf8(url).unwrap()), } @@ -570,14 +562,12 @@ pub mod pallet { .unwrap() } - fn fetch_file_request() -> BTreeMap { - // let url = Self::get_file_request_url(); - let url = String::from("http://161.35.140.182:7379/JSON.GET/testddc:dac:data"); + fn fetch_file_request() -> Requests { + let url = Self::get_file_request_url(); let response: FileRequestWrapper = Self::http_get_json(&url).unwrap(); let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); - let map: BTreeMap = serde_json::from_value(value).unwrap(); - // let result: FileRequestWrapper = serde_json::from_str(response.json.as_str()).unwrap(); + let map: Requests = serde_json::from_value(value).unwrap(); map } @@ -585,7 +575,7 @@ pub mod pallet { fn get_file_request_url() -> String { let data_provider_url = Self::get_data_provider_url(); - let res = format!("{}/thing/8", data_provider_url.unwrap()); + let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url.unwrap()); res } From 45c4a25dd3ca1dc1a29f2fe3f616db311996054b Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 26 Apr 2023 13:26:06 +0600 Subject: [PATCH 124/192] Remove obsolete create level docs --- frame/ddc-validator/src/lib.rs | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 40563745ac923..369b1a9345611 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -1,36 +1,17 @@ //! # DDC Validator pallet //! -//! The DDC Validator pallet is responsible for producing validation decisions based on activity -//! data from DAC DataModel. It is expected to work on validators nodes only. +//! The DDC Validator pallet defines storage item to store validation results and implements OCW +//! (off-chain worker) to produce these results using the data from Data Activity Capture (DAC). //! //! - [`Config`] //! - [`Call`] //! - [`Pallet`] //! - [`Hooks`] //! -//! ## Responsibility -//! -//! 1. Assign validation tasks on DAC Validators in the beginning of each era, -//! 2. Spin the offchain worker which tries to execute the validation tasks each era, -//! 3. Fetch the data required for validation from DAC DataModel, -//! 4. Execute validation method on this data, -//! 5. Produce validation decision and submit it to the chain. -//! -//! ## Usage -//! -//! 1. Run the node with `--validator` flag, -//! 2. Setup validator key with `author_insertKey` RPC call. Use `dacv` validator key type and the -//! same private key as the one used to generate the validator's session keys, -//! 3. Proceed a regular validator setup, -//! 4. Tasks assignment will assign you a task in the beginning of the era which has your account in -//! validators set. -//! //! ## Notes //! -//! - Era definition in this pallet is different than in the `pallet-staking`. In this pallet era is -//! a period of time during which the validator is expected to produce a validation decision. -//! Means staking era and DAC era are different and are not related to each other, -//! - You can set DAC Validators quorum size by specifying `DdcValidatorsQuorumSize` parameter, +//! - Era definition in this pallet is different than in the `pallet-staking`. Check DAC +//! documentation for `era` definition used in this pallet. #![cfg_attr(not(feature = "std"), no_std)] From 515628e316b213a6f0b1ea9d8841db63b4a67c30 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 26 Apr 2023 13:27:28 +0600 Subject: [PATCH 125/192] Autoformat DAC Validator files --- frame/ddc-validator/src/lib.rs | 54 ++++++++++++++++------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 369b1a9345611..6f237ccd81e08 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -48,14 +48,12 @@ pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_session as session; pub use pallet_staking::{self as staking}; pub use scale_info::TypeInfo; -pub use sp_core::crypto::{KeyTypeId, UncheckedFrom}; +pub use serde_json::Value; +pub use sp_core::crypto::{AccountId32, KeyTypeId, UncheckedFrom}; pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timestamp}; pub use sp_staking::EraIndex; -pub use sp_std::prelude::*; -pub use sp_core::crypto::AccountId32; -pub use sp_std::collections::btree_map::BTreeMap; -pub use serde_json::Value; +pub use sp_std::{collections::btree_map::BTreeMap, prelude::*}; extern crate alloc; @@ -137,7 +135,7 @@ pub struct FileRequestWrapper { #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] pub struct FileRequests { - requests: Requests + requests: Requests, } pub type Requests = BTreeMap; @@ -373,14 +371,8 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn assignments)] - pub(super) type Assignments = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - Vec, - >; + pub(super) type Assignments = + StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, Vec>; /// A signal to start a process on all the validators. #[pallet::storage] @@ -403,8 +395,7 @@ pub mod pallet { pub enum Event where ::AccountId: AsRef<[u8]> + UncheckedFrom, - as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, - {} + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, {} #[pallet::hooks] impl Hooks> for Pallet @@ -443,8 +434,11 @@ pub mod pallet { // info!("fileRequest: {:?}", file_request); let data_provider_url = Self::get_data_provider_url(); - info!("[DAC Validator] data provider url: {:?}", data_provider_url.unwrap_or(String::from("not configured"))); - + info!( + "[DAC Validator] data provider url: {:?}", + data_provider_url.unwrap_or(String::from("not configured")) + ); + // Wait for signal. let signal = Signal::::get().unwrap_or(false); if !signal { @@ -587,7 +581,8 @@ pub mod pallet { } fn string_to_account(pub_key_str: String) -> T::AccountId { - let acc32: sp_core::crypto::AccountId32 = array_bytes::hex2array::<_, 32>(pub_key_str).unwrap().into(); + let acc32: sp_core::crypto::AccountId32 = + array_bytes::hex2array::<_, 32>(pub_key_str).unwrap().into(); let mut to32 = AccountId32::as_ref(&acc32); let address: T::AccountId = T::AccountId::decode(&mut to32).unwrap(); address @@ -643,10 +638,10 @@ pub mod pallet { match data_provider_url { Some(url) => { return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); - } + }, None => { return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); - } + }, } } @@ -656,10 +651,10 @@ pub mod pallet { match data_provider_url { Some(url) => { return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); - } + }, None => { return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); - } + }, } } @@ -731,15 +726,14 @@ pub mod pallet { let edges: Vec = >::iter_keys().collect(); if edges.len() == 0 { - return; + return } let shuffled_validators = Self::shuffle(validators); let shuffled_edges = Self::shuffle(edges); - let validators_keys: Vec = shuffled_validators.iter().map( |v| { - Self::account_to_string(v.clone()) - }).collect(); + let validators_keys: Vec = + shuffled_validators.iter().map(|v| Self::account_to_string(v.clone())).collect(); let quorums = Self::split(validators_keys, quorum_size); let edges_groups = Self::split(shuffled_edges, quorums.len()); @@ -749,7 +743,11 @@ pub mod pallet { for (i, quorum) in quorums.iter().enumerate() { let edges_group = &edges_groups[i]; for validator in quorum { - Assignments::::insert(era, Self::string_to_account(validator.clone()), edges_group); + Assignments::::insert( + era, + Self::string_to_account(validator.clone()), + edges_group, + ); } } } From c502f77949b41df515d579de15157232bc39fb79 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 26 Apr 2023 16:01:29 +0600 Subject: [PATCH 126/192] Move DAC interaction code to a separate module --- frame/ddc-validator/src/dac.rs | 338 +++++++++++++++++++++++++++++++++ frame/ddc-validator/src/lib.rs | 335 -------------------------------- 2 files changed, 338 insertions(+), 335 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index b24eb0cc5a467..20f94ac65bec5 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -1 +1,339 @@ //! A module with Data Activity Capture (DAC) interaction. + +use crate::*; +use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +use codec::{Decode, Encode}; +use serde_json::Value; +use sp_staking::EraIndex; +use sp_std::collections::btree_map::BTreeMap; + +#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct RedisFtAggregate { + #[serde(rename = "FT.AGGREGATE")] + pub ft_aggregate: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] +#[serde(untagged)] +pub enum FtAggregate { + Length(u32), + Node(Vec), +} + +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesSent { + node_public_key: String, + era: EraIndex, + sum: u32, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileRequestWrapper { + #[serde(rename = "JSON.GET")] + json: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileRequests { + requests: Requests, +} + +pub type Requests = BTreeMap; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileRequest { + file_request_id: String, + file_info: FileInfo, + bucket_id: i64, + timestamp: i64, + chunks: BTreeMap, + user_public_key: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct Chunk { + log: Log, + cid: String, + ack: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct Ack { + bytes_received: i64, + user_timestamp: i64, + nonce: String, + node_public_key: String, + user_public_key: String, + signature: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct Log { + #[serde(rename = "type")] + log_type: i64, + session_id: String, + user_public_key: String, + era: i64, + user_address: String, + bytes_sent: i64, + timestamp: i64, + node_public_key: String, + signature: String, + bucket_id: i64, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct FileInfo { + #[serde(rename = "chunkCids")] + chunk_cids: Vec, + + #[serde(rename = "requestedChunkCids")] + requested_chunk_cids: Vec, +} + +impl BytesSent { + pub fn new(aggregate: RedisFtAggregate) -> BytesSent { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => + return BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, + sum: node[5].parse::().expect("bytesSentSum must be convertible to u32"), + }, + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec![]; + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesSent { + node_public_key: node[1].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, + sum: node[5] + .parse::() + .expect("bytesSentSum must be convertible to u32"), + }; + + res.push(node); + }, + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res + } +} + +#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] +pub struct BytesReceived { + node_public_key: String, + era: EraIndex, + sum: u32, +} + +impl BytesReceived { + pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { + let data = aggregate.ft_aggregate[1].clone(); + + match data { + FtAggregate::Node(node) => + return BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertible to u32"), + }, + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + pub fn get_all(aggregation: RedisFtAggregate) -> Vec { + let mut res: Vec = vec![]; + for i in 1..aggregation.ft_aggregate.len() { + let data = aggregation.ft_aggregate[i].clone(); + match data { + FtAggregate::Node(node) => { + let node = BytesReceived { + node_public_key: node[1].clone(), + era: node[3].clone().parse::().expect("era must be convertible u32") + as EraIndex, + sum: node[5] + .parse::() + .expect("bytesReceivedSum must be convertible to u32"), + }; + + res.push(node); + }, + FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), + } + } + + return res + } +} + +fn fetch_file_request() -> Requests { + let url = Self::get_file_request_url(); + + let response: FileRequestWrapper = Self::http_get_json(&url).unwrap(); + let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); + let map: Requests = serde_json::from_value(value).unwrap(); + + map +} + +fn get_file_request_url() -> String { + let data_provider_url = Self::get_data_provider_url(); + + let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url.unwrap()); + + res +} + +fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::new(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::new(bytes_received_res); + + (bytes_sent, bytes_received) +} + +fn fetch_data1(era: EraIndex) -> (Vec, Vec) { + info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + // Todo: handle the error + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + let bytes_sent = BytesSent::get_all(bytes_sent_res); + + // Todo: handle the error + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + let bytes_received = BytesReceived::get_all(bytes_received_res); + + (bytes_sent, bytes_received) +} + +fn fetch_data2(era: EraIndex) -> (String, Vec, String, Vec) { + let bytes_sent_query = Self::get_bytes_sent_query_url(era); + let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + let bytes_sent = BytesSent::get_all(bytes_sent_res); + + let bytes_received_query = Self::get_bytes_received_query_url(era); + let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + let bytes_received = BytesReceived::get_all(bytes_received_res); + + (bytes_sent_query, bytes_sent, bytes_received_query, bytes_received) +} + +fn get_bytes_received_query_url(era: EraIndex) -> String { + let data_provider_url = Self::get_data_provider_url(); + + match data_provider_url { + Some(url) => { + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); + }, + None => { + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); + }, + } +} + +fn http_get_json(url: &str) -> ResultStr { + let body = Self::http_get_request(url).map_err(|err| { + error!("[DAC Validator] Error while getting {}: {:?}", url, err); + "HTTP GET error" + })?; + + let parsed = serde_json::from_slice(&body).map_err(|err| { + warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + "HTTP JSON parse error" + }); + + parsed +} + +fn http_get_request(http_url: &str) -> Result, http::Error> { + // info!("[DAC Validator] Sending request to: {:?}", http_url); + + // Initiate an external HTTP GET request. This is using high-level wrappers from + // `sp_runtime`. + let request = http::Request::get(http_url); + + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + // Next we fully read the response body and collect it to a vector of bytes. + Ok(response.body().collect::>()) +} + +fn filter_data( + s: &Vec, + r: &Vec, + a: &T::AccountId, +) -> (BytesSent, BytesReceived) { + let ac = Self::account_to_string(a.clone()); + + let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); + let filtered_r = &*r.into_iter().find(|br| br.node_public_key == ac).unwrap(); + + (filtered_s.clone(), filtered_r.clone()) +} + +fn get_bytes_sent_query_url(era: EraIndex) -> String { + let data_provider_url = Self::get_data_provider_url(); + + match data_provider_url { + Some(url) => { + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); + }, + None => { + return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); + }, + } +} diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6f237ccd81e08..a4f3d565920e0 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -100,198 +100,6 @@ pub struct ValidationDecision { pub totals: DacTotalAggregates, } -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct RedisFtAggregate { - #[serde(rename = "FT.AGGREGATE")] - pub ft_aggregate: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(crate = "alt_serde")] -#[serde(untagged)] -pub enum FtAggregate { - Length(u32), - Node(Vec), -} - -#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] -pub struct BytesSent { - node_public_key: String, - era: EraIndex, - sum: u32, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct FileRequestWrapper { - #[serde(rename = "JSON.GET")] - json: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct FileRequests { - requests: Requests, -} - -pub type Requests = BTreeMap; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct FileRequest { - file_request_id: String, - file_info: FileInfo, - bucket_id: i64, - timestamp: i64, - chunks: BTreeMap, - user_public_key: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct Chunk { - log: Log, - cid: String, - ack: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct Ack { - bytes_received: i64, - user_timestamp: i64, - nonce: String, - node_public_key: String, - user_public_key: String, - signature: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct Log { - #[serde(rename = "type")] - log_type: i64, - session_id: String, - user_public_key: String, - era: i64, - user_address: String, - bytes_sent: i64, - timestamp: i64, - node_public_key: String, - signature: String, - bucket_id: i64, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "alt_serde")] -#[serde(rename_all = "camelCase")] -pub struct FileInfo { - #[serde(rename = "chunkCids")] - chunk_cids: Vec, - - #[serde(rename = "requestedChunkCids")] - requested_chunk_cids: Vec, -} - -impl BytesSent { - pub fn new(aggregate: RedisFtAggregate) -> BytesSent { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => - return BytesSent { - node_public_key: node[1].clone(), - era: node[3].clone().parse::().expect("era must be convertible u32") - as EraIndex, - sum: node[5].parse::().expect("bytesSentSum must be convertible to u32"), - }, - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } - - pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec![]; - for i in 1..aggregation.ft_aggregate.len() { - let data = aggregation.ft_aggregate[i].clone(); - match data { - FtAggregate::Node(node) => { - let node = BytesSent { - node_public_key: node[1].clone(), - era: node[3].clone().parse::().expect("era must be convertible u32") - as EraIndex, - sum: node[5] - .parse::() - .expect("bytesSentSum must be convertible to u32"), - }; - - res.push(node); - }, - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } - - return res - } -} - -#[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] -pub struct BytesReceived { - node_public_key: String, - era: EraIndex, - sum: u32, -} - -impl BytesReceived { - pub fn new(aggregate: RedisFtAggregate) -> BytesReceived { - let data = aggregate.ft_aggregate[1].clone(); - - match data { - FtAggregate::Node(node) => - return BytesReceived { - node_public_key: node[1].clone(), - era: node[3].clone().parse::().expect("era must be convertible u32") - as EraIndex, - sum: node[5] - .parse::() - .expect("bytesReceivedSum must be convertible to u32"), - }, - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } - - pub fn get_all(aggregation: RedisFtAggregate) -> Vec { - let mut res: Vec = vec![]; - for i in 1..aggregation.ft_aggregate.len() { - let data = aggregation.ft_aggregate[i].clone(); - match data { - FtAggregate::Node(node) => { - let node = BytesReceived { - node_public_key: node[1].clone(), - era: node[3].clone().parse::().expect("era must be convertible u32") - as EraIndex, - sum: node[5] - .parse::() - .expect("bytesReceivedSum must be convertible to u32"), - }; - - res.push(node); - }, - FtAggregate::Length(_) => panic!("[DAC Validator] Not a Node"), - } - } - - return res - } -} - pub mod crypto { use super::KEY_TYPE; use frame_system::offchain::AppCrypto; @@ -537,42 +345,6 @@ pub mod pallet { .unwrap() } - fn fetch_file_request() -> Requests { - let url = Self::get_file_request_url(); - - let response: FileRequestWrapper = Self::http_get_json(&url).unwrap(); - let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); - let map: Requests = serde_json::from_value(value).unwrap(); - - map - } - - fn get_file_request_url() -> String { - let data_provider_url = Self::get_data_provider_url(); - - let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url.unwrap()); - - res - } - - fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); - // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); - let bytes_sent = BytesSent::new(bytes_sent_res); - - // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); - let bytes_received = BytesReceived::new(bytes_received_res); - - (bytes_sent, bytes_received) - } - fn account_to_string(account: T::AccountId) -> String { let to32 = T::AccountId::encode(&account); let pub_key_str = array_bytes::bytes2hex("", to32); @@ -588,113 +360,6 @@ pub mod pallet { address } - fn filter_data( - s: &Vec, - r: &Vec, - a: &T::AccountId, - ) -> (BytesSent, BytesReceived) { - let ac = Self::account_to_string(a.clone()); - - let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); - let filtered_r = &*r.into_iter().find(|br| br.node_public_key == ac).unwrap(); - - (filtered_s.clone(), filtered_r.clone()) - } - - fn fetch_data1(era: EraIndex) -> (Vec, Vec) { - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); - // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); - let bytes_sent = BytesSent::get_all(bytes_sent_res); - - // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); - let bytes_received = BytesReceived::get_all(bytes_received_res); - - (bytes_sent, bytes_received) - } - - fn fetch_data2(era: EraIndex) -> (String, Vec, String, Vec) { - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); - let bytes_sent = BytesSent::get_all(bytes_sent_res); - - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = - Self::http_get_json(&bytes_received_query).unwrap(); - let bytes_received = BytesReceived::get_all(bytes_received_res); - - (bytes_sent_query, bytes_sent, bytes_received_query, bytes_received) - } - - fn get_bytes_sent_query_url(era: EraIndex) -> String { - let data_provider_url = Self::get_data_provider_url(); - - match data_provider_url { - Some(url) => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); - }, - None => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); - }, - } - } - - fn get_bytes_received_query_url(era: EraIndex) -> String { - let data_provider_url = Self::get_data_provider_url(); - - match data_provider_url { - Some(url) => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); - }, - None => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); - }, - } - } - - fn http_get_json(url: &str) -> ResultStr { - let body = Self::http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); - "HTTP GET error" - })?; - - let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); - "HTTP JSON parse error" - }); - - parsed - } - - fn http_get_request(http_url: &str) -> Result, http::Error> { - // info!("[DAC Validator] Sending request to: {:?}", http_url); - - // Initiate an external HTTP GET request. This is using high-level wrappers from - // `sp_runtime`. - let request = http::Request::get(http_url); - - let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); - - let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; - - let response = - pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; - - if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); - return Err(http::Error::Unknown) - } - - // Next we fully read the response body and collect it to a vector of bytes. - Ok(response.body().collect::>()) - } - fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); From 5a3149ff329bd8a15831f2f791e6d2135481d553 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 26 Apr 2023 19:12:06 +0600 Subject: [PATCH 127/192] Add DAC Validator utils module --- frame/ddc-validator/src/utils.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 frame/ddc-validator/src/utils.rs diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs new file mode 100644 index 0000000000000..5c8c5cf4f726b --- /dev/null +++ b/frame/ddc-validator/src/utils.rs @@ -0,0 +1,18 @@ +use alloc::string::String; +use codec::{Decode, Encode}; +use sp_core::crypto::AccountId32; + +pub fn account_to_string(account: T::AccountId) -> String { + let to32 = T::AccountId::encode(&account); + let pub_key_str = array_bytes::bytes2hex("", to32); + + pub_key_str +} + +pub fn string_to_account(pub_key_str: String) -> T::AccountId { + let acc32: sp_core::crypto::AccountId32 = + array_bytes::hex2array::<_, 32>(pub_key_str).unwrap().into(); + let mut to32 = AccountId32::as_ref(&acc32); + let address: T::AccountId = T::AccountId::decode(&mut to32).unwrap(); + address +} From 762592c7e3eab4efcadb607500270e36fabbd364 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 26 Apr 2023 19:13:49 +0600 Subject: [PATCH 128/192] Fix dac module usage in ddc-validator crate --- frame/ddc-validator/src/dac.rs | 106 ++++++++++++++++----------------- frame/ddc-validator/src/lib.rs | 53 ++++++----------- 2 files changed, 69 insertions(+), 90 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 20f94ac65bec5..32d1fd7937682 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -1,11 +1,17 @@ //! A module with Data Activity Capture (DAC) interaction. -use crate::*; +use alloc::{format, string::String}; // ToDo: remove String usage use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; use codec::{Decode, Encode}; +use frame_support::log::{error, info, warn}; use serde_json::Value; +use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; -use sp_std::collections::btree_map::BTreeMap; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; + +use crate::utils; + +pub const HTTP_TIMEOUT_MS: u64 = 30_000; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(crate = "alt_serde")] @@ -27,7 +33,7 @@ pub enum FtAggregate { pub struct BytesSent { node_public_key: String, era: EraIndex, - sum: u32, + pub sum: u32, } #[derive(Serialize, Deserialize, Debug)] @@ -153,7 +159,7 @@ impl BytesSent { pub struct BytesReceived { node_public_key: String, era: EraIndex, - sum: u32, + pub sum: u32, } impl BytesReceived { @@ -199,85 +205,82 @@ impl BytesReceived { } } -fn fetch_file_request() -> Requests { - let url = Self::get_file_request_url(); +fn get_file_request_url(data_provider_url: &String) -> String { + let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url); + + res +} - let response: FileRequestWrapper = Self::http_get_json(&url).unwrap(); +pub(crate) fn fetch_file_request(url: &String) -> Requests { + let response: FileRequestWrapper = http_get_json(&url).unwrap(); let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); let map: Requests = serde_json::from_value(value).unwrap(); map } -fn get_file_request_url() -> String { - let data_provider_url = Self::get_data_provider_url(); - - let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url.unwrap()); - - res -} - -fn fetch_data(era: EraIndex, cdn_node: &T::AccountId) -> (BytesSent, BytesReceived) { +pub(crate) fn fetch_data( + data_provider_url: &String, + era: EraIndex, + cdn_node: &T::AccountId, +) -> (BytesSent, BytesReceived) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + let bytes_sent_query = get_bytes_sent_query_url(data_provider_url, era); + let bytes_sent_res: RedisFtAggregate = http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + let bytes_received_query = get_bytes_received_query_url(data_provider_url, era); + let bytes_received_res: RedisFtAggregate = http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); (bytes_sent, bytes_received) } -fn fetch_data1(era: EraIndex) -> (Vec, Vec) { +pub(crate) fn fetch_data1( + data_provider_url: &String, + era: EraIndex, +) -> (Vec, Vec) { info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); + let bytes_sent_query = get_bytes_sent_query_url(data_provider_url, era); + let bytes_sent_res: RedisFtAggregate = http_get_json(&bytes_sent_query).unwrap(); info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::get_all(bytes_sent_res); // Todo: handle the error - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + let bytes_received_query = get_bytes_received_query_url(data_provider_url, era); + let bytes_received_res: RedisFtAggregate = http_get_json(&bytes_received_query).unwrap(); info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::get_all(bytes_received_res); (bytes_sent, bytes_received) } -fn fetch_data2(era: EraIndex) -> (String, Vec, String, Vec) { - let bytes_sent_query = Self::get_bytes_sent_query_url(era); - let bytes_sent_res: RedisFtAggregate = Self::http_get_json(&bytes_sent_query).unwrap(); +pub(crate) fn fetch_data2( + data_provider_url: &String, + era: EraIndex, +) -> (String, Vec, String, Vec) { + let bytes_sent_query = get_bytes_sent_query_url(data_provider_url, era); + let bytes_sent_res: RedisFtAggregate = http_get_json(&bytes_sent_query).unwrap(); let bytes_sent = BytesSent::get_all(bytes_sent_res); - let bytes_received_query = Self::get_bytes_received_query_url(era); - let bytes_received_res: RedisFtAggregate = Self::http_get_json(&bytes_received_query).unwrap(); + let bytes_received_query = get_bytes_received_query_url(data_provider_url, era); + let bytes_received_res: RedisFtAggregate = http_get_json(&bytes_received_query).unwrap(); let bytes_received = BytesReceived::get_all(bytes_received_res); (bytes_sent_query, bytes_sent, bytes_received_query, bytes_received) } -fn get_bytes_received_query_url(era: EraIndex) -> String { - let data_provider_url = Self::get_data_provider_url(); - - match data_provider_url { - Some(url) => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", url, era, era); - }, - None => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", DEFAULT_DATA_PROVIDER_URL, era, era); - }, - } +fn get_bytes_received_query_url(data_provider_url: &String, era: EraIndex) -> String { + format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", data_provider_url, era, era) } -fn http_get_json(url: &str) -> ResultStr { - let body = Self::http_get_request(url).map_err(|err| { +fn http_get_json(url: &str) -> crate::ResultStr { + let body = http_get_request(url).map_err(|err| { error!("[DAC Validator] Error while getting {}: {:?}", url, err); "HTTP GET error" })?; @@ -312,12 +315,12 @@ fn http_get_request(http_url: &str) -> Result, http::Error> { Ok(response.body().collect::>()) } -fn filter_data( +fn filter_data( s: &Vec, r: &Vec, a: &T::AccountId, ) -> (BytesSent, BytesReceived) { - let ac = Self::account_to_string(a.clone()); + let ac = utils::account_to_string::(a.clone()); let filtered_s = &*s.into_iter().find(|bs| bs.node_public_key == ac).unwrap(); let filtered_r = &*r.into_iter().find(|br| br.node_public_key == ac).unwrap(); @@ -325,15 +328,6 @@ fn filter_data( (filtered_s.clone(), filtered_r.clone()) } -fn get_bytes_sent_query_url(era: EraIndex) -> String { - let data_provider_url = Self::get_data_provider_url(); - - match data_provider_url { - Some(url) => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", url, era, era); - }, - None => { - return format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", DEFAULT_DATA_PROVIDER_URL, era, era); - }, - } +fn get_bytes_sent_query_url(data_provider_url: &String, era: EraIndex) -> String { + format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", data_provider_url, era, era) } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a4f3d565920e0..cefd0da33bf0c 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -16,6 +16,7 @@ #![cfg_attr(not(feature = "std"), no_std)] mod dac; +mod utils; mod validation; #[cfg(test)] @@ -66,15 +67,13 @@ type ResultStr = Result; pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); -pub const HTTP_TIMEOUT_MS: u64 = 30_000; - -const TIME_START_MS: u128 = 1_672_531_200_000; -const ERA_DURATION_MS: u128 = 120_000; -const ERA_IN_BLOCKS: u8 = 20; +pub const TIME_START_MS: u128 = 1_672_531_200_000; +pub const ERA_DURATION_MS: u128 = 120_000; +pub const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. -const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; -const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; +pub const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; +pub const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; /// Aggregated values from DAC that describe CDN node's activity during a certain era. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] @@ -238,14 +237,12 @@ pub mod pallet { return } - let file_request = Self::fetch_file_request(); - // info!("fileRequest: {:?}", file_request); + let data_provider_url = + Self::get_data_provider_url().unwrap_or(String::from(DEFAULT_DATA_PROVIDER_URL)); + info!("[DAC Validator] data provider url: {:?}", &data_provider_url,); - let data_provider_url = Self::get_data_provider_url(); - info!( - "[DAC Validator] data provider url: {:?}", - data_provider_url.unwrap_or(String::from("not configured")) - ); + let file_request = dac::fetch_file_request(&data_provider_url); + // info!("fileRequest: {:?}", file_request); // Wait for signal. let signal = Signal::::get().unwrap_or(false); @@ -256,7 +253,8 @@ pub mod pallet { // Read from DAC. let current_era = Self::get_current_era(); - let (sent_query, sent, received_query, received) = Self::fetch_data2(current_era - 1); + let (sent_query, sent, received_query, received) = + dac::fetch_data2(&data_provider_url, current_era - 1); log::info!( "🔎 DAC Validator is fetching data from DAC, current era: {:?}, bytes sent query: {:?}, bytes sent response: {:?}, bytes received query: {:?}, bytes received response: {:?}", current_era, @@ -345,22 +343,7 @@ pub mod pallet { .unwrap() } - fn account_to_string(account: T::AccountId) -> String { - let to32 = T::AccountId::encode(&account); - let pub_key_str = array_bytes::bytes2hex("", to32); - - pub_key_str - } - - fn string_to_account(pub_key_str: String) -> T::AccountId { - let acc32: sp_core::crypto::AccountId32 = - array_bytes::hex2array::<_, 32>(pub_key_str).unwrap().into(); - let mut to32 = AccountId32::as_ref(&acc32); - let address: T::AccountId = T::AccountId::decode(&mut to32).unwrap(); - address - } - - fn validate(bytes_sent: BytesSent, bytes_received: BytesReceived) -> bool { + fn validate(bytes_sent: dac::BytesSent, bytes_received: dac::BytesReceived) -> bool { let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); return if percentage_difference > 0.0 && @@ -397,8 +380,10 @@ pub mod pallet { let shuffled_validators = Self::shuffle(validators); let shuffled_edges = Self::shuffle(edges); - let validators_keys: Vec = - shuffled_validators.iter().map(|v| Self::account_to_string(v.clone())).collect(); + let validators_keys: Vec = shuffled_validators + .iter() + .map(|v| utils::account_to_string::(v.clone())) + .collect(); let quorums = Self::split(validators_keys, quorum_size); let edges_groups = Self::split(shuffled_edges, quorums.len()); @@ -410,7 +395,7 @@ pub mod pallet { for validator in quorum { Assignments::::insert( era, - Self::string_to_account(validator.clone()), + utils::string_to_account::(validator.clone()), edges_group, ); } From 6f300cc502a85d1b0afb792bc02a6e8a8a1ffd6d Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 27 Apr 2023 19:09:16 +0600 Subject: [PATCH 129/192] Use log crate directly --- frame/ddc-validator/src/dac.rs | 21 ++++++++++----------- frame/ddc-validator/src/lib.rs | 11 +++++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 32d1fd7937682..a6e450219f6dc 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -3,7 +3,6 @@ use alloc::{format, string::String}; // ToDo: remove String usage use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; use codec::{Decode, Encode}; -use frame_support::log::{error, info, warn}; use serde_json::Value; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; @@ -224,17 +223,17 @@ pub(crate) fn fetch_data( era: EraIndex, cdn_node: &T::AccountId, ) -> (BytesSent, BytesReceived) { - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + log::info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = get_bytes_sent_query_url(data_provider_url, era); let bytes_sent_res: RedisFtAggregate = http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + log::info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::new(bytes_sent_res); // Todo: handle the error let bytes_received_query = get_bytes_received_query_url(data_provider_url, era); let bytes_received_res: RedisFtAggregate = http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + log::info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::new(bytes_received_res); (bytes_sent, bytes_received) @@ -244,17 +243,17 @@ pub(crate) fn fetch_data1( data_provider_url: &String, era: EraIndex, ) -> (Vec, Vec) { - info!("[DAC Validator] DAC Validator is running. Current era is {}", era); + log::info!("[DAC Validator] DAC Validator is running. Current era is {}", era); // Todo: handle the error let bytes_sent_query = get_bytes_sent_query_url(data_provider_url, era); let bytes_sent_res: RedisFtAggregate = http_get_json(&bytes_sent_query).unwrap(); - info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); + log::info!("[DAC Validator] Bytes sent sum is fetched: {:?}", bytes_sent_res); let bytes_sent = BytesSent::get_all(bytes_sent_res); // Todo: handle the error let bytes_received_query = get_bytes_received_query_url(data_provider_url, era); let bytes_received_res: RedisFtAggregate = http_get_json(&bytes_received_query).unwrap(); - info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); + log::info!("[DAC Validator] Bytes received sum is fetched:: {:?}", bytes_received_res); let bytes_received = BytesReceived::get_all(bytes_received_res); (bytes_sent, bytes_received) @@ -281,12 +280,12 @@ fn get_bytes_received_query_url(data_provider_url: &String, era: EraIndex) -> St fn http_get_json(url: &str) -> crate::ResultStr { let body = http_get_request(url).map_err(|err| { - error!("[DAC Validator] Error while getting {}: {:?}", url, err); + log::error!("[DAC Validator] Error while getting {}: {:?}", url, err); "HTTP GET error" })?; let parsed = serde_json::from_slice(&body).map_err(|err| { - warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); + log::warn!("[DAC Validator] Error while parsing JSON from {}: {:?}", url, err); "HTTP JSON parse error" }); @@ -294,7 +293,7 @@ fn http_get_json(url: &str) -> crate::ResultStr { } fn http_get_request(http_url: &str) -> Result, http::Error> { - // info!("[DAC Validator] Sending request to: {:?}", http_url); + // log::info!("[DAC Validator] Sending request to: {:?}", http_url); // Initiate an external HTTP GET request. This is using high-level wrappers from // `sp_runtime`. @@ -307,7 +306,7 @@ fn http_get_request(http_url: &str) -> Result, http::Error> { let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; if response.code != 200 { - warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); + log::warn!("[DAC Validator] http_get_request unexpected status code: {}", response.code); return Err(http::Error::Unknown) } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index cefd0da33bf0c..78fd8ffe9349d 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -32,7 +32,6 @@ pub use core::fmt::Debug; pub use frame_support::{ decl_event, decl_module, decl_storage, dispatch::DispatchResult, - log::{error, info, warn}, pallet_prelude::*, parameter_types, storage, traits::{Currency, Randomness, UnixTime}, @@ -216,10 +215,10 @@ pub mod pallet { } let era = Self::get_current_era(); - info!("current era: {:?}", era); + log::info!("current era: {:?}", era); if let Some(last_managed_era) = >::get() { - info!("last_managed_era: {:?}", last_managed_era); + log::info!("last_managed_era: {:?}", last_managed_era); if last_managed_era >= era { return 0 } @@ -239,10 +238,10 @@ pub mod pallet { let data_provider_url = Self::get_data_provider_url().unwrap_or(String::from(DEFAULT_DATA_PROVIDER_URL)); - info!("[DAC Validator] data provider url: {:?}", &data_provider_url,); + log::info!("[DAC Validator] data provider url: {:?}", &data_provider_url); let file_request = dac::fetch_file_request(&data_provider_url); - // info!("fileRequest: {:?}", file_request); + // log::info!("fileRequest: {:?}", file_request); // Wait for signal. let signal = Signal::::get().unwrap_or(false); @@ -320,7 +319,7 @@ pub mod pallet { None => { let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}. Using default for now.", url_key); - warn!("{}", msg); + log::warn!("{}", msg); Some(String::from(DEFAULT_DATA_PROVIDER_URL)) }, Some(url) => Some(String::from_utf8(url).unwrap()), From 368b91095524454466ebeb74a0f84b6bb685fab7 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 27 Apr 2023 15:16:36 +0200 Subject: [PATCH 130/192] Implement bytes aggregation --- frame/ddc-validator/src/dac.rs | 98 ++++++++++++++++++++++++++++++---- frame/ddc-validator/src/lib.rs | 8 ++- 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index a6e450219f6dc..b3bfc8af2e8c8 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -6,10 +6,18 @@ use codec::{Decode, Encode}; use serde_json::Value; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; -use sp_std::{collections::btree_map::BTreeMap, prelude::*}; +pub use sp_std::{ + collections::{ + btree_map::BTreeMap, + btree_set::BTreeSet, + }, + prelude::* +}; + use crate::utils; +pub type TimestampInSec = u64; pub const HTTP_TIMEOUT_MS: u64 = 30_000; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -58,8 +66,8 @@ pub type Requests = BTreeMap; pub struct FileRequest { file_request_id: String, file_info: FileInfo, - bucket_id: i64, - timestamp: i64, + bucket_id: u64, + timestamp: u64, chunks: BTreeMap, user_public_key: String, } @@ -77,8 +85,8 @@ pub struct Chunk { #[serde(crate = "alt_serde")] #[serde(rename_all = "camelCase")] pub struct Ack { - bytes_received: i64, - user_timestamp: i64, + bytes_received: u64, + user_timestamp: u64, nonce: String, node_public_key: String, user_public_key: String, @@ -90,16 +98,16 @@ pub struct Ack { #[serde(rename_all = "camelCase")] pub struct Log { #[serde(rename = "type")] - log_type: i64, + log_type: u64, session_id: String, user_public_key: String, - era: i64, + era: u64, user_address: String, - bytes_sent: i64, - timestamp: i64, + bytes_sent: u64, + timestamp: u64, node_public_key: String, signature: String, - bucket_id: i64, + bucket_id: u64, } #[derive(Serialize, Deserialize, Debug)] @@ -204,6 +212,76 @@ impl BytesReceived { } } +fn get_timestamps_with_ack(file_requests: &FileRequests) -> Vec { + let mut timestamps:Vec = Vec::new(); + + for (_, file_request) in &file_requests.requests { + for (_, chunk) in &file_request.chunks { + if let Some(ack) = &chunk.ack { + timestamps.push(chunk.log.timestamp); + } + } + } + + timestamps.sort(); + + timestamps +} + +pub fn get_proved_deliveried_bytes_sum(file_requests: &FileRequests) -> u64 { + let ack_timestamps= get_timestamps_with_ack(file_requests); + let mut total_bytes_received = 0u64; + + for (_, file_request) in &file_requests.requests { + for (_, chunk) in &file_request.chunks { + if let Some(ack) = &chunk.ack { + total_bytes_received += ack.bytes_received; + } else { + total_bytes_received += get_proved_delivered_bytes(chunk, &ack_timestamps); + } + } + } + + total_bytes_received +} + +fn get_proved_delivered_bytes(chunk: &Chunk, ack_timestamps: &Vec) -> u64 { + let log_timestamp = chunk.log.timestamp; + let neighbors = get_closer_neighbors(log_timestamp, &ack_timestamps); + let is_proved = is_lies_within_threshold(log_timestamp, neighbors, 42); + + if is_proved { + return chunk.log.bytes_sent; + } else { + 0 + } +} + +fn get_closer_neighbors(timestamp: TimestampInSec, timestamps: &Vec) -> (TimestampInSec, TimestampInSec) { + let mut before = 0; + let mut after = TimestampInSec::MAX; + for ts in timestamps { + if ts < ×tamp { + before = before.max(*ts); + } else if ts > ×tamp { + after = after.min(*ts); + } + } + + (before, after) +} + +fn is_lies_within_threshold(timestamp: TimestampInSec, borders: (TimestampInSec, TimestampInSec), threshold: TimestampInSec) -> bool { + let left_distance = timestamp - borders.0; + let right_distance = borders.1 - timestamp; + + if left_distance < threshold || right_distance < threshold { + return true; + } + + false +} + fn get_file_request_url(data_provider_url: &String) -> String { let res = format!("{}/JSON.GET/testddc:dac:data", data_provider_url); diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 78fd8ffe9349d..b075b5ecaa7e1 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -53,7 +53,12 @@ pub use sp_core::crypto::{AccountId32, KeyTypeId, UncheckedFrom}; pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timestamp}; pub use sp_staking::EraIndex; -pub use sp_std::{collections::btree_map::BTreeMap, prelude::*}; +pub use sp_std::{ + collections::{ + btree_map::BTreeMap, + }, + prelude::* +}; extern crate alloc; @@ -241,7 +246,6 @@ pub mod pallet { log::info!("[DAC Validator] data provider url: {:?}", &data_provider_url); let file_request = dac::fetch_file_request(&data_provider_url); - // log::info!("fileRequest: {:?}", file_request); // Wait for signal. let signal = Signal::::get().unwrap_or(false); From 8d785c075fe3bdef3e572d31f56e00e99ff262fb Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 27 Apr 2023 20:02:17 +0200 Subject: [PATCH 131/192] Clean up code --- frame/ddc-validator/src/dac.rs | 8 ++++---- frame/ddc-validator/src/lib.rs | 34 ++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index b3bfc8af2e8c8..dc160db84a0a0 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -212,10 +212,10 @@ impl BytesReceived { } } -fn get_timestamps_with_ack(file_requests: &FileRequests) -> Vec { +fn get_timestamps_with_ack(file_requests: &Requests) -> Vec { let mut timestamps:Vec = Vec::new(); - for (_, file_request) in &file_requests.requests { + for (_, file_request) in file_requests { for (_, chunk) in &file_request.chunks { if let Some(ack) = &chunk.ack { timestamps.push(chunk.log.timestamp); @@ -228,11 +228,11 @@ fn get_timestamps_with_ack(file_requests: &FileRequests) -> Vec timestamps } -pub fn get_proved_deliveried_bytes_sum(file_requests: &FileRequests) -> u64 { +pub fn get_proved_deliveried_bytes_sum(file_requests: &Requests) -> u64 { let ack_timestamps= get_timestamps_with_ack(file_requests); let mut total_bytes_received = 0u64; - for (_, file_request) in &file_requests.requests { + for (_, file_request) in file_requests { for (_, chunk) in &file_request.chunks { if let Some(ack) = &chunk.ack { total_bytes_received += ack.bytes_received; diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b075b5ecaa7e1..6848ad3e1f41b 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -76,6 +76,7 @@ pub const ERA_DURATION_MS: u128 = 120_000; pub const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. +// pub const DEFAULT_DATA_PROVIDER_URL: &str = "https://dev-dac-redis.network-dev.aws.cere.io"; pub const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; pub const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; @@ -201,6 +202,11 @@ pub mod pallet { #[pallet::getter(fn last_managed_era)] pub type LastManagedEra = StorageValue<_, EraIndex>; + #[pallet::error] + pub enum Error { + // TBA + } + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event @@ -241,11 +247,14 @@ pub mod pallet { return } - let data_provider_url = - Self::get_data_provider_url().unwrap_or(String::from(DEFAULT_DATA_PROVIDER_URL)); - log::info!("[DAC Validator] data provider url: {:?}", &data_provider_url); + let data_provider_url = Self::get_data_provider_url(); + info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); + + let mock_data_url = Self::get_mock_data_url(); - let file_request = dac::fetch_file_request(&data_provider_url); + let file_request = dac::fetch_file_request(&mock_data_url); + let bytes_sum = dac::get_proved_deliveried_bytes_sum(&file_request); + info!("Proved bytes sum: {:?}", bytes_sum); // Wait for signal. let signal = Signal::::get().unwrap_or(false); @@ -313,23 +322,28 @@ pub mod pallet { ::AccountId: AsRef<[u8]> + UncheckedFrom, as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { - fn get_data_provider_url() -> Option { + fn get_data_provider_url() -> String { let url_ref = sp_io::offchain::local_storage_get( sp_core::offchain::StorageKind::PERSISTENT, DATA_PROVIDER_URL_KEY, ); match url_ref { + Some(url) => String::from_utf8(url).expect("Data provider URL should be valid UTF-8 string"), None => { - let url_key = String::from_utf8(DATA_PROVIDER_URL_KEY.to_vec()).unwrap(); - let msg = format!("[DAC Validator] Data provider URL is not configured. Please configure it using offchain_localStorageSet with key {:?}. Using default for now.", url_key); - log::warn!("{}", msg); - Some(String::from(DEFAULT_DATA_PROVIDER_URL)) + String::from(DEFAULT_DATA_PROVIDER_URL) }, - Some(url) => Some(String::from_utf8(url).unwrap()), } } + fn get_mock_data_url() -> String { + let data_url = Self::get_data_provider_url(); + let mock_url = "/JSON.GET/testddc:dac:data/INDENT/0"; + let url = format!("{}{}", data_url, mock_url); + + url + } + fn get_signer() -> ResultStr> { let signer = Signer::<_, _>::any_account(); if !signer.can_sign() { From 057fa5341829cd3f3a2974694c801081d58a9fc5 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 28 Apr 2023 13:18:40 +0200 Subject: [PATCH 132/192] Fix mock data url --- frame/ddc-validator/src/dac.rs | 4 ++-- frame/ddc-validator/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index dc160db84a0a0..11c8c8802716b 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -228,14 +228,14 @@ fn get_timestamps_with_ack(file_requests: &Requests) -> Vec { timestamps } -pub fn get_proved_deliveried_bytes_sum(file_requests: &Requests) -> u64 { +pub fn get_proved_delivered_bytes_sum(file_requests: &Requests) -> u64 { let ack_timestamps= get_timestamps_with_ack(file_requests); let mut total_bytes_received = 0u64; for (_, file_request) in file_requests { for (_, chunk) in &file_request.chunks { if let Some(ack) = &chunk.ack { - total_bytes_received += ack.bytes_received; + total_bytes_received += &chunk.log.bytes_sent; } else { total_bytes_received += get_proved_delivered_bytes(chunk, &ack_timestamps); } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6848ad3e1f41b..51aa4ac8893f4 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -253,7 +253,7 @@ pub mod pallet { let mock_data_url = Self::get_mock_data_url(); let file_request = dac::fetch_file_request(&mock_data_url); - let bytes_sum = dac::get_proved_deliveried_bytes_sum(&file_request); + let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); info!("Proved bytes sum: {:?}", bytes_sum); // Wait for signal. @@ -338,7 +338,7 @@ pub mod pallet { fn get_mock_data_url() -> String { let data_url = Self::get_data_provider_url(); - let mock_url = "/JSON.GET/testddc:dac:data/INDENT/0"; + let mock_url = "/JSON.GET/testddc:dac:data"; let url = format!("{}{}", data_url, mock_url); url From b68e32f74a69b41b2195ccc8d65e9fb61e7b48f4 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 28 Apr 2023 13:29:03 +0200 Subject: [PATCH 133/192] Fix info macro --- frame/ddc-validator/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 51aa4ac8893f4..40d52dcd85a75 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -248,13 +248,13 @@ pub mod pallet { } let data_provider_url = Self::get_data_provider_url(); - info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); + log::info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); let mock_data_url = Self::get_mock_data_url(); let file_request = dac::fetch_file_request(&mock_data_url); let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); - info!("Proved bytes sum: {:?}", bytes_sum); + log::info!("Proved bytes sum: {:?}", bytes_sum); // Wait for signal. let signal = Signal::::get().unwrap_or(false); From 1b66843dac359d4d7944f460703634389f894363 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 28 Apr 2023 19:48:28 +0600 Subject: [PATCH 134/192] DAC aggregates reading function --- Cargo.lock | 1 + frame/ddc-validator/Cargo.toml | 2 ++ frame/ddc-validator/src/dac.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 198afca1784e3..f9980013ae4e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5869,6 +5869,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", + "lite-json", "log 0.4.17", "pallet-balances", "pallet-contracts", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index ff6dbd53b3fd6..0657d768f205a 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -11,6 +11,7 @@ frame-support = { version = "4.0.0-dev", default-features = false, path = "../su frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } log = { version = "0.4.17", default-features = false } +lite-json = { version = "0.2.0", default-features = false } pallet-contracts = { version = '4.0.0-dev', default-features = false, path = "../contracts" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../ddc-staking" } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } @@ -32,6 +33,7 @@ std = [ "frame-support/std", "frame-system/std", "frame-election-provider-support/std", + "lite-json/std", "pallet-contracts/std", "pallet-ddc-staking/std", "pallet-session/std", diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 11c8c8802716b..2ab312b148c18 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -3,6 +3,7 @@ use alloc::{format, string::String}; // ToDo: remove String usage use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; use codec::{Decode, Encode}; +use lite_json::json::JsonValue; use serde_json::Value; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; @@ -408,3 +409,29 @@ fn filter_data( fn get_bytes_sent_query_url(data_provider_url: &String, era: EraIndex) -> String { format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesSent/AS/bytesSentSum", data_provider_url, era, era) } + +pub(crate) fn fetch_aggregates( + data_provider_url: &String, + era: EraIndex, +) -> Result { + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + let url = + format!("{}/JSON.GET/ddc:dac:aggregation:nodes:{}?type=query", data_provider_url, era); + let request = http::Request::get(url.as_str()); + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + if response.code != 200 { + log::warn!("Unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + let body = response.body().collect::>(); + let body_str = sp_std::str::from_utf8(&body).map_err(|_| { + log::warn!("No UTF-8 body"); + http::Error::Unknown + })?; + let json = lite_json::parse_json(body_str).map_err(|_| { + log::warn!("No JSON body"); + http::Error::Unknown + })?; + Ok(json) +} From 3102c848853d3df18adab125ac9fdcec3f2b9f58 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 28 Apr 2023 19:50:11 +0600 Subject: [PATCH 135/192] Print the number of broken sessions by CDN node --- frame/ddc-validator/src/lib.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 40d52dcd85a75..0661efb66ada0 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -43,6 +43,7 @@ pub use frame_system::{ offchain::{AppCrypto, CreateSignedTransaction, SendSignedTransaction, Signer, SigningTypes}, pallet_prelude::*, }; +pub use lite_json::json::JsonValue; pub use pallet::*; pub use pallet_ddc_staking::{self as ddc_staking}; pub use pallet_session as session; @@ -256,6 +257,34 @@ pub mod pallet { let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); log::info!("Proved bytes sum: {:?}", bytes_sum); + // Print the number of broken sessions per CDN node. + let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data + let aggregates_obj = aggregates_value.as_object().unwrap(); + aggregates_obj + .into_iter() + .for_each(|(cdn_node_pubkey, cdn_node_aggregates_value)| { + // iterate over aggregates for each node + let cdn_node_aggregates_obj = cdn_node_aggregates_value.as_object().unwrap(); + // Extract `nodeInterruptedSessions` field + let (_, cdn_node_interrupted_sessions_value) = cdn_node_aggregates_obj + .into_iter() + .find(|(key, _)| key.iter().copied().eq("nodeInterruptedSessions".chars())) + .unwrap(); + let cdn_node_interrupted_sessions_obj = + cdn_node_interrupted_sessions_value.as_object().unwrap(); + // Prepare CDN pubkey without heap allocated string + let cdn_node_pubkey_vecu8: Vec = + cdn_node_pubkey.iter().map(|c| *c as u8).collect(); + let cdn_node_pubkey_str = + sp_std::str::from_utf8(&cdn_node_pubkey_vecu8).unwrap(); + log::info!( + "Broken sessions per CDN node | Node {}: {} sessions broken", + cdn_node_pubkey_str, + cdn_node_interrupted_sessions_obj.len(), /* count sessions broken by the + * node */ + ); + }); + // Wait for signal. let signal = Signal::::get().unwrap_or(false); if !signal { From 7b1fdfe56c8194a64ef987f3e94d8698f01672f4 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 28 Apr 2023 19:51:25 +0600 Subject: [PATCH 136/192] Autoformat DAC Validator files --- frame/ddc-validator/Cargo.toml | 2 +- frame/ddc-validator/src/dac.rs | 27 +++++++++++++++------------ frame/ddc-validator/src/lib.rs | 14 ++++---------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 0657d768f205a..66b8deec881c3 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -52,4 +52,4 @@ std = [ pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } -pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } \ No newline at end of file +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 2ab312b148c18..9f0c0f5ac8cd5 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -8,14 +8,10 @@ use serde_json::Value; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; pub use sp_std::{ - collections::{ - btree_map::BTreeMap, - btree_set::BTreeSet, - }, - prelude::* + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + prelude::*, }; - use crate::utils; pub type TimestampInSec = u64; @@ -214,7 +210,7 @@ impl BytesReceived { } fn get_timestamps_with_ack(file_requests: &Requests) -> Vec { - let mut timestamps:Vec = Vec::new(); + let mut timestamps: Vec = Vec::new(); for (_, file_request) in file_requests { for (_, chunk) in &file_request.chunks { @@ -230,7 +226,7 @@ fn get_timestamps_with_ack(file_requests: &Requests) -> Vec { } pub fn get_proved_delivered_bytes_sum(file_requests: &Requests) -> u64 { - let ack_timestamps= get_timestamps_with_ack(file_requests); + let ack_timestamps = get_timestamps_with_ack(file_requests); let mut total_bytes_received = 0u64; for (_, file_request) in file_requests { @@ -252,13 +248,16 @@ fn get_proved_delivered_bytes(chunk: &Chunk, ack_timestamps: &Vec) -> (TimestampInSec, TimestampInSec) { +fn get_closer_neighbors( + timestamp: TimestampInSec, + timestamps: &Vec, +) -> (TimestampInSec, TimestampInSec) { let mut before = 0; let mut after = TimestampInSec::MAX; for ts in timestamps { @@ -272,12 +271,16 @@ fn get_closer_neighbors(timestamp: TimestampInSec, timestamps: &Vec bool { +fn is_lies_within_threshold( + timestamp: TimestampInSec, + borders: (TimestampInSec, TimestampInSec), + threshold: TimestampInSec, +) -> bool { let left_distance = timestamp - borders.0; let right_distance = borders.1 - timestamp; if left_distance < threshold || right_distance < threshold { - return true; + return true } false diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0661efb66ada0..0aeaf7778869d 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -54,12 +54,7 @@ pub use sp_core::crypto::{AccountId32, KeyTypeId, UncheckedFrom}; pub use sp_io::crypto::sr25519_public_keys; pub use sp_runtime::offchain::{http, storage::StorageValueRef, Duration, Timestamp}; pub use sp_staking::EraIndex; -pub use sp_std::{ - collections::{ - btree_map::BTreeMap, - }, - prelude::* -}; +pub use sp_std::{collections::btree_map::BTreeMap, prelude::*}; extern crate alloc; @@ -358,10 +353,9 @@ pub mod pallet { ); match url_ref { - Some(url) => String::from_utf8(url).expect("Data provider URL should be valid UTF-8 string"), - None => { - String::from(DEFAULT_DATA_PROVIDER_URL) - }, + Some(url) => + String::from_utf8(url).expect("Data provider URL should be valid UTF-8 string"), + None => String::from(DEFAULT_DATA_PROVIDER_URL), } } From 2c5b7ac3e796bfad2011eb5d39514d5c4e8bf239 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 28 Apr 2023 16:45:59 +0200 Subject: [PATCH 137/192] Add const for threshold configuration --- frame/ddc-validator/src/dac.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 9f0c0f5ac8cd5..52497eed5c3b1 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -16,6 +16,7 @@ use crate::utils; pub type TimestampInSec = u64; pub const HTTP_TIMEOUT_MS: u64 = 30_000; +pub const FAILED_CONTENT_CONSUMER_THRESHOLD: TimestampInSec = 100; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(crate = "alt_serde")] @@ -245,7 +246,7 @@ pub fn get_proved_delivered_bytes_sum(file_requests: &Requests) -> u64 { fn get_proved_delivered_bytes(chunk: &Chunk, ack_timestamps: &Vec) -> u64 { let log_timestamp = chunk.log.timestamp; let neighbors = get_closer_neighbors(log_timestamp, &ack_timestamps); - let is_proved = is_lies_within_threshold(log_timestamp, neighbors, 42); + let is_proved = is_lies_within_threshold(log_timestamp, neighbors, FAILED_CONTENT_CONSUMER_THRESHOLD); if is_proved { return chunk.log.bytes_sent From a6ef9af1d43bdc41f13811584168f69714dea93a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 3 May 2023 13:52:53 +0600 Subject: [PATCH 138/192] DAC Validators' "shared memory" module --- frame/ddc-validator/src/lib.rs | 1 + frame/ddc-validator/src/shm.rs | 1 + 2 files changed, 2 insertions(+) create mode 100644 frame/ddc-validator/src/shm.rs diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 0aeaf7778869d..e0c481342df54 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -16,6 +16,7 @@ #![cfg_attr(not(feature = "std"), no_std)] mod dac; +mod shm; mod utils; mod validation; diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs new file mode 100644 index 0000000000000..baad0fa83b6b0 --- /dev/null +++ b/frame/ddc-validator/src/shm.rs @@ -0,0 +1 @@ +//! Validators' "shared memory" module. From adfcae9db4675a039ddbdab46fdd0b624d3c27c3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 4 May 2023 15:27:37 +0600 Subject: [PATCH 139/192] Borrow DAC data for validation, remove ownership --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index e0c481342df54..11a082deb5f79 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -384,7 +384,7 @@ pub mod pallet { .unwrap() } - fn validate(bytes_sent: dac::BytesSent, bytes_received: dac::BytesReceived) -> bool { + fn validate(bytes_sent: &dac::BytesSent, bytes_received: &dac::BytesReceived) -> bool { let percentage_difference = 1f32 - (bytes_received.sum as f32 / bytes_sent.sum as f32); return if percentage_difference > 0.0 && From 8626282ce4fae03c03c97ac38ca86b83637b3ac3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 4 May 2023 17:53:48 +0600 Subject: [PATCH 140/192] Make all DAC data types fields public --- frame/ddc-validator/src/dac.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 52497eed5c3b1..2bf6cde8578d6 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -36,8 +36,8 @@ pub enum FtAggregate { #[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] pub struct BytesSent { - node_public_key: String, - era: EraIndex, + pub node_public_key: String, + pub era: EraIndex, pub sum: u32, } @@ -162,8 +162,8 @@ impl BytesSent { #[derive(Clone, Debug, Encode, Decode, scale_info::TypeInfo, PartialEq)] pub struct BytesReceived { - node_public_key: String, - era: EraIndex, + pub node_public_key: String, + pub era: EraIndex, pub sum: u32, } From e77bb4dc8b71fdf7b03ad0e63c988eb6dd2490bc Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 4 May 2023 17:56:38 +0600 Subject: [PATCH 141/192] Create intermediate validation decisions --- frame/ddc-validator/src/lib.rs | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 11a082deb5f79..a6381e40127d0 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -300,6 +300,45 @@ pub mod pallet { received_query, received, ); + + // Create intermediate validation decisions + // ======================================== + + // All validators validate all CDN nodes. + let edges: Vec = >::iter_keys().collect(); + for edge in edges.iter() { + // Get string type CDN node pubkey + let edge_pubkey: String = utils::account_to_string::(edge.clone()); + + // Get bytes sent and received for the CDN node + let Some(node_sent) = sent + .iter() + .find(|bytes_sent| bytes_sent.node_public_key == edge_pubkey) else { + log::warn!("No logs to validate {:?}", edge); + continue + }; + let Some(client_received) = received + .iter() + .find(|bytes_received| bytes_received.node_public_key == edge_pubkey) else { + log::warn!("No acks to validate {:?}", edge); + continue + }; + + // Proof-of-delivery validation + let validation_result = Self::validate(node_sent, client_received); + + // Prepare an intermediate validation decision + let _validation_decision = ValidationDecision { + result: validation_result, + payload: [0u8; 256], // ToDo: put a hash of the validated data here + totals: DacTotalAggregates { + sent: node_sent.sum as u64, + received: client_received.sum as u64, + failed_by_client: 0, // ToDo + failure_rate: 0, // ToDo + }, + }; + } } } From 09c27d91f49fc5d47d6352b8581a06c930e5d205 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 4 May 2023 18:54:53 +0600 Subject: [PATCH 142/192] Remove let...else unstable feature usage --- frame/ddc-validator/src/lib.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a6381e40127d0..391804e9eb964 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -311,18 +311,26 @@ pub mod pallet { let edge_pubkey: String = utils::account_to_string::(edge.clone()); // Get bytes sent and received for the CDN node - let Some(node_sent) = sent + let node_sent: &dac::BytesSent = match sent .iter() - .find(|bytes_sent| bytes_sent.node_public_key == edge_pubkey) else { + .find(|bytes_sent| bytes_sent.node_public_key == edge_pubkey) + { + Some(node_sent) => node_sent, + None => { log::warn!("No logs to validate {:?}", edge); continue - }; - let Some(client_received) = received + }, + }; + let client_received: &dac::BytesReceived = match received .iter() - .find(|bytes_received| bytes_received.node_public_key == edge_pubkey) else { + .find(|bytes_received| bytes_received.node_public_key == edge_pubkey) + { + Some(client_received) => client_received, + None => { log::warn!("No acks to validate {:?}", edge); continue - }; + }, + }; // Proof-of-delivery validation let validation_result = Self::validate(node_sent, client_received); From 5e7d85d23c4811e8ac11867aad5890717015a1c7 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 5 May 2023 18:34:18 +0600 Subject: [PATCH 143/192] Base64 encoding func for validators' shared memory --- Cargo.lock | 7 +++++++ frame/ddc-validator/Cargo.toml | 1 + frame/ddc-validator/src/shm.rs | 11 +++++++++++ 3 files changed, 19 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f9980013ae4e0..3787bef87b1a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,6 +491,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "base64ct" version = "1.3.3" @@ -5866,6 +5872,7 @@ version = "0.1.0" dependencies = [ "alt_serde", "array-bytes", + "base64 0.21.0", "frame-election-provider-support", "frame-support", "frame-system", diff --git a/frame/ddc-validator/Cargo.toml b/frame/ddc-validator/Cargo.toml index 66b8deec881c3..7e2d7f563281a 100644 --- a/frame/ddc-validator/Cargo.toml +++ b/frame/ddc-validator/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] array-bytes = "6.0.0" alt_serde = { version = "1", default-features = false, features = ["derive"] } +base64 = { version = "0.21.0", default-features = false } codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index baad0fa83b6b0..ab8698d0d8f48 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -1 +1,12 @@ //! Validators' "shared memory" module. + +use base64::prelude::*; +use sp_std::prelude::*; + +/// Encodes a vector of bytes into a vector of characters using base64 encoding. +pub fn base64_encode(input: &Vec) -> Vec { + let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity + buf.resize(1024, 0); + BASE64_STANDARD.encode_slice(input, &mut buf).unwrap(); // ToDo: handle error + buf.iter().map(|&byte| byte as char).collect() +} From 3087509422eec3298aab71df3954ad3910790096 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 5 May 2023 19:51:00 +0600 Subject: [PATCH 144/192] A func to publish intermediate validation result --- frame/ddc-validator/src/shm.rs | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index ab8698d0d8f48..6ae033e77ffd3 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -1,8 +1,14 @@ //! Validators' "shared memory" module. +use alloc::{format, string::String}; // ToDo: remove String usage use base64::prelude::*; +use lite_json::json::JsonValue; +use sp_runtime::offchain::{http, Duration}; +use sp_staking::EraIndex; use sp_std::prelude::*; +const HTTP_TIMEOUT_MS: u64 = 30_000; + /// Encodes a vector of bytes into a vector of characters using base64 encoding. pub fn base64_encode(input: &Vec) -> Vec { let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity @@ -10,3 +16,43 @@ pub fn base64_encode(input: &Vec) -> Vec { BASE64_STANDARD.encode_slice(input, &mut buf).unwrap(); // ToDo: handle error buf.iter().map(|&byte| byte as char).collect() } + +/// Publish intermediate validation result to redis. +pub fn share_intermediate_validation_result( + shared_memory_webdis_url: &String, + era: EraIndex, + validator: &String, + cdn_node: &String, + validation_result: bool, + validation_decision_encoded: &String, +) -> Result { + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + let validation_result_string = String::from(if validation_result { "true" } else { "false" }); + let validation_decision_string = String::from(validation_decision_encoded); + let url = format!( + "{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{{\"result\":{},\"data\":{}}}", + shared_memory_webdis_url, + validator, + cdn_node, + era, + validation_result_string, + validation_decision_string, + ); + let request = http::Request::get(url.as_str()); + let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + if response.code != 200 { + log::warn!("Unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + let body = response.body().collect::>(); + let body_str = sp_std::str::from_utf8(&body).map_err(|_| { + log::warn!("No UTF-8 body"); + http::Error::Unknown + })?; + let json = lite_json::parse_json(body_str).map_err(|_| { + log::warn!("No JSON body"); + http::Error::Unknown + })?; + Ok(json) +} From c4c5cccf604557c93f7d587c7082a43e30f910e3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 5 May 2023 20:29:07 +0600 Subject: [PATCH 145/192] Reset validators launch signal each block --- frame/ddc-validator/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 391804e9eb964..406250e977fab 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -222,6 +222,8 @@ pub mod pallet { return 0 } + Signal::::set(Some(false)); + let era = Self::get_current_era(); log::info!("current era: {:?}", era); From e57f82c79d2a77a567db42e8917a10acd49524bf Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 5 May 2023 20:29:38 +0600 Subject: [PATCH 146/192] Publish intermediate validation results to shm --- frame/ddc-validator/src/lib.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 406250e977fab..4f4657115d0fa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -338,7 +338,7 @@ pub mod pallet { let validation_result = Self::validate(node_sent, client_received); // Prepare an intermediate validation decision - let _validation_decision = ValidationDecision { + let validation_decision = ValidationDecision { result: validation_result, payload: [0u8; 256], // ToDo: put a hash of the validated data here totals: DacTotalAggregates { @@ -348,7 +348,38 @@ pub mod pallet { failure_rate: 0, // ToDo }, }; + + // Encode validation decision to base64 + let validation_decision_serialized: Vec = validation_decision.encode(); + let validation_decision_base64 = + shm::base64_encode(&validation_decision_serialized); + log::info!( + "Intermediate validation decision for CDN node {:?}: , base64 encoded: {:?}", + validation_decision, + validation_decision_base64, + ); + + // Prepare values to publish validation decision and publish it + let validator_id_string = String::from("validator1"); // ToDo: get validator ID + let edge_id_string = utils::account_to_string::(edge.clone()); + let validation_decision_base64_string = + validation_decision_base64.iter().cloned().collect::(); + let response = shm::share_intermediate_validation_result( + &data_provider_url, + current_era - 1, + &validator_id_string, + &edge_id_string, + validation_result, + &validation_decision_base64_string, + ); + let response_text = response.unwrap().to_string(); + log::info!("Redis response: {:?}", response_text) } + log::info!( + "Intermediate validation results published for {} CDN nodes in era {:?}", + edges.len(), + current_era - 1 + ); } } From 1c6e3594af748c551b072da37c64c2843121c3d8 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 4 May 2023 16:16:35 +0200 Subject: [PATCH 147/192] Implement validator decisions fetching Cargo fmt Post mocked final decision Parse intermediate decisions Adjust get_final_decisions according to new structs Fix input structures Remove unused code Replace Post with Put Run cargo fmt --- frame/ddc-validator/src/dac.rs | 165 ++++++++++++++++++++++++++++++++- frame/ddc-validator/src/lib.rs | 5 + 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 2bf6cde8578d6..36e5bd4ef6b0f 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -4,8 +4,13 @@ use alloc::{format, string::String}; // ToDo: remove String usage use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; use codec::{Decode, Encode}; use lite_json::json::JsonValue; +use log::info; use serde_json::Value; -use sp_runtime::offchain::{http, Duration}; +use sp_runtime::offchain::{ + http, + http::{Method, Request}, + Duration, +}; use sp_staking::EraIndex; pub use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, @@ -119,6 +124,52 @@ pub struct FileInfo { requested_chunk_cids: Vec, } +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct ValidationResult { + data: String, + result: bool, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct ValidationResults(BTreeMap); + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct Edges(BTreeMap); + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct Wrapper { + #[serde(rename = "HGET")] + decisions: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct ResultLog { + validator_id: String, + data: String, + result: bool, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct ResultsLog(Vec>); + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct FinalDecision { + data: String, + result: bool, + results_log: ResultsLog, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct FinalDecisions(BTreeMap); + impl BytesSent { pub fn new(aggregate: RedisFtAggregate) -> BytesSent { let data = aggregate.ft_aggregate[1].clone(); @@ -246,7 +297,8 @@ pub fn get_proved_delivered_bytes_sum(file_requests: &Requests) -> u64 { fn get_proved_delivered_bytes(chunk: &Chunk, ack_timestamps: &Vec) -> u64 { let log_timestamp = chunk.log.timestamp; let neighbors = get_closer_neighbors(log_timestamp, &ack_timestamps); - let is_proved = is_lies_within_threshold(log_timestamp, neighbors, FAILED_CONTENT_CONSUMER_THRESHOLD); + let is_proved = + is_lies_within_threshold(log_timestamp, neighbors, FAILED_CONTENT_CONSUMER_THRESHOLD); if is_proved { return chunk.log.bytes_sent @@ -439,3 +491,112 @@ pub(crate) fn fetch_aggregates( })?; Ok(json) } + +pub(crate) fn make_http_put(url: &String, payload: &String) -> Result<(), http::Error> { + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); + + let request = http::Request::new(url.as_str()) + .method(Method::Put) + .body(vec![payload.as_bytes()]); + + let pending_req = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; + let response = pending_req.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; + + if response.code != 200 { + log::warn!("Unexpected status code: {}", response.code); + return Err(http::Error::Unknown) + } + + let body = response.body().collect::>(); + let body_str = sp_std::str::from_utf8(&body).map_err(|_| { + log::warn!("No UTF-8 body"); + http::Error::Unknown + })?; + + let json = lite_json::parse_json(body_str).map_err(|_| { + log::warn!("No JSON body"); + http::Error::Unknown + })?; + + Ok(()) +} + +pub(crate) fn fetch_validators_decisions( + data_provider_url: &String, + era: EraIndex, +) -> Result { + let url = format!("{}/HGET/mock:ddc:dac:decisions_to_eras/{}", data_provider_url, era); + let wrapper: Wrapper = http_get_json(&url).unwrap(); + + Ok(wrapper) +} + +pub(crate) fn post_final_decision( + data_provider_url: &String, + era: EraIndex, + decision: FinalDecision, +) -> Result<(), http::Error> { + let url = format!("{}/HSET/mock:ddc:dac:final_decision_to_era/{}", data_provider_url, era); + + let payload_str: String = serde_json::to_string(&decision).unwrap(); + let res = make_http_put(&url, &payload_str); + + res +} + +pub(crate) fn get_final_decision(decisions: &ValidationResults) -> FinalDecision { + let mut validators_on_edge = 0u32; + let mut positive_count = 0u32; + + let mut results_log: Vec> = Vec::new(); + for (validator_id, decision) in decisions.0.iter() { + let result = decision.result; + + if result == true { + positive_count += 1; + } + + let result_log_value = + ResultLog { validator_id: validator_id.clone(), data: String::from("Base64"), result }; + + let mut result_log: BTreeMap = BTreeMap::new(); + result_log.insert(validator_id.clone(), result_log_value); + results_log.push(result_log); + + validators_on_edge += 1; + } + + let threshold = validators_on_edge / 2; + + let mut validation_result = false; + if positive_count > threshold { + validation_result = true; + } + + let final_decision = FinalDecision { + data: String::from("Base64"), + result: validation_result, + results_log: ResultsLog(results_log), + }; + + final_decision +} + +pub(crate) fn finalize_decisions( + data_provider_url: &String, + era: EraIndex, + edge: &String, +) -> Result<(), http::Error> { + let wrapper = fetch_validators_decisions(data_provider_url, era).unwrap(); + let edges: Edges = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); + let result = edges.0.get(edge).unwrap(); + info!("decisions: {:?}", result); + + let final_decision = get_final_decision(&result); + + info!("final_decision: {:?}", final_decision); + + let res = post_final_decision(data_provider_url, era, final_decision); + + res +} diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 4f4657115d0fa..a9ea535e10edf 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -255,6 +255,11 @@ pub mod pallet { let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); log::info!("Proved bytes sum: {:?}", bytes_sum); + let assigned_edge = + String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); + dac::finalize_decisions(&data_provider_url, 123345 as EraIndex, &assigned_edge) + .unwrap(); + // Print the number of broken sessions per CDN node. let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data let aggregates_obj = aggregates_value.as_object().unwrap(); From 3dd9e9386fc5a64e4769d0be88979b4c29efb909 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 10 May 2023 11:19:09 +0600 Subject: [PATCH 148/192] Fix shm save_validation_result_by_node request arg --- frame/ddc-validator/src/shm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index 6ae033e77ffd3..d782f0c4b0155 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -30,7 +30,7 @@ pub fn share_intermediate_validation_result( let validation_result_string = String::from(if validation_result { "true" } else { "false" }); let validation_decision_string = String::from(validation_decision_encoded); let url = format!( - "{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{{\"result\":{},\"data\":{}}}", + "{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{{\"result\":{},\"data\":\"{}\"}}", shared_memory_webdis_url, validator, cdn_node, From e700623244c0c2dffd3fd2fad21ddc694e482744 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 10 May 2023 11:43:56 +0600 Subject: [PATCH 149/192] Module docs for validators' shared memory mod --- frame/ddc-validator/src/shm.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index d782f0c4b0155..777f31d1169d4 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -1,4 +1,9 @@ //! Validators' "shared memory" module. +//! +//! It implements a step of the DAC and Validation sequence when validators share their intermediate +//! validation results with each other. The design of the "shared memory" is expected to become like +//! transactions pool or peers list in the future, but for now it works on the centralized Redis +//! server which we maintain for DAC DataModel. use alloc::{format, string::String}; // ToDo: remove String usage use base64::prelude::*; From 7e21d89c3454d90475a9e39b666910866527c670 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 10 May 2023 11:53:49 +0600 Subject: [PATCH 150/192] Payments module for DDC Validator pallet --- frame/ddc-validator/src/lib.rs | 1 + frame/ddc-validator/src/payments.rs | 1 + 2 files changed, 2 insertions(+) create mode 100644 frame/ddc-validator/src/payments.rs diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a9ea535e10edf..820aaf5dce5ec 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -16,6 +16,7 @@ #![cfg_attr(not(feature = "std"), no_std)] mod dac; +mod payments; mod shm; mod utils; mod validation; diff --git a/frame/ddc-validator/src/payments.rs b/frame/ddc-validator/src/payments.rs new file mode 100644 index 0000000000000..a1a26ade5908d --- /dev/null +++ b/frame/ddc-validator/src/payments.rs @@ -0,0 +1 @@ +//! Payments module to calculate and store a withdrawal per content owner and a payout per CDN node. From 311d8be5c4f527d7488257643061fa401d695424 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 12:22:48 +0600 Subject: [PATCH 151/192] Reward points storage item in ddc-staking --- frame/ddc-staking/src/lib.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index a8c3535c62357..a1d0d0a364d8b 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -24,12 +24,15 @@ use sp_runtime::{ }; use sp_staking::EraIndex; -use sp_std::prelude::*; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; pub use pallet::*; const DDC_STAKING_ID: LockIdentifier = *b"ddcstake"; // DDC maintainer's stake +/// Counter for the number of "reward" points earned by a given staker. +pub type RewardPoint = u32; + /// The balance type of this pallet. pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -39,6 +42,21 @@ parameter_types! { pub MaxUnlockingChunks: u32 = 32; } +/// Reward points of an era. Used to split era total payout between stakers. +#[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] +pub struct EraRewardPoints { + /// Total number of points. Equals the sum of reward points for each staker. + total: RewardPoint, + /// The reward points earned by a given staker. + individual: BTreeMap, +} + +impl Default for EraRewardPoints { + fn default() -> Self { + EraRewardPoints { total: Default::default(), individual: BTreeMap::new() } + } +} + /// Just a Balance/BlockNumber tuple to encode when a chunk of funds will be unlocked. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct UnlockChunk { @@ -178,6 +196,12 @@ pub mod pallet { #[pallet::getter(fn current_era)] pub type CurrentEra = StorageValue<_, EraIndex>; + /// The reward each CDN participant earned in the era. + #[pallet::storage] + #[pallet::getter(fn eras_edges_reward_points)] + pub type ErasEdgesRewardPoints = + StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; + #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { From 8708dc019c23b58949251dfdb25be732bef80679 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 12:55:43 +0600 Subject: [PATCH 152/192] Autoformat DDC Staking files --- frame/ddc-staking/src/benchmarking.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frame/ddc-staking/src/benchmarking.rs b/frame/ddc-staking/src/benchmarking.rs index 85b22a0832a33..a15f6f34728e0 100644 --- a/frame/ddc-staking/src/benchmarking.rs +++ b/frame/ddc-staking/src/benchmarking.rs @@ -78,7 +78,7 @@ benchmarks! { let controller = create_funded_user::("controller", USER_SEED, 100); let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); - let amount = T::Currency::minimum_balance() * 10u32.into(); + let amount = T::Currency::minimum_balance() * 10u32.into(); whitelist_account!(stash); }: _(RawOrigin::Signed(stash.clone()), controller_lookup, amount) verify { @@ -90,9 +90,9 @@ benchmarks! { // clean up any existing state. clear_storages_and_edges::(); - let origin_balance = MinStorageBond::::get().max(T::Currency::minimum_balance()); + let origin_balance = MinStorageBond::::get().max(T::Currency::minimum_balance()); - let scenario = AccountsScenario::::new(origin_balance)?; + let scenario = AccountsScenario::::new(origin_balance)?; // Original benchmark staking code (/frame/staking/src/benchmarking.rs) let max_additional = BalanceOf::::try_from(u128::MAX).map_err(|_| "balance expected to be a u128").unwrap() - origin_balance; @@ -117,7 +117,7 @@ benchmarks! { clear_storages_and_edges::(); let total_issuance = T::Currency::total_issuance(); - + // Constant taken from original benchmark staking code (/frame/staking/src/benchmarking.rs) let origin_balance = BalanceOf::::try_from(952_994_955_240_703u128) .map_err(|_| "balance expected to be a u128") @@ -127,7 +127,7 @@ benchmarks! { let stash = scenario.origin_stash1.clone(); let controller = scenario.origin_controller1.clone(); // unbond half of initial balance - let amount = origin_balance / 2u32.into(); + let amount = origin_balance / 2u32.into(); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_bonded: BalanceOf = ledger.active; @@ -143,7 +143,7 @@ benchmarks! { let (stash, controller) = create_stash_controller::(0, 100)?; let amount = T::Currency::minimum_balance() * 5u32.into(); // Half of total DddcStaking::::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?; - CurrentEra::::put(EraIndex::max_value()); + CurrentEra::::put(EraIndex::max_value()); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_total: BalanceOf = ledger.total; whitelist_account!(controller); From 26b37ed5c5c37ee5848acef88d49e9c828ef8587 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 13:07:36 +0600 Subject: [PATCH 153/192] New call to payout to DDC stakers --- frame/ddc-staking/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index a1d0d0a364d8b..60cadbc7e3fa8 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -528,9 +528,20 @@ pub mod pallet { } Ok(()) } + + /// Pay out all the stakers for a single era. + #[pallet::weight(100_000)] + pub fn payout_stakers(origin: OriginFor, era: EraIndex) -> DispatchResult { + ensure_signed(origin)?; + Self::do_payout_stakers(era) + } } impl Pallet { + pub(super) fn do_payout_stakers(era: EraIndex) -> DispatchResult { + Ok(()) + } + /// Update the ledger for a controller. /// /// This will also update the stash lock. From 775b1f14b1f724ab9b73b128ae1037199791f1ef Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 11 May 2023 10:31:28 +0200 Subject: [PATCH 154/192] Get final decision --- frame/ddc-validator/src/dac.rs | 102 +++++++++++++++++++-------------- frame/ddc-validator/src/lib.rs | 4 +- 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 36e5bd4ef6b0f..e9253f051b5c2 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -6,6 +6,7 @@ use codec::{Decode, Encode}; use lite_json::json::JsonValue; use log::info; use serde_json::Value; +use sp_runtime::generic::Era; use sp_runtime::offchain::{ http, http::{Method, Request}, @@ -124,20 +125,16 @@ pub struct FileInfo { requested_chunk_cids: Vec, } -#[derive(Debug, Deserialize, Serialize)] -#[serde(crate = "alt_serde")] -pub(crate) struct ValidationResult { - data: String, - result: bool, -} +type EdgeId = String; +type ValidatorId = String; -#[derive(Debug, Deserialize, Serialize)] -#[serde(crate = "alt_serde")] -pub(crate) struct ValidationResults(BTreeMap); +// #[derive(Debug, Deserialize, Serialize)] +// #[serde(crate = "alt_serde")] +// pub(crate) struct ValidationResults(BTreeMap); #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] -pub(crate) struct Edges(BTreeMap); +pub(crate) struct EdgesToResults(BTreeMap>); #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] @@ -148,22 +145,28 @@ pub(crate) struct Wrapper { #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] -pub(crate) struct ResultLog { - validator_id: String, - data: String, +pub(crate) struct ValidationResult { + validator_id: ValidatorId, + edge_id: EdgeId, result: bool, + received: u64, + sent: u64, + era: EraIndex, } -#[derive(Debug, Deserialize, Serialize)] -#[serde(crate = "alt_serde")] -pub(crate) struct ResultsLog(Vec>); +// #[derive(Debug, Deserialize, Serialize)] +// #[serde(crate = "alt_serde")] +// pub(crate) struct ResultsLogs(Vec); #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] pub(crate) struct FinalDecision { - data: String, result: bool, - results_log: ResultsLog, + edge_id: EdgeId, + era: EraIndex, + received: u64, + sent: u64, + results_logs: Vec, } #[derive(Debug, Deserialize, Serialize)] @@ -544,24 +547,17 @@ pub(crate) fn post_final_decision( res } -pub(crate) fn get_final_decision(decisions: &ValidationResults) -> FinalDecision { +pub(crate) fn get_final_decision(decisions: Vec) -> FinalDecision { let mut validators_on_edge = 0u32; let mut positive_count = 0u32; - let mut results_log: Vec> = Vec::new(); - for (validator_id, decision) in decisions.0.iter() { - let result = decision.result; - - if result == true { + let mut results_logs: Vec = Vec::new(); + for decision in decisions { + if decision.result == true { positive_count += 1; } - let result_log_value = - ResultLog { validator_id: validator_id.clone(), data: String::from("Base64"), result }; - - let mut result_log: BTreeMap = BTreeMap::new(); - result_log.insert(validator_id.clone(), result_log_value); - results_log.push(result_log); + results_logs.push(decision); validators_on_edge += 1; } @@ -574,29 +570,47 @@ pub(crate) fn get_final_decision(decisions: &ValidationResults) -> FinalDecision } let final_decision = FinalDecision { - data: String::from("Base64"), result: validation_result, - results_log: ResultsLog(results_log), + edge_id: results_logs[0].edge_id.clone(), + results_logs, + // Todo: Implement fn to get the values from intermediate decisions + received: 0, + sent: 0, + era: 0, }; final_decision } -pub(crate) fn finalize_decisions( +// pub(crate) fn finalize_decisions( +// data_provider_url: &String, +// era: EraIndex, +// edge: &String, +// ) -> Result<(), http::Error> { +// let wrapper = fetch_validators_decisions(data_provider_url, era).unwrap(); +// let edges: Edges = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); +// let result = edges.0.get(edge).unwrap(); +// info!("decisions: {:?}", result); +// +// let final_decision = get_final_decision(&result); +// +// info!("final_decision: {:?}", final_decision); +// +// let res = post_final_decision(data_provider_url, era, final_decision); +// +// res +// } + +pub(crate) fn get_validation_results( data_provider_url: &String, era: EraIndex, edge: &String, -) -> Result<(), http::Error> { +) -> Result, http::Error> { let wrapper = fetch_validators_decisions(data_provider_url, era).unwrap(); - let edges: Edges = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); - let result = edges.0.get(edge).unwrap(); - info!("decisions: {:?}", result); + let mut edges: EdgesToResults = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); + let results = edges.0.remove(edge).unwrap(); - let final_decision = get_final_decision(&result); - - info!("final_decision: {:?}", final_decision); - - let res = post_final_decision(data_provider_url, era, final_decision); - - res + Ok(results) } + +// pub(crate) fn save_final_decision(); diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index a9ea535e10edf..386b18efc93b9 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -257,8 +257,10 @@ pub mod pallet { let assigned_edge = String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); - dac::finalize_decisions(&data_provider_url, 123345 as EraIndex, &assigned_edge) + let validations_res = dac::get_validation_results(&data_provider_url, 5 as EraIndex, &assigned_edge) .unwrap(); + let final_res = dac::get_final_decision(validations_res); + log::info!("final_res: {:?}", final_res); // Print the number of broken sessions per CDN node. let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data From 0f96a7f32dc65934fbf8ef6158b4fb32c41f3912 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 11 May 2023 10:32:33 +0200 Subject: [PATCH 155/192] Run cargo fmt --- frame/ddc-validator/src/dac.rs | 12 +++++++----- frame/ddc-validator/src/lib.rs | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index e9253f051b5c2..da789361fe094 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -6,11 +6,13 @@ use codec::{Decode, Encode}; use lite_json::json::JsonValue; use log::info; use serde_json::Value; -use sp_runtime::generic::Era; -use sp_runtime::offchain::{ - http, - http::{Method, Request}, - Duration, +use sp_runtime::{ + generic::Era, + offchain::{ + http, + http::{Method, Request}, + Duration, + }, }; use sp_staking::EraIndex; pub use sp_std::{ diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 386b18efc93b9..10c359694079e 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -257,8 +257,9 @@ pub mod pallet { let assigned_edge = String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); - let validations_res = dac::get_validation_results(&data_provider_url, 5 as EraIndex, &assigned_edge) - .unwrap(); + let validations_res = + dac::get_validation_results(&data_provider_url, 5 as EraIndex, &assigned_edge) + .unwrap(); let final_res = dac::get_final_decision(validations_res); log::info!("final_res: {:?}", final_res); From 25f9ac278c79c2f645d99366326308a145647f34 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 11 May 2023 12:48:34 +0200 Subject: [PATCH 156/192] Replace FinalDecision wiht ValidationDecision --- frame/ddc-validator/src/dac.rs | 41 ++++++++------------------------ frame/ddc-validator/src/utils.rs | 22 +++++++++++++++++ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index da789361fe094..1ed5d217ec575 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -19,8 +19,7 @@ pub use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, prelude::*, }; - -use crate::utils; +use crate::{DacTotalAggregates, utils, ValidationDecision}; pub type TimestampInSec = u64; pub const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -549,7 +548,7 @@ pub(crate) fn post_final_decision( res } -pub(crate) fn get_final_decision(decisions: Vec) -> FinalDecision { +pub(crate) fn get_final_decision(decisions: Vec) -> ValidationDecision { let mut validators_on_edge = 0u32; let mut positive_count = 0u32; @@ -571,38 +570,20 @@ pub(crate) fn get_final_decision(decisions: Vec) -> FinalDecis validation_result = true; } - let final_decision = FinalDecision { + let final_decision= ValidationDecision { result: validation_result, - edge_id: results_logs[0].edge_id.clone(), - results_logs, - // Todo: Implement fn to get the values from intermediate decisions - received: 0, - sent: 0, - era: 0, + payload: utils::get_hashed(&results_logs), + totals: DacTotalAggregates { + received: 0, + sent: 0, + failed_by_client: 0, + failure_rate: 0, + }, }; final_decision } -// pub(crate) fn finalize_decisions( -// data_provider_url: &String, -// era: EraIndex, -// edge: &String, -// ) -> Result<(), http::Error> { -// let wrapper = fetch_validators_decisions(data_provider_url, era).unwrap(); -// let edges: Edges = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); -// let result = edges.0.get(edge).unwrap(); -// info!("decisions: {:?}", result); -// -// let final_decision = get_final_decision(&result); -// -// info!("final_decision: {:?}", final_decision); -// -// let res = post_final_decision(data_provider_url, era, final_decision); -// -// res -// } - pub(crate) fn get_validation_results( data_provider_url: &String, era: EraIndex, @@ -614,5 +595,3 @@ pub(crate) fn get_validation_results( Ok(results) } - -// pub(crate) fn save_final_decision(); diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs index 5c8c5cf4f726b..bf3ed3cc41f9e 100644 --- a/frame/ddc-validator/src/utils.rs +++ b/frame/ddc-validator/src/utils.rs @@ -1,6 +1,11 @@ use alloc::string::String; use codec::{Decode, Encode}; use sp_core::crypto::AccountId32; +use sp_io::hashing::blake2_256; +use crate::dac::ValidationResult; +pub use sp_std::{ + prelude::*, +}; pub fn account_to_string(account: T::AccountId) -> String { let to32 = T::AccountId::encode(&account); @@ -16,3 +21,20 @@ pub fn string_to_account(pub_key_str: String) -> T::Acc let address: T::AccountId = T::AccountId::decode(&mut to32).unwrap(); address } + +pub(crate) fn hash(data: &String) -> [u8; 32] { + let hash = blake2_256(data.as_bytes()); + let mut result = [0u8; 32]; + result.copy_from_slice(&hash); + + result +} + +pub(crate) fn get_hashed(data: &Vec) -> [u8; 256] { + let results_log = serde_json::to_string(data).unwrap(); + let mut payload:[u8; 256] = [0; 256]; + let hashed_results = hash(&results_log); + payload[..32].copy_from_slice(&hashed_results); + + payload +} From a9907ef557d045e372ff70d2766bd773ad98f864 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 17:17:49 +0600 Subject: [PATCH 157/192] Use u64 for DDC staker's reward points --- frame/ddc-staking/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 60cadbc7e3fa8..ec45025b04a96 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -31,7 +31,7 @@ pub use pallet::*; const DDC_STAKING_ID: LockIdentifier = *b"ddcstake"; // DDC maintainer's stake /// Counter for the number of "reward" points earned by a given staker. -pub type RewardPoint = u32; +pub type RewardPoint = u64; /// The balance type of this pallet. pub type BalanceOf = From fd2ebc9be3b840d86c6f914a5c9d2a3a3584e9f9 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 17:22:17 +0600 Subject: [PATCH 158/192] A func to add reward points to CDN participants --- frame/ddc-staking/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index ec45025b04a96..548981f6d18f2 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -645,5 +645,18 @@ pub mod pallet { outcome } + + /// Add reward points to CDN participants using their stash account ID. + pub fn reward_by_ids( + era: EraIndex, + stakers_points: impl IntoIterator, + ) { + >::mutate(era, |era_rewards| { + for (staker, points) in stakers_points.into_iter() { + *era_rewards.individual.entry(staker).or_default() += points; + era_rewards.total += points; + } + }); + } } } From b1495c920d911a6b3d3be23897b3ed2ad3ad966c Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 17:27:52 +0600 Subject: [PATCH 159/192] New call to set reward points for CDN participants --- frame/ddc-staking/src/lib.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 548981f6d18f2..c1ec154e054de 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -535,6 +535,23 @@ pub mod pallet { ensure_signed(origin)?; Self::do_payout_stakers(era) } + + #[pallet::weight(100_000)] + pub fn set_era_reward_points( + origin: OriginFor, + era: EraIndex, + stakers_points: Vec<(T::AccountId, u64)>, + ) -> DispatchResult { + ensure_signed(origin)?; + + // ToDo: ensure origin is a validator eligible to set rewards + // ToDo: check that a staker mentioned only once + // ToDo: check that all accounts had an active stake at the era + + Self::reward_by_ids(era, stakers_points); + + Ok(()) + } } impl Pallet { From cc180732a04c19193fcb160be10d0bb401f3bd19 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 17:30:20 +0600 Subject: [PATCH 160/192] Error on staker duplicates in rewards points list --- frame/ddc-staking/src/lib.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index c1ec154e054de..51e325006f3b2 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -24,7 +24,10 @@ use sp_runtime::{ }; use sp_staking::EraIndex; -use sp_std::{collections::btree_map::BTreeMap, prelude::*}; +use sp_std::{ + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + prelude::*, +}; pub use pallet::*; @@ -241,6 +244,8 @@ pub mod pallet { // An account already declared a desire to participate in the network with a certain role // and to take another role it should call `chill` first. AlreadyInRole, + /// Two or more occurrences of a staker account in rewards points list. + DuplicateRewardPoints, } #[pallet::call] @@ -545,7 +550,14 @@ pub mod pallet { ensure_signed(origin)?; // ToDo: ensure origin is a validator eligible to set rewards - // ToDo: check that a staker mentioned only once + + // Check that a staker mentioned only once, fail with an error otherwise. + let unique_stakers_count = + stakers_points.iter().map(|(staker, _)| staker).collect::>().len(); + if unique_stakers_count != stakers_points.len() { + Err(Error::::DuplicateRewardPoints)? + } + // ToDo: check that all accounts had an active stake at the era Self::reward_by_ids(era, stakers_points); From 54d59318f742d6a5408fd56edee3bbebdc680055 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 18:29:11 +0600 Subject: [PATCH 161/192] Doc comment for the `set_era_reward_points` call --- frame/ddc-staking/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 51e325006f3b2..b04843711a430 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -541,6 +541,15 @@ pub mod pallet { Self::do_payout_stakers(era) } + /// Set reward points for CDN participants at the given era. + /// + /// The dispatch origin for this call must be _Signed_ by the validator. + /// + /// `stakers_points` is a vector of (stash account ID, reward points) pairs. The rewards + /// distribution will be based on total reward points, with each CDN participant receiving a + /// proportionate reward based on their individual reward points. + /// + /// See also [`ErasEdgesRewardPoints`]. #[pallet::weight(100_000)] pub fn set_era_reward_points( origin: OriginFor, From b7a7c113fa9292442307d60eb46e1df39c3681da Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Thu, 11 May 2023 18:34:10 +0600 Subject: [PATCH 162/192] Refer to a similar item in `pallet_staking` --- frame/ddc-staking/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index b04843711a430..8f75b8789cadf 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -200,6 +200,8 @@ pub mod pallet { pub type CurrentEra = StorageValue<_, EraIndex>; /// The reward each CDN participant earned in the era. + /// + /// See also [`pallet_staking::ErasRewardPoints`]. #[pallet::storage] #[pallet::getter(fn eras_edges_reward_points)] pub type ErasEdgesRewardPoints = From 04b4c71663784171a20303a201125d3ddf862d32 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 11 May 2023 17:09:39 +0200 Subject: [PATCH 163/192] Implement find_largest_group --- frame/ddc-validator/src/dac.rs | 61 +++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 1ed5d217ec575..5bcb53ce82d61 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -144,7 +144,7 @@ pub(crate) struct Wrapper { decisions: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] #[serde(crate = "alt_serde")] pub(crate) struct ValidationResult { validator_id: ValidatorId, @@ -549,30 +549,12 @@ pub(crate) fn post_final_decision( } pub(crate) fn get_final_decision(decisions: Vec) -> ValidationDecision { - let mut validators_on_edge = 0u32; - let mut positive_count = 0u32; - - let mut results_logs: Vec = Vec::new(); - for decision in decisions { - if decision.result == true { - positive_count += 1; - } - - results_logs.push(decision); - - validators_on_edge += 1; - } - - let threshold = validators_on_edge / 2; - - let mut validation_result = false; - if positive_count > threshold { - validation_result = true; - } + let common_decisions = find_largest_group(decisions).unwrap(); + let decision_example = common_decisions.get(0).unwrap(); let final_decision= ValidationDecision { - result: validation_result, - payload: utils::get_hashed(&results_logs), + result: decision_example.result, + payload: utils::get_hashed(&common_decisions), totals: DacTotalAggregates { received: 0, sent: 0, @@ -595,3 +577,36 @@ pub(crate) fn get_validation_results( Ok(results) } + +fn find_largest_group(decisions: Vec) -> Option> { + let mut groups: Vec> = Vec::new(); + let half = decisions.len() / 2; + + for decision in decisions { + let mut found_group = false; + + for group in &mut groups { + if group.iter().all(|x| x.result == decision.result && x.received == decision.received && x.sent == decision.sent) { + group.push(decision.clone()); + found_group = true; + break; + } + } + + if !found_group { + groups.push(vec![decision]); + } + } + + let largest_group = groups + .into_iter() + .max_by_key(|group| group.len()) + .unwrap_or(Vec::new()); + + + if largest_group.len() > half { + Some(largest_group) + } else { + None + } +} From ffc6e91117c90ed953b9560ddb65a852fd81f04d Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 12 May 2023 09:51:22 +0200 Subject: [PATCH 164/192] Send tx --- frame/ddc-validator/src/dac.rs | 4 ++-- frame/ddc-validator/src/lib.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 5bcb53ce82d61..f964c594759ee 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -556,8 +556,8 @@ pub(crate) fn get_final_decision(decisions: Vec) -> Validation result: decision_example.result, payload: utils::get_hashed(&common_decisions), totals: DacTotalAggregates { - received: 0, - sent: 0, + received: decision_example.received, + sent: decision_example.sent, failed_by_client: 0, failure_rate: 0, }, diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 10c359694079e..dfe80316eec2d 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -261,6 +261,17 @@ pub mod pallet { dac::get_validation_results(&data_provider_url, 5 as EraIndex, &assigned_edge) .unwrap(); let final_res = dac::get_final_decision(validations_res); + + let signer = Self::get_signer().unwrap(); + + let tx_res = signer.send_signed_transaction(|_acct| { + Call::set_validation_decision { + era: 5 as EraIndex, + cdn_node: utils::string_to_account::(assigned_edge.clone()), + validation_decision: final_res.clone(), + } + }); + log::info!("final_res: {:?}", final_res); // Print the number of broken sessions per CDN node. From 92f7768b38bf426198b4816d42775284f6082667 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 12 May 2023 09:55:07 +0200 Subject: [PATCH 165/192] Cargo fmt --- frame/ddc-validator/src/dac.rs | 26 +++++++++----------------- frame/ddc-validator/src/lib.rs | 10 ++++------ frame/ddc-validator/src/utils.rs | 8 +++----- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index f964c594759ee..9f74e309399af 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -1,5 +1,6 @@ //! A module with Data Activity Capture (DAC) interaction. +use crate::{utils, DacTotalAggregates, ValidationDecision}; use alloc::{format, string::String}; // ToDo: remove String usage use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; use codec::{Decode, Encode}; @@ -19,7 +20,6 @@ pub use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, prelude::*, }; -use crate::{DacTotalAggregates, utils, ValidationDecision}; pub type TimestampInSec = u64; pub const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -129,10 +129,6 @@ pub struct FileInfo { type EdgeId = String; type ValidatorId = String; -// #[derive(Debug, Deserialize, Serialize)] -// #[serde(crate = "alt_serde")] -// pub(crate) struct ValidationResults(BTreeMap); - #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] pub(crate) struct EdgesToResults(BTreeMap>); @@ -155,10 +151,6 @@ pub(crate) struct ValidationResult { era: EraIndex, } -// #[derive(Debug, Deserialize, Serialize)] -// #[serde(crate = "alt_serde")] -// pub(crate) struct ResultsLogs(Vec); - #[derive(Debug, Deserialize, Serialize)] #[serde(crate = "alt_serde")] pub(crate) struct FinalDecision { @@ -552,7 +544,7 @@ pub(crate) fn get_final_decision(decisions: Vec) -> Validation let common_decisions = find_largest_group(decisions).unwrap(); let decision_example = common_decisions.get(0).unwrap(); - let final_decision= ValidationDecision { + let final_decision = ValidationDecision { result: decision_example.result, payload: utils::get_hashed(&common_decisions), totals: DacTotalAggregates { @@ -586,10 +578,14 @@ fn find_largest_group(decisions: Vec) -> Option) -> Option half { Some(largest_group) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index dfe80316eec2d..fad5faab26536 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -264,12 +264,10 @@ pub mod pallet { let signer = Self::get_signer().unwrap(); - let tx_res = signer.send_signed_transaction(|_acct| { - Call::set_validation_decision { - era: 5 as EraIndex, - cdn_node: utils::string_to_account::(assigned_edge.clone()), - validation_decision: final_res.clone(), - } + let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { + era: 5 as EraIndex, + cdn_node: utils::string_to_account::(assigned_edge.clone()), + validation_decision: final_res.clone(), }); log::info!("final_res: {:?}", final_res); diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs index bf3ed3cc41f9e..3c5c90c8028c9 100644 --- a/frame/ddc-validator/src/utils.rs +++ b/frame/ddc-validator/src/utils.rs @@ -1,11 +1,9 @@ +use crate::dac::ValidationResult; use alloc::string::String; use codec::{Decode, Encode}; use sp_core::crypto::AccountId32; use sp_io::hashing::blake2_256; -use crate::dac::ValidationResult; -pub use sp_std::{ - prelude::*, -}; +pub use sp_std::prelude::*; pub fn account_to_string(account: T::AccountId) -> String { let to32 = T::AccountId::encode(&account); @@ -32,7 +30,7 @@ pub(crate) fn hash(data: &String) -> [u8; 32] { pub(crate) fn get_hashed(data: &Vec) -> [u8; 256] { let results_log = serde_json::to_string(data).unwrap(); - let mut payload:[u8; 256] = [0; 256]; + let mut payload: [u8; 256] = [0; 256]; let hashed_results = hash(&results_log); payload[..32].copy_from_slice(&hashed_results); From cb89717ec7163f88628300c557723b5c81e5edee Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 12 May 2023 10:07:07 +0200 Subject: [PATCH 166/192] Put mocked era to get_validation_results --- frame/ddc-validator/src/dac.rs | 2 +- frame/ddc-validator/src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 9f74e309399af..68442f1b14abd 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -563,7 +563,7 @@ pub(crate) fn get_validation_results( era: EraIndex, edge: &String, ) -> Result, http::Error> { - let wrapper = fetch_validators_decisions(data_provider_url, era).unwrap(); + let wrapper = fetch_validators_decisions(data_provider_url, 5 as EraIndex).unwrap(); // Era is mocked for now let mut edges: EdgesToResults = serde_json::from_str(wrapper.decisions.as_str()).unwrap(); let results = edges.0.remove(edge).unwrap(); diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index fad5faab26536..9bde10b40e0ef 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -246,6 +246,7 @@ pub mod pallet { return } + let current_era = Self::get_current_era(); let data_provider_url = Self::get_data_provider_url(); log::info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); @@ -258,14 +259,14 @@ pub mod pallet { let assigned_edge = String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); let validations_res = - dac::get_validation_results(&data_provider_url, 5 as EraIndex, &assigned_edge) + dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) .unwrap(); let final_res = dac::get_final_decision(validations_res); let signer = Self::get_signer().unwrap(); let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { - era: 5 as EraIndex, + era: current_era, cdn_node: utils::string_to_account::(assigned_edge.clone()), validation_decision: final_res.clone(), }); @@ -308,7 +309,6 @@ pub mod pallet { } // Read from DAC. - let current_era = Self::get_current_era(); let (sent_query, sent, received_query, received) = dac::fetch_data2(&data_provider_url, current_era - 1); log::info!( From 5da383fc41d3ec010e42269daa42d8db806e0924 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 12 May 2023 17:16:49 +0600 Subject: [PATCH 167/192] Make public `ddc_staking::EraRewardPoints` fields --- frame/ddc-staking/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 8f75b8789cadf..6f09cebdecaf9 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -49,9 +49,9 @@ parameter_types! { #[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct EraRewardPoints { /// Total number of points. Equals the sum of reward points for each staker. - total: RewardPoint, + pub total: RewardPoint, /// The reward points earned by a given staker. - individual: BTreeMap, + pub individual: BTreeMap, } impl Default for EraRewardPoints { From 94f097180583df2b66ad4d83ab5aa349095d50e8 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 12 May 2023 17:21:28 +0600 Subject: [PATCH 168/192] Add `set_era_reward_points` call to ddc-validator --- frame/ddc-validator/src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index dbf0f1c277393..2b7b3236e18aa 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -438,6 +438,34 @@ pub mod pallet { Ok(()) } + + /// Set reward points for CDN participants at the given era. + /// + /// ToDo: remove it when the off-chain worker will be able to set reward points using the + /// same call defined in `pallet-ddc-staking`. + /// + /// `stakers_points` is a vector of (stash account ID, reward points) pairs. The rewards + /// distribution will be based on total reward points, with each CDN participant receiving a + /// proportionate reward based on their individual reward points. + /// + /// See also [`pallet_ddc_staking::ErasEdgesRewardPoints`]. + #[pallet::weight(100_000)] + pub fn set_era_reward_points( + origin: OriginFor, + era: EraIndex, + stakers_points: Vec<(T::AccountId, u64)>, + ) -> DispatchResult { + ensure_signed(origin)?; + + >::mutate(era, |era_rewards| { + for (staker, points) in stakers_points.into_iter() { + *era_rewards.individual.entry(staker).or_default() += points; + era_rewards.total += points; + } + }); + + Ok(()) + } } impl Pallet From 7116ac82a4a2b5378c21e75d54f6f5d7cd688e16 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Fri, 12 May 2023 19:55:54 +0600 Subject: [PATCH 169/192] Set era reward points from off-chain worker --- frame/ddc-validator/src/lib.rs | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 2b7b3236e18aa..9f9dbdd302deb 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -398,6 +398,66 @@ pub mod pallet { edges.len(), current_era - 1 ); + + // Set CDN nodes' reward points + // ============================ + + // Let's use a mock data until we have a real final validation decisions for all the CDN + // nodes. + let mock_final_validation_decisions: Vec<(T::AccountId, ValidationDecision)> = vec![ + ( + utils::string_to_account::( + "0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1".into(), + ), + ValidationDecision { + result: true, + payload: [0u8; 256], + totals: DacTotalAggregates { + sent: 100, + received: 100, + failed_by_client: 0, + failure_rate: 0, + }, + }, + ), + ( + utils::string_to_account::( + "0xa2d14e71b52e5695e72c0567926bc68b68bda74df5c1ccf1d4ba612c153ff66b".into(), + ), + ValidationDecision { + result: true, + payload: [0u8; 256], + totals: DacTotalAggregates { + sent: 200, + received: 200, + failed_by_client: 0, + failure_rate: 0, + }, + }, + ), + ]; + + // Calculate CDN nodes reward points from validation decision aggregates + let cdn_nodes_reward_points: Vec<(T::AccountId, u64)> = mock_final_validation_decisions + .into_iter() + .filter(|(_, validation_decision)| validation_decision.result) // skip misbehaving + .map(|(cdn_node, validation_decision)| { + // ToDo: should we use `sent` or `received` or anything else as a reward point? + (cdn_node, validation_decision.totals.sent) + }) + .collect(); + + // Store CDN node reward points on-chain + let signer: Signer = Signer::<_, _>::any_account(); + if !signer.can_sign() { + log::warn!("No local accounts available to set era reward points. Consider adding one via `author_insertKey` RPC."); + return + } + // ToDo: replace local call by a call from `ddc-staking` pallet + let _tx_res = signer.send_signed_transaction(|_account| Call::set_era_reward_points { + era: current_era - 1, + stakers_points: cdn_nodes_reward_points.clone(), + }); } } From 9966d265e64a1639abb7c23fa84dec958d67886a Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Mon, 15 May 2023 14:05:53 +0600 Subject: [PATCH 170/192] Prevent panic on redis shared memory comm error --- frame/ddc-validator/src/lib.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 9f9dbdd302deb..36a708d99c1f2 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -390,8 +390,13 @@ pub mod pallet { validation_result, &validation_decision_base64_string, ); - let response_text = response.unwrap().to_string(); - log::info!("Redis response: {:?}", response_text) + match response { + Ok(response) => log::info!("Shared memory response: {:?}", response.to_string()), + Err(e) => { + log::error!("Shared memory error: {:?}", e); + continue + }, + } } log::info!( "Intermediate validation results published for {} CDN nodes in era {:?}", From 38550b52e10473dd6f78846f553b367f0aebd091 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 12 May 2023 18:35:21 +0200 Subject: [PATCH 171/192] accounts first version --- Cargo.lock | 17 ++ bin/node/runtime/Cargo.toml | 2 + bin/node/runtime/src/lib.rs | 14 ++ frame/ddc-accounts/Cargo.toml | 31 +++ frame/ddc-accounts/src/lib.rs | 416 ++++++++++++++++++++++++++++++++++ 5 files changed, 480 insertions(+) create mode 100644 frame/ddc-accounts/Cargo.toml create mode 100644 frame/ddc-accounts/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3787bef87b1a2..600c56f222904 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5066,6 +5066,7 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", + "pallet-ddc-accounts", "pallet-ddc-metrics-offchain-worker", "pallet-ddc-staking", "pallet-ddc-validator", @@ -5824,6 +5825,22 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-ddc-accounts" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "log 0.4.17", + "parity-scale-codec", + "scale-info", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", + "substrate-test-utils", +] + [[package]] name = "pallet-ddc-metrics-offchain-worker" version = "2.0.1" diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 178490e3e7106..aae22edcda732 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -100,6 +100,7 @@ pallet-erc20 = { version = "2.0.0", default-features = false, path = "../../../f pallet-ddc-metrics-offchain-worker = { version = "2.0.0", default-features = false, path = "../../../frame/ddc-metrics-offchain-worker" } pallet-ddc-staking = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-staking" } pallet-ddc-validator= { version = "0.1.0", default-features = false, path = "../../../frame/ddc-validator" } +pallet-ddc-accounts = { version = "0.1.0", default-features = false, path = "../../../frame/ddc-accounts" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } @@ -176,6 +177,7 @@ std = [ "pallet-ddc-metrics-offchain-worker/std", "pallet-ddc-staking/std", "pallet-ddc-validator/std", + "pallet-ddc-accounts/std", ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index d423ca254762a..7d4cd0b189165 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -82,6 +82,7 @@ pub use pallet_chainbridge; pub use pallet_ddc_metrics_offchain_worker; pub use pallet_ddc_staking; pub use pallet_ddc_validator; +pub use pallet_ddc_accounts; #[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; @@ -1233,6 +1234,18 @@ impl pallet_ddc_staking::Config for Runtime { type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } +parameter_types! { + pub const DDC_ACCOUNTS_PALLET_ID: PalletId = PalletId(*b"accounts"); // DDC maintainer's stake +} + +impl pallet_ddc_accounts::Config for Runtime { + type BondingDuration = BondingDuration; + type Currency = Balances; + type Event = Event; + type PalletId = DDC_ACCOUNTS_PALLET_ID; + type TimeProvider = pallet_timestamp::Pallet; +} + parameter_types! { pub const DdcValidatorsQuorumSize: u32 = 3; pub const ValidationThreshold: u32 = 5; @@ -1301,6 +1314,7 @@ construct_runtime!( DdcMetricsOffchainWorker: pallet_ddc_metrics_offchain_worker::{Pallet, Call, Storage, Event}, DdcStaking: pallet_ddc_staking, DdcValidator: pallet_ddc_validator, + DdcAccounts: pallet_ddc_accounts } ); diff --git a/frame/ddc-accounts/Cargo.toml b/frame/ddc-accounts/Cargo.toml new file mode 100644 index 0000000000000..38e73bde4614a --- /dev/null +++ b/frame/ddc-accounts/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "pallet-ddc-accounts" +version = "0.1.0" +edition = "2021" + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +log = { version = "0.4.17", default-features = false } + +[dev-dependencies] +substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-io/std", + "sp-runtime/std", + "sp-staking/std", + "sp-std/std", +] diff --git a/frame/ddc-accounts/src/lib.rs b/frame/ddc-accounts/src/lib.rs new file mode 100644 index 0000000000000..2b0dca3eb875d --- /dev/null +++ b/frame/ddc-accounts/src/lib.rs @@ -0,0 +1,416 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![recursion_limit = "256"] + +use codec::{Decode, Encode, HasCompact}; + +use frame_support::{ + parameter_types, + traits::{Currency, DefensiveSaturating, LockIdentifier, WithdrawReasons, ExistenceRequirement, UnixTime}, + BoundedVec, + PalletId +}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{AccountIdConversion, AtLeast32BitUnsigned, Saturating, Zero}, + RuntimeDebug, +}; + +use sp_staking::EraIndex; +use sp_std::prelude::*; + +pub use pallet::*; + +pub const TIME_START_MS: u128 = 1_672_531_200_000; +pub const ERA_DURATION_MS: u128 = 120_000; +pub const ERA_IN_BLOCKS: u8 = 20; + +/// The balance type of this pallet. +pub type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; + +parameter_types! { + /// A limit to the number of pending unlocks an account may have in parallel. + pub MaxUnlockingChunks: u32 = 32; +} + +/// Just a Balance/BlockNumber tuple to encode when a chunk of funds will be unlocked. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +pub struct UnlockChunk { + /// Amount of funds to be unlocked. + #[codec(compact)] + value: Balance, + /// Era number at which point it'll be unlocked. + #[codec(compact)] + era: EraIndex, +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +pub struct AccountsLedger { + /// The stash account whose balance is actually locked and can be used for CDN usage. + pub stash: AccountId, + /// The total amount of the stash's balance that we are currently accounting for. + /// It's just `active` plus all the `unlocking` balances. + #[codec(compact)] + pub total: Balance, + /// The total amount of the stash's balance that will be accessible for CDN payments in any forthcoming + /// rounds. + #[codec(compact)] + pub active: Balance, + /// Any balance that is becoming free, which may eventually be transferred out of the stash + /// (assuming that the content owner has to pay for network usage). It is assumed that this will be treated as a first + /// in, first out queue where the new (higher value) eras get pushed on the back. + pub unlocking: BoundedVec, MaxUnlockingChunks>, +} + +impl + AccountsLedger +{ + /// Initializes the default object using the given stash. + pub fn default_from(stash: AccountId) -> Self { + Self { stash, total: Zero::zero(), active: Zero::zero(), unlocking: Default::default() } + } + + /// Remove entries from `unlocking` that are sufficiently old and reduce the + /// total by the sum of their balances. + fn consolidate_unlocked(self, current_era: EraIndex) -> Self { + let mut total = self.total; + let unlocking: BoundedVec<_, _> = self + .unlocking + .into_iter() + .filter(|chunk| { + log::info!("Chunk era: {:?}", chunk.era); + if chunk.era > current_era { + true + } else { + total = total.saturating_sub(chunk.value); + false + } + }) + .collect::>() + .try_into() + .expect( + "filtering items from a bounded vec always leaves length less than bounds. qed", + ); + + Self { stash: self.stash, total, active: self.active, unlocking } + } +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::{ + pallet_prelude::*, sp_runtime::traits::StaticLookup, traits::LockableCurrency, + Blake2_128Concat, + }; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The accounts's pallet id, used for deriving its sovereign account ID. + #[pallet::constant] + type PalletId: Get; + type Currency: LockableCurrency; + type Event: From> + IsType<::Event>; + /// Number of eras that staked funds must remain bonded for. + #[pallet::constant] + type BondingDuration: Get; + type TimeProvider: UnixTime; + } + + /// Map from all locked "stash" accounts to the controller account. + #[pallet::storage] + #[pallet::getter(fn bonded)] + pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; + + /// Map from all (unlocked) "controller" accounts to the info regarding the staking. + #[pallet::storage] + #[pallet::getter(fn ledger)] + pub type Ledger = + StorageMap<_, Blake2_128Concat, T::AccountId, AccountsLedger>>; + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + pub enum Event { + /// An account has bonded this amount. \[stash, amount\] + /// + /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, + /// it will not be emitted for staking rewards when they are added to stake. + Deposited(T::AccountId, BalanceOf), + /// An account has unbonded this amount. \[stash, amount\] + Unbonded(T::AccountId, BalanceOf), + /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` + /// from the unlocking queue. \[stash, amount\] + Withdrawn(T::AccountId, BalanceOf), + } + + #[pallet::error] + pub enum Error { + /// Not a controller account. + NotController, + /// Not a stash account. + NotStash, + /// Controller is already paired. + AlreadyPaired, + /// Cannot deposit dust + InsufficientDeposit, + /// Can not schedule more unlock chunks. + NoMoreChunks, + /// Internal state has become somehow corrupted and the operation cannot continue. + BadState, + } + + #[pallet::call] + impl Pallet { + /// Take the origin account as a stash and lock up `value` of its balance. `controller` will + /// be the account that controls it. + /// + /// `value` must be more than the `minimum_balance` specified by `T::Currency`. + /// + /// The dispatch origin for this call must be _Signed_ by the stash account. + /// + /// Emits `Deposited`. + #[pallet::weight(10_000)] pub fn deposit( + origin: OriginFor, + controller: ::Source, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + let controller = T::Lookup::lookup(controller)?; + + if >::contains_key(&controller) { + Err(Error::::AlreadyPaired)? + } + + // Reject a deposit which is considered to be _dust_. + if value < T::Currency::minimum_balance() { + Err(Error::::InsufficientDeposit)? + } + + frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; + + let stash_balance = T::Currency::free_balance(&stash); + let value = value.min(stash_balance); + Self::deposit_event(Event::::Deposited(stash.clone(), value)); + let item = + AccountsLedger { stash: stash.clone(), total: value, active: value, unlocking: Default::default() }; + Self::update_ledger_and_deposit( &stash, &controller, &item); + Ok(()) + } + + /// Add some extra amount that have appeared in the stash `free_balance` into the balance up + /// for CDN payments. + /// + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// + /// Emits `Deposited`. + #[pallet::weight(10_000)] + pub fn deposit_extra( + origin: OriginFor, + #[pallet::compact] max_additional: BalanceOf, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + + let stash_balance = T::Currency::free_balance(&stash); + let extra = stash_balance.min(max_additional); + ledger.total += extra; + ledger.active += extra; + // Last check: the new active amount of ledger must be more than ED. + ensure!( + ledger.active >= T::Currency::minimum_balance(), + Error::::InsufficientDeposit + ); + + Self::update_ledger_and_deposit(&stash, &controller, &ledger); + + Self::deposit_event(Event::::Deposited(stash.clone(), extra)); + + Ok(()) + } + + /// Schedule a portion of the stash to be unlocked ready for transfer out after the bond + /// period ends. If this leaves an amount actively bonded less than + /// T::Currency::minimum_balance(), then it is increased to the full amount. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move + /// the funds out of management ready for transfer. + /// + /// No more than a limited number of unlocking chunks (see `MaxUnlockingChunks`) + /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need + /// to be called first to remove some of the chunks (if possible). + /// + /// Emits `Unbonded`. + /// + /// See also [`Call::withdraw_unbonded`]. + #[pallet::weight(10_000)] + pub fn unbond( + origin: OriginFor, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!( + ledger.unlocking.len() < MaxUnlockingChunks::get() as usize, + Error::::NoMoreChunks, + ); + + let mut value = value.min(ledger.active); + + if !value.is_zero() { + ledger.active -= value; + + // Avoid there being a dust balance left in the accounts system. + if ledger.active < T::Currency::minimum_balance() { + value += ledger.active; + ledger.active = Zero::zero(); + } + + // Note: bonding for extra era to allow for accounting + let era = Self::get_current_era() + T::BondingDuration::get(); + log::info!("Era for the unbond: {:?}", era); + + if let Some(mut chunk) = + ledger.unlocking.last_mut().filter(|chunk| chunk.era == era) + { + // To keep the chunk count down, we only keep one chunk per era. Since + // `unlocking` is a FiFo queue, if a chunk exists for `era` we know that it will + // be the last one. + chunk.value = chunk.value.defensive_saturating_add(value) + } else { + ledger + .unlocking + .try_push(UnlockChunk { value, era }) + .map_err(|_| Error::::NoMoreChunks)?; + }; + + Self::update_ledger(&controller, &ledger); + + Self::deposit_event(Event::::Unbonded(ledger.stash, value)); + } + Ok(()) + } + + + /// Remove any unlocked chunks from the `unlocking` queue from our management. + /// + /// This essentially frees up that balance to be used by the stash account to do + /// whatever it wants. + /// + /// The dispatch origin for this call must be _Signed_ by the controller. + /// + /// Emits `Withdrawn`. + /// + /// See also [`Call::unbond`]. + #[pallet::weight(10_000)] + pub fn withdraw_unbonded(origin: OriginFor) -> DispatchResult { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let (stash, old_total) = (ledger.stash.clone(), ledger.total); + let current_era = Self::get_current_era(); + ledger = ledger.consolidate_unlocked(current_era); + log::info!("Current era: {:?}", current_era); + + if ledger.unlocking.is_empty() && ledger.active < T::Currency::minimum_balance() { + log::info!("Killing stash"); + // This account must have called `unbond()` with some value that caused the active + // portion to fall below existential deposit + will have no more unlocking chunks + // left. We can now safely remove all accounts-related information. + Self::kill_stash(&stash)?; + } else { + log::info!("Updating ledger"); + // This was the consequence of a partial unbond. just update the ledger and move on. + Self::update_ledger(&controller, &ledger); + }; + + log::info!("Current total: {:?}", ledger.total); + log::info!("Old total: {:?}", old_total); + + // `old_total` should never be less than the new total because + // `consolidate_unlocked` strictly subtracts balance. + if ledger.total < old_total { + log::info!("Preparing for transfer"); + // Already checked that this won't overflow by entry condition. + let value = old_total - ledger.total; + + let account_id = Self::account_id(); + + T::Currency::transfer( + &account_id, + &stash, + value, + ExistenceRequirement::KeepAlive, + )?; + Self::deposit_event(Event::::Withdrawn(stash, value)); + } + + Ok(()) + } + } + + impl Pallet { + pub fn account_id() -> T::AccountId { + T::PalletId::get().into_account() + } + /// Update the ledger for a controller. + /// + /// This will also deposit the funds to pallet. + fn update_ledger_and_deposit( + stash: &T::AccountId, + controller: &T::AccountId, + ledger: &AccountsLedger>, + ) { + let account_id = Self::account_id(); + + T::Currency::transfer( + stash, + &account_id, + ledger.total, + ExistenceRequirement::KeepAlive, + )?; + >::insert(controller, ledger); + } + + /// Update the ledger for a controller. + fn update_ledger( + controller: &T::AccountId, + ledger: &AccountsLedger>, + ) { + >::insert(controller, ledger); + } + + /// Remove all associated data of a stash account from the accounts system. + /// + /// Assumes storage is upgraded before calling. + /// + /// This is called: + /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. + fn kill_stash(stash: &T::AccountId) -> DispatchResult { + let controller = >::get(stash).ok_or(Error::::NotStash)?; + + >::remove(stash); + >::remove(&controller); + + frame_system::Pallet::::dec_consumers(stash); + + Ok(()) + } + + // Get the current era. + fn get_current_era() -> EraIndex { + ((T::TimeProvider::now().as_millis() - TIME_START_MS) / ERA_DURATION_MS) + .try_into() + .unwrap() + } + } +} From b77cb01928a0b1e1bafab784d592fd4685e945f1 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Fri, 12 May 2023 19:28:07 +0200 Subject: [PATCH 172/192] fix code quality --- bin/node/runtime/src/lib.rs | 4 ++-- frame/ddc-accounts/src/lib.rs | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 7d4cd0b189165..a0e532bb19853 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1235,14 +1235,14 @@ impl pallet_ddc_staking::Config for Runtime { } parameter_types! { - pub const DDC_ACCOUNTS_PALLET_ID: PalletId = PalletId(*b"accounts"); // DDC maintainer's stake + pub const Ddc_Accounts_Pallet_Id: PalletId = PalletId(*b"accounts"); // DDC maintainer's stake } impl pallet_ddc_accounts::Config for Runtime { type BondingDuration = BondingDuration; type Currency = Balances; type Event = Event; - type PalletId = DDC_ACCOUNTS_PALLET_ID; + type PalletId = Ddc_Accounts_Pallet_Id; type TimeProvider = pallet_timestamp::Pallet; } diff --git a/frame/ddc-accounts/src/lib.rs b/frame/ddc-accounts/src/lib.rs index 2b0dca3eb875d..78f07c5252754 100644 --- a/frame/ddc-accounts/src/lib.rs +++ b/frame/ddc-accounts/src/lib.rs @@ -5,7 +5,7 @@ use codec::{Decode, Encode, HasCompact}; use frame_support::{ parameter_types, - traits::{Currency, DefensiveSaturating, LockIdentifier, WithdrawReasons, ExistenceRequirement, UnixTime}, + traits::{Currency, DefensiveSaturating, ExistenceRequirement, UnixTime}, BoundedVec, PalletId }; @@ -155,6 +155,8 @@ pub mod pallet { NotController, /// Not a stash account. NotStash, + /// Stash is already bonded. + AlreadyBonded, /// Controller is already paired. AlreadyPaired, /// Cannot deposit dust @@ -182,6 +184,10 @@ pub mod pallet { ) -> DispatchResult { let stash = ensure_signed(origin)?; + if >::contains_key(&stash) { + Err(Error::::AlreadyBonded)? + } + let controller = T::Lookup::lookup(controller)?; if >::contains_key(&controller) { @@ -195,12 +201,14 @@ pub mod pallet { frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; + >::insert(&stash, &controller); + let stash_balance = T::Currency::free_balance(&stash); let value = value.min(stash_balance); Self::deposit_event(Event::::Deposited(stash.clone(), value)); let item = AccountsLedger { stash: stash.clone(), total: value, active: value, unlocking: Default::default() }; - Self::update_ledger_and_deposit( &stash, &controller, &item); + Self::update_ledger_and_deposit( &stash, &controller, &item)?; Ok(()) } @@ -230,7 +238,7 @@ pub mod pallet { Error::::InsufficientDeposit ); - Self::update_ledger_and_deposit(&stash, &controller, &ledger); + Self::update_ledger_and_deposit(&stash, &controller, &ledger)?; Self::deposit_event(Event::::Deposited(stash.clone(), extra)); @@ -369,7 +377,7 @@ pub mod pallet { stash: &T::AccountId, controller: &T::AccountId, ledger: &AccountsLedger>, - ) { + ) -> DispatchResult { let account_id = Self::account_id(); T::Currency::transfer( @@ -379,6 +387,8 @@ pub mod pallet { ExistenceRequirement::KeepAlive, )?; >::insert(controller, ledger); + + Ok(()) } /// Update the ledger for a controller. From 1b95ff99814dea2fdaca988eb810cf146845f111 Mon Sep 17 00:00:00 2001 From: Raid Ateir Date: Mon, 15 May 2023 12:23:19 +0200 Subject: [PATCH 173/192] add existential balance for pallet accounts --- bin/node/cli/src/chain_spec.rs | 1 + bin/node/testing/src/genesis.rs | 1 + frame/ddc-accounts/src/lib.rs | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index b992c97961718..2af6af29703c2 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -360,6 +360,7 @@ pub fn testnet_genesis( }, vesting: Default::default(), transaction_payment: Default::default(), + ddc_accounts: Default::default(), } } diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index bfae67285d5d2..961ed3fe87564 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -89,5 +89,6 @@ pub fn config_endowed(code: Option<&[u8]>, extra_endowed: Vec) -> Gen society: SocietyConfig { members: vec![alice(), bob()], pot: 0, max_members: 999 }, vesting: Default::default(), transaction_payment: Default::default(), + ddc_accounts: Default::default(), } } diff --git a/frame/ddc-accounts/src/lib.rs b/frame/ddc-accounts/src/lib.rs index 78f07c5252754..62acb3a827ec9 100644 --- a/frame/ddc-accounts/src/lib.rs +++ b/frame/ddc-accounts/src/lib.rs @@ -167,6 +167,27 @@ pub mod pallet { BadState, } + #[pallet::genesis_config] + pub struct GenesisConfig; + + #[cfg(feature = "std")] + impl Default for GenesisConfig { + fn default() -> Self { + Self + } + } + + #[pallet::genesis_build] + impl GenesisBuild for GenesisConfig { + fn build(&self) { + let account_id = >::account_id(); + let min = T::Currency::minimum_balance(); + if T::Currency::free_balance(&account_id) < min { + let _ = T::Currency::make_free_balance_be(&account_id, min); + } + } + } + #[pallet::call] impl Pallet { /// Take the origin account as a stash and lock up `value` of its balance. `controller` will From 5e65bb26038ccca710cb065819dd8f34d12c914b Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Mon, 15 May 2023 09:58:37 +0200 Subject: [PATCH 174/192] Validate once per era --- frame/ddc-validator/src/lib.rs | 37 ++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 36a708d99c1f2..bc10d8ea9ddb1 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -248,31 +248,34 @@ pub mod pallet { } let current_era = Self::get_current_era(); + let last_managed_era = Self::last_managed_era().unwrap(); let data_provider_url = Self::get_data_provider_url(); log::info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); - let mock_data_url = Self::get_mock_data_url(); + if current_era > last_managed_era { + let mock_data_url = Self::get_mock_data_url(); - let file_request = dac::fetch_file_request(&mock_data_url); - let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); - log::info!("Proved bytes sum: {:?}", bytes_sum); + let file_request = dac::fetch_file_request(&mock_data_url); + let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); + log::info!("Proved bytes sum: {:?}", bytes_sum); - let assigned_edge = - String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); - let validations_res = - dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) - .unwrap(); - let final_res = dac::get_final_decision(validations_res); + let assigned_edge = + String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); + let validations_res = + dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) + .unwrap(); + let final_res = dac::get_final_decision(validations_res); - let signer = Self::get_signer().unwrap(); + let signer = Self::get_signer().unwrap(); - let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { - era: current_era, - cdn_node: utils::string_to_account::(assigned_edge.clone()), - validation_decision: final_res.clone(), - }); + let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { + era: current_era, + cdn_node: utils::string_to_account::(assigned_edge.clone()), + validation_decision: final_res.clone(), + }); - log::info!("final_res: {:?}", final_res); + log::info!("final_res: {:?}", final_res); + } // Print the number of broken sessions per CDN node. let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data From 8327f5fc37428c8da1c3322e234fbf476ecbf590 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Tue, 16 May 2023 16:06:00 +0600 Subject: [PATCH 175/192] Return early if no `dacv` key found, don't panic --- frame/ddc-validator/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index bc10d8ea9ddb1..671556d5f025e 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -266,7 +266,13 @@ pub mod pallet { .unwrap(); let final_res = dac::get_final_decision(validations_res); - let signer = Self::get_signer().unwrap(); + let signer = match Self::get_signer() { + Ok(signer) => signer, + Err(e) => { + log::info!("{}", e); + return + }, + }; let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { era: current_era, From 6b0e30e28bf7dfe9d64d07bfdaf21bdca77b7f92 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 12:33:25 +0600 Subject: [PATCH 176/192] Use `log` crate in pallet-ddc-staking --- Cargo.lock | 1 + frame/ddc-staking/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 600c56f222904..68de650126d5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5874,6 +5874,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", + "log 0.4.17", "parity-scale-codec", "scale-info", "sp-io", diff --git a/frame/ddc-staking/Cargo.toml b/frame/ddc-staking/Cargo.toml index 0e4beac59b0f9..97c02a71551a5 100644 --- a/frame/ddc-staking/Cargo.toml +++ b/frame/ddc-staking/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +log = { version = "0.4.17", default-features = false } scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } From 644add66d6cd3e946403dc042fb960d04099d9e3 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 12:34:13 +0600 Subject: [PATCH 177/192] Add `pallet-ddc-accounts` to workspace --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index dd98b83860671..26444267aa22b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ members = [ "frame/election-provider-support/benchmarking", "frame/election-provider-support/solution-type", "frame/election-provider-support/solution-type/fuzzer", + "frame/ddc-accounts", "frame/ddc-metrics-offchain-worker", "frame/ddc-pallet", "frame/ddc-staking", From 7a085f81de32a11fa9fbc516fa0369c857cae915 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 14:21:59 +0600 Subject: [PATCH 178/192] Pass CDN payout source account to ddc-staking --- bin/node/runtime/src/lib.rs | 1 + frame/ddc-staking/src/lib.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a0e532bb19853..efb01ba9566de 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1231,6 +1231,7 @@ impl pallet_ddc_staking::Config for Runtime { type BondingDuration = BondingDuration; type Currency = Balances; type Event = Event; + type StakersPayoutSource = Ddc_Accounts_Pallet_Id; type WeightInfo = pallet_ddc_staking::weights::SubstrateWeight; } diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 6f09cebdecaf9..0350c6846be4b 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -15,7 +15,7 @@ use codec::{Decode, Encode, HasCompact}; use frame_support::{ parameter_types, traits::{Currency, DefensiveSaturating, LockIdentifier, WithdrawReasons}, - BoundedVec, + BoundedVec, PalletId, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -155,6 +155,8 @@ pub mod pallet { /// Number of eras that staked funds must remain bonded for. #[pallet::constant] type BondingDuration: Get; + /// To derive an account for withdrawing CDN rewards. + type StakersPayoutSource: Get; /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } From f05d62ca3103ced24dd98e13e2695d340ddc739d Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 14:24:26 +0600 Subject: [PATCH 179/192] Print CDN payout source account in debug log --- frame/ddc-staking/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index 0350c6846be4b..ecf878900ed04 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -19,7 +19,7 @@ use frame_support::{ }; use scale_info::TypeInfo; use sp_runtime::{ - traits::{AtLeast32BitUnsigned, CheckedSub, Saturating, Zero}, + traits::{AccountIdConversion, AtLeast32BitUnsigned, CheckedSub, Saturating, Zero}, RuntimeDebug, }; @@ -581,6 +581,8 @@ pub mod pallet { impl Pallet { pub(super) fn do_payout_stakers(era: EraIndex) -> DispatchResult { + let source: T::AccountId = T::StakersPayoutSource::get().into_account(); + log::debug!("Will payout to DDC stakers from {:?} account for era {:?}", source, era); Ok(()) } From a8cd20c918a6632f3a8dcd32a119c5a7e1d155ea Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 15:53:54 +0600 Subject: [PATCH 180/192] Don't panic in absence of `LastManagedEra` value --- frame/ddc-validator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 671556d5f025e..e915963940b10 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -248,7 +248,7 @@ pub mod pallet { } let current_era = Self::get_current_era(); - let last_managed_era = Self::last_managed_era().unwrap(); + let last_managed_era = Self::last_managed_era().unwrap_or(0); let data_provider_url = Self::get_data_provider_url(); log::info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); From 73e0371d18c76d12f9b52d79d548a4158f39b64a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 17 May 2023 13:37:48 +0200 Subject: [PATCH 181/192] Remove unused code from pallet-staking --- frame/staking/src/lib.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 61f093c8b26bd..037dcc86ab2a8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -380,20 +380,6 @@ impl Default for EraRewardPoints { } } -#[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] -pub struct EraDacRewardPoints { - /// Total number of points. Equals the sum of reward points for each validator. - total: RewardPoint, - /// The reward points earned by a given validator. - individual: BTreeMap, -} - -impl Default for EraDacRewardPoints { - fn default() -> Self { - EraDacRewardPoints { total: Default::default(), individual: BTreeMap::new() } - } -} - /// Indicates the initial status of the staker. #[derive(RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, Clone))] From 1e591a7a65bb685ab7374b9d2e9f302fe95c98e8 Mon Sep 17 00:00:00 2001 From: "Alisher A. Khassanov" Date: Wed, 17 May 2023 18:45:12 +0600 Subject: [PATCH 182/192] Implement payout to DDC stakers --- frame/ddc-staking/src/lib.rs | 51 +++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/frame/ddc-staking/src/lib.rs b/frame/ddc-staking/src/lib.rs index ecf878900ed04..ced37c72a13c0 100644 --- a/frame/ddc-staking/src/lib.rs +++ b/frame/ddc-staking/src/lib.rs @@ -14,13 +14,15 @@ use codec::{Decode, Encode, HasCompact}; use frame_support::{ parameter_types, - traits::{Currency, DefensiveSaturating, LockIdentifier, WithdrawReasons}, + traits::{ + Currency, DefensiveSaturating, ExistenceRequirement, LockIdentifier, WithdrawReasons, + }, BoundedVec, PalletId, }; use scale_info::TypeInfo; use sp_runtime::{ traits::{AccountIdConversion, AtLeast32BitUnsigned, CheckedSub, Saturating, Zero}, - RuntimeDebug, + Perbill, RuntimeDebug, }; use sp_staking::EraIndex; @@ -581,8 +583,49 @@ pub mod pallet { impl Pallet { pub(super) fn do_payout_stakers(era: EraIndex) -> DispatchResult { - let source: T::AccountId = T::StakersPayoutSource::get().into_account(); - log::debug!("Will payout to DDC stakers from {:?} account for era {:?}", source, era); + // ToDo: check that the era is finished + // ToDo: check reward points are set + + // An account we withdraw the funds from and the amount of funds to withdraw. + let payout_source_account: T::AccountId = T::StakersPayoutSource::get().into_account(); + let payout_budget = T::Currency::free_balance(&payout_source_account); + let era_reward_points: EraRewardPoints = + >::get(&era); + log::debug!( + "Will payout to DDC stakers for era {:?} from account {:?} with total budget {:?} \ + , there are {:?} stakers earned {:?} reward points", + era, + payout_source_account, + payout_budget, + era_reward_points.individual.len(), + era_reward_points.total, + ); + + // Transfer a part of the budget to each CDN participant rewarded this era. + for (stash, points) in era_reward_points.individual { + let part = Perbill::from_rational(points, era_reward_points.total); + let reward: BalanceOf = part * payout_budget; + log::debug!( + "Rewarding {:?} with {:?} points, its part is {:?}, reward size {:?}, balance \ + on payout source account {:?}", + stash, + points, + part, + reward, + T::Currency::free_balance(&payout_source_account) + ); + T::Currency::transfer( + &payout_source_account, + &stash, + reward, + ExistenceRequirement::AllowDeath, + )?; // ToDo: all success or noop + } + log::debug!( + "Balance left on payout source account {:?}", + T::Currency::free_balance(&payout_source_account), + ); + Ok(()) } From 74029a265e20ca3e11d0409d26b5964181c8a34e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 17 May 2023 13:37:48 +0200 Subject: [PATCH 183/192] Remove unused code from pallet-staking --- frame/staking/src/pallet/impls.rs | 39 ++----------------------------- frame/staking/src/pallet/mod.rs | 13 ++++------- 2 files changed, 6 insertions(+), 46 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 7485765ae0b3a..f6d60a27b3fcd 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -16,13 +16,12 @@ // limitations under the License. //! Implementations for the Staking FRAME Pallet. + use frame_election_provider_support::{ data_provider, ElectionDataProvider, ElectionProvider, ScoreProvider, SortedListProvider, Supports, VoteWeight, VoterOf, }; use frame_support::{ - decl_event, decl_module, decl_storage, - log::{error, info, warn}, pallet_prelude::*, traits::{ Currency, CurrencyToVote, Defensive, EstimateNextNewSession, Get, Imbalance, @@ -45,7 +44,7 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, Exposure, ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, - SessionInterface, StakingLedger, ValidatorPrefs, EraDacRewardPoints, + SessionInterface, StakingLedger, ValidatorPrefs, }; use super::{pallet::*, STAKING_ID}; @@ -628,40 +627,6 @@ impl Pallet { era_rewards.total += points; } }); - let erasRewardPoints = >::get(active_era.index); - } - } - - pub fn reward_by_ids2(validators_points: impl IntoIterator) { - if let Some(active_era) = Self::active_era() { - >::mutate(active_era.index, |era_rewards| { - let mut p = EraDacRewardPoints { - total: era_rewards.total.clone(), - individual: era_rewards.individual.clone(), - }; - for (validator, points) in validators_points.into_iter() { - *p.individual.entry(validator).or_default() += points; - p.total += points; - } - return p; - }); - let dacRewardPoints = >::get(active_era.index); - } - } - - pub fn get_reward_points(validators_points: impl IntoIterator) { - info!("Hello from get_reward_points"); - if let Some(active_era) = Self::active_era() { - >::mutate(active_era.index, |era_rewards| { - for (validator, points) in validators_points.into_iter() { - // info!("validator: {:?}, points: {:?}", validator, points); - // *era_rewards.individual.entry(validator).or_default() += points; - info!("reading era_rewards.total: {:?}", era_rewards.total); - // era_rewards.total += points; - } - }); - let dacRewardPoints = >::get(active_era.index); - info!("reading dacRewardPoints: {:?}", dacRewardPoints); } } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 65058d6b3998d..5af51be85e3c7 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -40,10 +40,10 @@ mod impls; pub use impls::*; use crate::{ - slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraDacRewardPoints, EraPayout, - EraRewardPoints, Exposure, Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, - PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakingLedger, - UnappliedSlash, UnlockChunk, ValidatorPrefs, + slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, EraRewardPoints, Exposure, + Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, Releases, + RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, + ValidatorPrefs, }; const STAKING_ID: LockIdentifier = *b"staking "; @@ -399,11 +399,6 @@ pub mod pallet { pub type ErasRewardPoints = StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; - #[pallet::storage] - #[pallet::getter(fn eras_dac_reward_points)] - pub type ErasDacRewardPoints = - StorageMap<_, Twox64Concat, EraIndex, EraDacRewardPoints, ValueQuery>; - /// The total amount staked for the last `HISTORY_DEPTH` eras. /// If total hasn't been set or has been removed then 0 stake is returned. #[pallet::storage] From 085c0802a7556ce7b7412cdeeb76884222905f13 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Tue, 16 May 2023 17:29:53 +0200 Subject: [PATCH 184/192] Implement validate_edges --- frame/ddc-validator/src/lib.rs | 97 +++++++++++++++++++++++----------- frame/system/src/offchain.rs | 4 ++ 2 files changed, 71 insertions(+), 30 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index e915963940b10..6c3372f336dc5 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -252,36 +252,41 @@ pub mod pallet { let data_provider_url = Self::get_data_provider_url(); log::info!("[DAC Validator] Data provider URL: {:?}", &data_provider_url); - if current_era > last_managed_era { - let mock_data_url = Self::get_mock_data_url(); - - let file_request = dac::fetch_file_request(&mock_data_url); - let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); - log::info!("Proved bytes sum: {:?}", bytes_sum); - - let assigned_edge = - String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); - let validations_res = - dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) - .unwrap(); - let final_res = dac::get_final_decision(validations_res); - - let signer = match Self::get_signer() { - Ok(signer) => signer, - Err(e) => { - log::info!("{}", e); - return - }, - }; - - let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { - era: current_era, - cdn_node: utils::string_to_account::(assigned_edge.clone()), - validation_decision: final_res.clone(), - }); - - log::info!("final_res: {:?}", final_res); - } + // `If` commented for testing purposes + // if current_era > last_managed_era { + Self::validate_edges(); + //} + + // if current_era > last_managed_era { + // let mock_data_url = Self::get_mock_data_url(); + // + // let file_request = dac::fetch_file_request(&mock_data_url); + // let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); + // log::info!("Proved bytes sum: {:?}", bytes_sum); + // + // let assigned_edge = + // String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); + // let validations_res = + // dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) + // .unwrap(); + // let final_res = dac::get_final_decision(validations_res); + // + // let signer = match Self::get_signer() { + // Ok(signer) => signer, + // Err(e) => { + // log::info!("{}", e); + // return + // }, + // }; + // + // let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { + // era: current_era, + // cdn_node: utils::string_to_account::(assigned_edge.clone()), + // validation_decision: final_res.clone(), + // }); + // + // log::info!("final_res: {:?}", final_res); + // } // Print the number of broken sessions per CDN node. let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data @@ -677,5 +682,37 @@ pub mod pallet { random_number } + + fn validate_edges() { + let current_era = Self::get_current_era(); + let mock_data_url = Self::get_mock_data_url(); + let data_provider_url = Self::get_data_provider_url(); + + let signer = Self::get_signer().unwrap(); + let account = signer.get_any_account().unwrap(); + + let assigned_edges = Self::assignments(current_era, account.id).unwrap(); + + for assigned_edge in assigned_edges.iter() { + let file_request = dac::fetch_file_request(&mock_data_url); + let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); + let edge = utils::account_to_string::(assigned_edge.clone()); + + let validations_res = + dac::get_validation_results(&data_provider_url, current_era, &edge) + .unwrap(); + let final_res = dac::get_final_decision(validations_res); + + let signer = Self::get_signer().unwrap(); + + let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { + era: current_era, + cdn_node: utils::string_to_account::(edge.clone()), + validation_decision: final_res.clone(), + }); + + log::info!("final_res: {:?}", final_res); + } + } } } diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index bf52ab8e3791d..5cbe5b10a566c 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -135,6 +135,10 @@ impl, X> Signer Default::default() } + pub fn get_any_account(self) -> Option> { + self.accounts_from_keys().nth(0) + } + /// Use provided `accounts` for signing. /// /// Note that not all keys will be necessarily used. The provided From 02a6db48f759c4bd88d202ad60f7e65bd19f6db1 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 17 May 2023 13:32:11 +0200 Subject: [PATCH 185/192] Integrate job assignment --- frame/ddc-validator/src/dac.rs | 17 +++--- frame/ddc-validator/src/lib.rs | 92 ++++++++++++++++++++++++-------- frame/ddc-validator/src/shm.rs | 11 ++++ frame/ddc-validator/src/utils.rs | 16 +++--- 4 files changed, 101 insertions(+), 35 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 68442f1b14abd..a57ea2128f812 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -273,21 +273,24 @@ fn get_timestamps_with_ack(file_requests: &Requests) -> Vec { timestamps } -pub fn get_proved_delivered_bytes_sum(file_requests: &Requests) -> u64 { +pub fn get_served_bytes_sum(file_requests: &Requests) -> (u64, u64) { let ack_timestamps = get_timestamps_with_ack(file_requests); let mut total_bytes_received = 0u64; + let mut total_bytes_sent = 0u64; for (_, file_request) in file_requests { for (_, chunk) in &file_request.chunks { + total_bytes_sent += chunk.log.bytes_sent; + if let Some(ack) = &chunk.ack { - total_bytes_received += &chunk.log.bytes_sent; + total_bytes_received += ack.bytes_received; } else { total_bytes_received += get_proved_delivered_bytes(chunk, &ack_timestamps); } } } - total_bytes_received + (total_bytes_sent, total_bytes_received) } fn get_proved_delivered_bytes(chunk: &Chunk, ack_timestamps: &Vec) -> u64 { @@ -409,7 +412,7 @@ fn get_bytes_received_query_url(data_provider_url: &String, era: EraIndex) -> St format!("{}/FT.AGGREGATE/ddc:dac:searchCommonIndex/@era:[{}%20{}]/GROUPBY/2/@nodePublicKey/@era/REDUCE/SUM/1/@bytesReceived/AS/bytesReceivedSum", data_provider_url, era, era) } -fn http_get_json(url: &str) -> crate::ResultStr { +pub(crate) fn http_get_json(url: &str) -> crate::ResultStr { let body = http_get_request(url).map_err(|err| { log::error!("[DAC Validator] Error while getting {}: {:?}", url, err); "HTTP GET error" @@ -544,9 +547,11 @@ pub(crate) fn get_final_decision(decisions: Vec) -> Validation let common_decisions = find_largest_group(decisions).unwrap(); let decision_example = common_decisions.get(0).unwrap(); + let serialized_decisions = serde_json::to_string(&common_decisions).unwrap(); + let final_decision = ValidationDecision { result: decision_example.result, - payload: utils::get_hashed(&common_decisions), + payload: utils::hash(&serialized_decisions), totals: DacTotalAggregates { received: decision_example.received, sent: decision_example.sent, @@ -558,7 +563,7 @@ pub(crate) fn get_final_decision(decisions: Vec) -> Validation final_decision } -pub(crate) fn get_validation_results( +pub(crate) fn fetch_validation_results( data_provider_url: &String, era: EraIndex, edge: &String, diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 6c3372f336dc5..16b7f8041f90f 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -77,9 +77,11 @@ pub const ERA_IN_BLOCKS: u8 = 20; // pub const DEFAULT_DATA_PROVIDER_URL: &str = "https://dev-dac-redis.network-dev.aws.cere.io"; pub const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; pub const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; +pub const QUORUM_SIZE: usize = 3; /// Aggregated values from DAC that describe CDN node's activity during a certain era. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] pub struct DacTotalAggregates { /// Total bytes received by the client. pub received: u64, @@ -92,12 +94,13 @@ pub struct DacTotalAggregates { } /// Final DAC Validation decision. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, Serialize, Deserialize)] +#[serde(crate = "alt_serde")] pub struct ValidationDecision { /// Validation result. pub result: bool, /// A hash of the data used to produce validation result. - pub payload: [u8; 256], + pub payload: [u8; 32], /// Values aggregated from the payload. pub totals: DacTotalAggregates, } @@ -236,7 +239,7 @@ pub mod pallet { } >::put(era); - Self::assign(3usize); + Self::assign(QUORUM_SIZE); 0 } @@ -372,7 +375,7 @@ pub mod pallet { // Prepare an intermediate validation decision let validation_decision = ValidationDecision { result: validation_result, - payload: [0u8; 256], // ToDo: put a hash of the validated data here + payload: [0u8; 32], // ToDo: put a hash of the validated data here totals: DacTotalAggregates { sent: node_sent.sum as u64, received: client_received.sum as u64, @@ -430,7 +433,7 @@ pub mod pallet { ), ValidationDecision { result: true, - payload: [0u8; 256], + payload: [0u8; 32], totals: DacTotalAggregates { sent: 100, received: 100, @@ -445,7 +448,7 @@ pub mod pallet { ), ValidationDecision { result: true, - payload: [0u8; 256], + payload: [0u8; 32], totals: DacTotalAggregates { sent: 200, received: 200, @@ -601,6 +604,18 @@ pub mod pallet { } } + fn is_valid(bytes_sent: u64, bytes_received: u64) -> bool { + let percentage_difference = 1f32 - (bytes_received as f32 / bytes_sent as f32); + + return if percentage_difference > 0.0 && + (T::ValidationThreshold::get() as f32 - percentage_difference) > 0.0 + { + true + } else { + false + } + } + fn shuffle(mut list: Vec) -> Vec { let len = list.len(); for i in 1..len { @@ -689,29 +704,64 @@ pub mod pallet { let data_provider_url = Self::get_data_provider_url(); let signer = Self::get_signer().unwrap(); - let account = signer.get_any_account().unwrap(); + let validator = signer.get_any_account().unwrap().id; - let assigned_edges = Self::assignments(current_era, account.id).unwrap(); + let assigned_edges = Self::assignments(current_era, validator.clone()).unwrap(); for assigned_edge in assigned_edges.iter() { let file_request = dac::fetch_file_request(&mock_data_url); - let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); + let (bytes_sent, bytes_received) = dac::get_served_bytes_sum(&file_request); + let is_valid = Self::is_valid(bytes_sent, bytes_received); + + let payload = serde_json::to_string(&file_request).unwrap(); + let decision = ValidationDecision { + result: is_valid, + payload: utils::hash(&payload), + totals: DacTotalAggregates { + received: bytes_received, + sent: bytes_sent, + failed_by_client: 0, + failure_rate: 0, + } + }; + + let serialized_decision = serde_json::to_string(&decision).unwrap(); + let encoded_decision = shm::base64_encode(&serialized_decision.as_bytes().to_vec()); + let validator_str = utils::account_to_string::(validator.clone()); + let edge_str = utils::account_to_string::(assigned_edge.clone()); + + let encoded_decision_str = encoded_decision.iter().cloned().collect::(); + + let response = shm::share_intermediate_validation_result( + &data_provider_url, + current_era - 1, + &validator_str, + &edge_str, + is_valid, + &encoded_decision_str, + ); + let edge = utils::account_to_string::(assigned_edge.clone()); - let validations_res = - dac::get_validation_results(&data_provider_url, current_era, &edge) - .unwrap(); - let final_res = dac::get_final_decision(validations_res); + if let Ok(res) = response { + let validations_res = + dac::fetch_validation_results(&data_provider_url, current_era, &edge) + .unwrap(); - let signer = Self::get_signer().unwrap(); + if validations_res.len() == QUORUM_SIZE { + let final_res = dac::get_final_decision(validations_res); - let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { - era: current_era, - cdn_node: utils::string_to_account::(edge.clone()), - validation_decision: final_res.clone(), - }); + let signer = Self::get_signer().unwrap(); + + let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { + era: current_era, + cdn_node: utils::string_to_account::(edge.clone()), + validation_decision: final_res.clone(), + }); - log::info!("final_res: {:?}", final_res); + log::info!("final_res: {:?}", final_res); + } + } } } } diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index 777f31d1169d4..a20d1c70c39b1 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -11,6 +11,7 @@ use lite_json::json::JsonValue; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; use sp_std::prelude::*; +use crate::{dac, ValidationDecision}; const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -61,3 +62,13 @@ pub fn share_intermediate_validation_result( })?; Ok(json) } + +// pub(crate) fn fetch_intermediate_decisions(data_provider_url: &String, era: EraIndex) -> Vec { +// let url = "JSON.GET ddc:dac:shared:nodes:89731"; +// +// format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", data_provider_url, era); +// +// dac::http_get_json() +// +// +// } diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs index 3c5c90c8028c9..173d50f39b954 100644 --- a/frame/ddc-validator/src/utils.rs +++ b/frame/ddc-validator/src/utils.rs @@ -28,11 +28,11 @@ pub(crate) fn hash(data: &String) -> [u8; 32] { result } -pub(crate) fn get_hashed(data: &Vec) -> [u8; 256] { - let results_log = serde_json::to_string(data).unwrap(); - let mut payload: [u8; 256] = [0; 256]; - let hashed_results = hash(&results_log); - payload[..32].copy_from_slice(&hashed_results); - - payload -} +// pub(crate) fn get_hashed(data: &Vec) -> [u8; 256] { +// let results_log = serde_json::to_string(data).unwrap(); +// let mut payload: [u8; 256] = [0; 256]; +// let hashed_results = hash(&results_log); +// payload[..32].copy_from_slice(&hashed_results); +// +// payload +// } From cc7fd4c33dcf3ab9513f660cc03056f40c863078 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Wed, 17 May 2023 20:23:28 +0200 Subject: [PATCH 186/192] Implement fetch and decode intermediate validation results --- frame/ddc-validator/src/lib.rs | 13 ++++++++ frame/ddc-validator/src/shm.rs | 57 ++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 16b7f8041f90f..16949111aaf5a 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -698,6 +698,19 @@ pub mod pallet { random_number } + fn find_validators_from_quorum(validator_id: T::AccountId, era: EraIndex) -> Vec { + let validator_edges = Self::assignments(era, validator_id).unwrap(); + let mut quorum_members: Vec = Vec::new(); + + >::iter_prefix(era).for_each(|(candidate_id, edges)| { + if validator_edges == edges { + quorum_members.push(candidate_id); + } + }); + + quorum_members + } + fn validate_edges() { let current_era = Self::get_current_era(); let mock_data_url = Self::get_mock_data_url(); diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index a20d1c70c39b1..66dabb1d266b9 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -5,16 +5,39 @@ //! transactions pool or peers list in the future, but for now it works on the centralized Redis //! server which we maintain for DAC DataModel. -use alloc::{format, string::String}; // ToDo: remove String usage +use alloc::{format, string::String}; +pub use sp_std::{collections::btree_map::BTreeMap}; +// ToDo: remove String usage use base64::prelude::*; use lite_json::json::JsonValue; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; use sp_std::prelude::*; use crate::{dac, ValidationDecision}; +use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; const HTTP_TIMEOUT_MS: u64 = 30_000; +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +pub(crate) struct IntermediateDecisions { + validators_to_decisions: BTreeMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(crate = "alt_serde")] +struct IntermediateDecision { + result: bool, + data: String, +} + +pub fn base64_decode(input: &String) -> Vec { + let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity + buf.resize(1024, 0); + BASE64_STANDARD.decode_slice(input, &mut buf).unwrap(); // ToDo: handle error + buf.iter().map(|&char| char as u8).collect() +} + /// Encodes a vector of bytes into a vector of characters using base64 encoding. pub fn base64_encode(input: &Vec) -> Vec { let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity @@ -63,12 +86,26 @@ pub fn share_intermediate_validation_result( Ok(json) } -// pub(crate) fn fetch_intermediate_decisions(data_provider_url: &String, era: EraIndex) -> Vec { -// let url = "JSON.GET ddc:dac:shared:nodes:89731"; -// -// format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", data_provider_url, era); -// -// dac::http_get_json() -// -// -// } +pub(crate) fn get_intermediate_decisions(data_provider_url: &String, era: EraIndex) -> Vec { + let url = format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", data_provider_url, era); + + let decisions: IntermediateDecisions = dac::http_get_json(url.as_str()).unwrap(); + let decoded_decisions = decode_intermediate_decisions(decisions); + + decoded_decisions +} + +pub(crate) fn decode_intermediate_decisions(decisions: IntermediateDecisions) -> Vec { + let mut decoded_decisions: Vec = Vec::new(); + + for (_, decision) in decisions.validators_to_decisions.iter() { + let data = base64_decode(&decision.data); + + let data_str = String::from_utf8_lossy(&data); + let decoded_decision: ValidationDecision = serde_json::from_str(&data_str).unwrap(); + + decoded_decisions.push(decoded_decision); + } + + decoded_decisions +} From 90335ad2b251507fae3bbf953439d3317f72a320 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 18 May 2023 13:03:36 +0200 Subject: [PATCH 187/192] Integrate all parts --- frame/ddc-validator/src/dac.rs | 18 ++++++++++-------- frame/ddc-validator/src/lib.rs | 22 ++++++++++++++-------- frame/ddc-validator/src/shm.rs | 24 +++++++++++++++++++----- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index a57ea2128f812..855fcfad23a2b 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -543,18 +543,19 @@ pub(crate) fn post_final_decision( res } -pub(crate) fn get_final_decision(decisions: Vec) -> ValidationDecision { +pub(crate) fn get_final_decision(decisions: Vec) -> ValidationDecision { let common_decisions = find_largest_group(decisions).unwrap(); let decision_example = common_decisions.get(0).unwrap(); let serialized_decisions = serde_json::to_string(&common_decisions).unwrap(); let final_decision = ValidationDecision { + edge: decision_example.edge.clone(), result: decision_example.result, payload: utils::hash(&serialized_decisions), totals: DacTotalAggregates { - received: decision_example.received, - sent: decision_example.sent, + received: decision_example.totals.received, + sent: decision_example.totals.sent, failed_by_client: 0, failure_rate: 0, }, @@ -575,8 +576,8 @@ pub(crate) fn fetch_validation_results( Ok(results) } -fn find_largest_group(decisions: Vec) -> Option> { - let mut groups: Vec> = Vec::new(); +fn find_largest_group(decisions: Vec) -> Option> { + let mut groups: Vec> = Vec::new(); let half = decisions.len() / 2; for decision in decisions { @@ -585,12 +586,12 @@ fn find_largest_group(decisions: Vec) -> Option) -> Option(edge.clone()), result: validation_result, payload: [0u8; 32], // ToDo: put a hash of the validated data here totals: DacTotalAggregates { @@ -432,6 +435,7 @@ pub mod pallet { "0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1".into(), ), ValidationDecision { + edge: "test".into(), result: true, payload: [0u8; 32], totals: DacTotalAggregates { @@ -447,6 +451,7 @@ pub mod pallet { "0xa2d14e71b52e5695e72c0567926bc68b68bda74df5c1ccf1d4ba612c153ff66b".into(), ), ValidationDecision { + edge: "test".into(), result: true, payload: [0u8; 32], totals: DacTotalAggregates { @@ -698,13 +703,14 @@ pub mod pallet { random_number } - fn find_validators_from_quorum(validator_id: T::AccountId, era: EraIndex) -> Vec { - let validator_edges = Self::assignments(era, validator_id).unwrap(); - let mut quorum_members: Vec = Vec::new(); + fn find_validators_from_quorum(validator_id: &T::AccountId, era: &EraIndex) -> Vec { + let validator_edges = Self::assignments(era, &validator_id).unwrap(); + let mut quorum_members: Vec = Vec::new(); >::iter_prefix(era).for_each(|(candidate_id, edges)| { if validator_edges == edges { - quorum_members.push(candidate_id); + let candidate_id_str = utils::account_to_string::(candidate_id); + quorum_members.push(candidate_id_str); } }); @@ -728,6 +734,7 @@ pub mod pallet { let payload = serde_json::to_string(&file_request).unwrap(); let decision = ValidationDecision { + edge: utils::account_to_string::(assigned_edge.clone()), result: is_valid, payload: utils::hash(&payload), totals: DacTotalAggregates { @@ -757,9 +764,8 @@ pub mod pallet { let edge = utils::account_to_string::(assigned_edge.clone()); if let Ok(res) = response { - let validations_res = - dac::fetch_validation_results(&data_provider_url, current_era, &edge) - .unwrap(); + let quorum = Self::find_validators_from_quorum(&validator, ¤t_era); + let validations_res = shm::get_intermediate_decisions(&data_provider_url, ¤t_era, quorum); if validations_res.len() == QUORUM_SIZE { let final_res = dac::get_final_decision(validations_res); diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index 66dabb1d266b9..98409efa65172 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -18,13 +18,13 @@ use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; const HTTP_TIMEOUT_MS: u64 = 30_000; -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] #[serde(crate = "alt_serde")] pub(crate) struct IntermediateDecisions { validators_to_decisions: BTreeMap, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] #[serde(crate = "alt_serde")] struct IntermediateDecision { result: bool, @@ -86,11 +86,12 @@ pub fn share_intermediate_validation_result( Ok(json) } -pub(crate) fn get_intermediate_decisions(data_provider_url: &String, era: EraIndex) -> Vec { +pub(crate) fn get_intermediate_decisions(data_provider_url: &String, era: &EraIndex, quorum: Vec) -> Vec { let url = format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", data_provider_url, era); - let decisions: IntermediateDecisions = dac::http_get_json(url.as_str()).unwrap(); - let decoded_decisions = decode_intermediate_decisions(decisions); + let all_decisions: IntermediateDecisions = dac::http_get_json(url.as_str()).unwrap(); + let quorum_decisions = find_quorum_decisions(all_decisions, quorum); + let decoded_decisions = decode_intermediate_decisions(quorum_decisions); decoded_decisions } @@ -109,3 +110,16 @@ pub(crate) fn decode_intermediate_decisions(decisions: IntermediateDecisions) -> decoded_decisions } + +pub(crate) fn find_quorum_decisions(all_decisions: IntermediateDecisions, quorum: Vec) -> IntermediateDecisions { + let mut quorum_decisions: BTreeMap = BTreeMap::new(); + for (validator_id, decision) in all_decisions.validators_to_decisions.iter() { + if quorum.contains(validator_id) { + quorum_decisions.insert(validator_id.clone(), decision.clone()); + } + } + + IntermediateDecisions { + validators_to_decisions: quorum_decisions + } +} From 1196f0da9fee1b039b644113b917f571e732f4b6 Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 18 May 2023 18:23:34 +0200 Subject: [PATCH 188/192] Add more logs --- frame/ddc-validator/src/dac.rs | 3 + frame/ddc-validator/src/lib.rs | 102 ++++++++++++++------------------- frame/ddc-validator/src/shm.rs | 5 ++ 3 files changed, 50 insertions(+), 60 deletions(-) diff --git a/frame/ddc-validator/src/dac.rs b/frame/ddc-validator/src/dac.rs index 855fcfad23a2b..9f4b325bb53dc 100644 --- a/frame/ddc-validator/src/dac.rs +++ b/frame/ddc-validator/src/dac.rs @@ -345,9 +345,12 @@ fn get_file_request_url(data_provider_url: &String) -> String { } pub(crate) fn fetch_file_request(url: &String) -> Requests { + log::info!("fetch_file_request | url: {:?}", url); let response: FileRequestWrapper = http_get_json(&url).unwrap(); let value: Value = serde_json::from_str(response.json.as_str()).unwrap(); let map: Requests = serde_json::from_value(value).unwrap(); + + log::info!("response.json: {:?}", response.json); map } diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index c6e8135bcab77..45bdaf98d0c60 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -70,14 +70,15 @@ type ResultStr = Result; pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"dacv"); pub const TIME_START_MS: u128 = 1_672_531_200_000; -pub const ERA_DURATION_MS: u128 = 120_000; +// pub const ERA_DURATION_MS: u128 = 120_000; +pub const ERA_DURATION_MS: u128 = 30_000; pub const ERA_IN_BLOCKS: u8 = 20; /// Webdis in experimental cluster connected to Redis in dev. // pub const DEFAULT_DATA_PROVIDER_URL: &str = "https://dev-dac-redis.network-dev.aws.cere.io"; pub const DEFAULT_DATA_PROVIDER_URL: &str = "http://161.35.140.182:7379"; pub const DATA_PROVIDER_URL_KEY: &[u8; 32] = b"ddc-validator::data-provider-url"; -pub const QUORUM_SIZE: usize = 3; +pub const QUORUM_SIZE: usize = 1; /// Aggregated values from DAC that describe CDN node's activity during a certain era. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, Serialize, Deserialize)] @@ -136,6 +137,7 @@ pub mod crypto { #[frame_support::pallet] pub mod pallet { + use log::info; use super::*; #[pallet::pallet] @@ -262,64 +264,33 @@ pub mod pallet { Self::validate_edges(); //} - // if current_era > last_managed_era { - // let mock_data_url = Self::get_mock_data_url(); - // - // let file_request = dac::fetch_file_request(&mock_data_url); - // let bytes_sum = dac::get_proved_delivered_bytes_sum(&file_request); - // log::info!("Proved bytes sum: {:?}", bytes_sum); - // - // let assigned_edge = - // String::from("0xd4160f567d7265b9de2c7cbf1a5c931e5b3195efb2224f8706bfb53ea6eaacd1"); - // let validations_res = - // dac::get_validation_results(&data_provider_url, current_era, &assigned_edge) + // Print the number of broken sessions per CDN node. + // let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data + // let aggregates_obj = aggregates_value.as_object().unwrap(); + // aggregates_obj + // .into_iter() + // .for_each(|(cdn_node_pubkey, cdn_node_aggregates_value)| { + // // iterate over aggregates for each node + // let cdn_node_aggregates_obj = cdn_node_aggregates_value.as_object().unwrap(); + // // Extract `nodeInterruptedSessions` field + // let (_, cdn_node_interrupted_sessions_value) = cdn_node_aggregates_obj + // .into_iter() + // .find(|(key, _)| key.iter().copied().eq("nodeInterruptedSessions".chars())) // .unwrap(); - // let final_res = dac::get_final_decision(validations_res); - // - // let signer = match Self::get_signer() { - // Ok(signer) => signer, - // Err(e) => { - // log::info!("{}", e); - // return - // }, - // }; - // - // let tx_res = signer.send_signed_transaction(|_acct| Call::set_validation_decision { - // era: current_era, - // cdn_node: utils::string_to_account::(assigned_edge.clone()), - // validation_decision: final_res.clone(), + // let cdn_node_interrupted_sessions_obj = + // cdn_node_interrupted_sessions_value.as_object().unwrap(); + // // Prepare CDN pubkey without heap allocated string + // let cdn_node_pubkey_vecu8: Vec = + // cdn_node_pubkey.iter().map(|c| *c as u8).collect(); + // let cdn_node_pubkey_str = + // sp_std::str::from_utf8(&cdn_node_pubkey_vecu8).unwrap(); + // log::info!( + // "Broken sessions per CDN node | Node {}: {} sessions broken", + // cdn_node_pubkey_str, + // cdn_node_interrupted_sessions_obj.len(), /* count sessions broken by the + // * node */ + // ); // }); - // - // log::info!("final_res: {:?}", final_res); - // } - - // Print the number of broken sessions per CDN node. - let aggregates_value = dac::fetch_aggregates(&data_provider_url, 77436).unwrap(); // 77436 is for a mock data - let aggregates_obj = aggregates_value.as_object().unwrap(); - aggregates_obj - .into_iter() - .for_each(|(cdn_node_pubkey, cdn_node_aggregates_value)| { - // iterate over aggregates for each node - let cdn_node_aggregates_obj = cdn_node_aggregates_value.as_object().unwrap(); - // Extract `nodeInterruptedSessions` field - let (_, cdn_node_interrupted_sessions_value) = cdn_node_aggregates_obj - .into_iter() - .find(|(key, _)| key.iter().copied().eq("nodeInterruptedSessions".chars())) - .unwrap(); - let cdn_node_interrupted_sessions_obj = - cdn_node_interrupted_sessions_value.as_object().unwrap(); - // Prepare CDN pubkey without heap allocated string - let cdn_node_pubkey_vecu8: Vec = - cdn_node_pubkey.iter().map(|c| *c as u8).collect(); - let cdn_node_pubkey_str = - sp_std::str::from_utf8(&cdn_node_pubkey_vecu8).unwrap(); - log::info!( - "Broken sessions per CDN node | Node {}: {} sessions broken", - cdn_node_pubkey_str, - cdn_node_interrupted_sessions_obj.len(), /* count sessions broken by the - * node */ - ); - }); // Wait for signal. let signal = Signal::::get().unwrap_or(false); @@ -725,13 +696,17 @@ pub mod pallet { let signer = Self::get_signer().unwrap(); let validator = signer.get_any_account().unwrap().id; - let assigned_edges = Self::assignments(current_era, validator.clone()).unwrap(); + let assigned_edges = Self::assignments(current_era - 1, validator.clone()).unwrap(); + + info!("assigned_edges: {:?}", assigned_edges); for assigned_edge in assigned_edges.iter() { let file_request = dac::fetch_file_request(&mock_data_url); let (bytes_sent, bytes_received) = dac::get_served_bytes_sum(&file_request); let is_valid = Self::is_valid(bytes_sent, bytes_received); + info!("bytes_sent, bytes_received: {:?}, {:?}", bytes_sent, bytes_received); + let payload = serde_json::to_string(&file_request).unwrap(); let decision = ValidationDecision { edge: utils::account_to_string::(assigned_edge.clone()), @@ -745,6 +720,8 @@ pub mod pallet { } }; + info!("decision: {:?}", decision); + let serialized_decision = serde_json::to_string(&decision).unwrap(); let encoded_decision = shm::base64_encode(&serialized_decision.as_bytes().to_vec()); let validator_str = utils::account_to_string::(validator.clone()); @@ -761,12 +738,17 @@ pub mod pallet { &encoded_decision_str, ); - let edge = utils::account_to_string::(assigned_edge.clone()); + if let Err(res) = response.clone() { + log::error!("share_intermediate_validation_result request failed."); + } if let Ok(res) = response { + let edge = utils::account_to_string::(assigned_edge.clone()); let quorum = Self::find_validators_from_quorum(&validator, ¤t_era); let validations_res = shm::get_intermediate_decisions(&data_provider_url, ¤t_era, quorum); + log::info!("get_intermediate_decisions result: {:?}", validations_res); + if validations_res.len() == QUORUM_SIZE { let final_res = dac::get_final_decision(validations_res); diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index 98409efa65172..8bf188aef732e 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -67,6 +67,8 @@ pub fn share_intermediate_validation_result( validation_result_string, validation_decision_string, ); + + log::info!("share_intermediate_validation_result url: {:?}", url); let request = http::Request::get(url.as_str()); let pending = request.deadline(deadline).send().map_err(|_| http::Error::IoError)?; let response = pending.try_wait(deadline).map_err(|_| http::Error::DeadlineReached)??; @@ -79,6 +81,9 @@ pub fn share_intermediate_validation_result( log::warn!("No UTF-8 body"); http::Error::Unknown })?; + + log::info!("body_str: {:?}", body_str); + let json = lite_json::parse_json(body_str).map_err(|_| { log::warn!("No JSON body"); http::Error::Unknown From 1b5a2183dfb5017aac03e0861d44058bd559a02e Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Thu, 18 May 2023 18:37:51 +0200 Subject: [PATCH 189/192] Clean up --- frame/ddc-validator/src/utils.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs index 173d50f39b954..c805c101da9d7 100644 --- a/frame/ddc-validator/src/utils.rs +++ b/frame/ddc-validator/src/utils.rs @@ -27,12 +27,3 @@ pub(crate) fn hash(data: &String) -> [u8; 32] { result } - -// pub(crate) fn get_hashed(data: &Vec) -> [u8; 256] { -// let results_log = serde_json::to_string(data).unwrap(); -// let mut payload: [u8; 256] = [0; 256]; -// let hashed_results = hash(&results_log); -// payload[..32].copy_from_slice(&hashed_results); -// -// payload -// } From 9a8021fa01d40d6ae1328550603ad57ec1bfceec Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 19 May 2023 10:41:51 +0200 Subject: [PATCH 190/192] Get validator pubkey in another way --- frame/ddc-validator/src/lib.rs | 14 ++++++++++++-- frame/system/src/offchain.rs | 4 ---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 45bdaf98d0c60..04cffab55acbf 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -688,13 +688,23 @@ pub mod pallet { quorum_members } + fn get_public_key() -> Option { + match sr25519_public_keys(KEY_TYPE).first() { + Some(pubkey) => Some(T::AccountId::decode(&mut &pubkey.encode()[..]).unwrap()), + None => None + } + } + fn validate_edges() { let current_era = Self::get_current_era(); let mock_data_url = Self::get_mock_data_url(); let data_provider_url = Self::get_data_provider_url(); - let signer = Self::get_signer().unwrap(); - let validator = signer.get_any_account().unwrap().id; + // let signer = Self::get_signer().unwrap(); + // let validator = signer.get_any_account().unwrap().id; + let validator = Self::get_public_key().unwrap(); + + info!("validator: {:?}", validator); let assigned_edges = Self::assignments(current_era - 1, validator.clone()).unwrap(); diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index 5cbe5b10a566c..bf52ab8e3791d 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -135,10 +135,6 @@ impl, X> Signer Default::default() } - pub fn get_any_account(self) -> Option> { - self.accounts_from_keys().nth(0) - } - /// Use provided `accounts` for signing. /// /// Note that not all keys will be necessarily used. The provided From 6f134a521d0ef797115118ad7d467ff1dcd6752f Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 19 May 2023 16:14:20 +0200 Subject: [PATCH 191/192] Fix share_intermediate_validation_result --- frame/ddc-validator/src/lib.rs | 9 +++++++-- frame/ddc-validator/src/shm.rs | 21 ++++++++++++++------ frame/ddc-validator/src/utils.rs | 33 +++++++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index 04cffab55acbf..b48adc717a990 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -749,13 +749,18 @@ pub mod pallet { ); if let Err(res) = response.clone() { - log::error!("share_intermediate_validation_result request failed."); + log::error!("share_intermediate_validation_result request failed: {:?}", res); + } + + if let Ok(res) = response.clone() { + info!("shm res: {:?}", res.to_string()); } if let Ok(res) = response { let edge = utils::account_to_string::(assigned_edge.clone()); let quorum = Self::find_validators_from_quorum(&validator, ¤t_era); - let validations_res = shm::get_intermediate_decisions(&data_provider_url, ¤t_era, quorum); + let prev_era = (current_era - 1) as EraIndex; + let validations_res = shm::get_intermediate_decisions(&data_provider_url, &prev_era, quorum); log::info!("get_intermediate_decisions result: {:?}", validations_res); diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index 8bf188aef732e..b12266fdcd72e 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -13,7 +13,7 @@ use lite_json::json::JsonValue; use sp_runtime::offchain::{http, Duration}; use sp_staking::EraIndex; use sp_std::prelude::*; -use crate::{dac, ValidationDecision}; +use crate::{dac, utils, ValidationDecision}; use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; const HTTP_TIMEOUT_MS: u64 = 30_000; @@ -33,7 +33,7 @@ struct IntermediateDecision { pub fn base64_decode(input: &String) -> Vec { let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity - buf.resize(1024, 0); + buf.resize(392, 0); BASE64_STANDARD.decode_slice(input, &mut buf).unwrap(); // ToDo: handle error buf.iter().map(|&char| char as u8).collect() } @@ -41,7 +41,7 @@ pub fn base64_decode(input: &String) -> Vec { /// Encodes a vector of bytes into a vector of characters using base64 encoding. pub fn base64_encode(input: &Vec) -> Vec { let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity - buf.resize(1024, 0); + buf.resize(392, 0); BASE64_STANDARD.encode_slice(input, &mut buf).unwrap(); // ToDo: handle error buf.iter().map(|&byte| byte as char).collect() } @@ -58,14 +58,23 @@ pub fn share_intermediate_validation_result( let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); let validation_result_string = String::from(if validation_result { "true" } else { "false" }); let validation_decision_string = String::from(validation_decision_encoded); + let json = serde_json::json!({ + "result": validation_result_string, + "data": validation_decision_string, + }); + let json_str = serde_json::to_string(&json).unwrap(); + let unescaped_json = utils::unescape(&json_str); + let url_encoded_json = utils::url_encode(&unescaped_json); + + log::info!("json_str: {:?}", json_str); + let url = format!( - "{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{{\"result\":{},\"data\":\"{}\"}}", + "{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{}", shared_memory_webdis_url, validator, cdn_node, era, - validation_result_string, - validation_decision_string, + url_encoded_json, ); log::info!("share_intermediate_validation_result url: {:?}", url); diff --git a/frame/ddc-validator/src/utils.rs b/frame/ddc-validator/src/utils.rs index c805c101da9d7..36dc4492a3eb4 100644 --- a/frame/ddc-validator/src/utils.rs +++ b/frame/ddc-validator/src/utils.rs @@ -1,5 +1,5 @@ use crate::dac::ValidationResult; -use alloc::string::String; +use alloc::{format, string::String}; use codec::{Decode, Encode}; use sp_core::crypto::AccountId32; use sp_io::hashing::blake2_256; @@ -27,3 +27,34 @@ pub(crate) fn hash(data: &String) -> [u8; 32] { result } + +pub(crate) fn url_encode(input: &str) -> String { + let mut encoded = String::new(); + + for byte in input.bytes() { + match byte { + // Unreserved characters (alphanumeric and -_.~) + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + encoded.push(byte as char); + } + _ => { + encoded.push('%'); + encoded.push_str(&format!("{:02X}", byte)); + } + } + } + + encoded +} + +pub(crate) fn unescape(json: &str) -> String { + let mut result = String::new(); + + for ch in json.chars() { + if ch != '\\' { + result.push(ch); + } + } + + result +} \ No newline at end of file From 7145ad578ca8e9c34e3d5fff22b59e33a5c49b5a Mon Sep 17 00:00:00 2001 From: Maksim Ramanenkau Date: Fri, 19 May 2023 18:12:12 +0200 Subject: [PATCH 192/192] Fix bugs --- frame/ddc-validator/src/lib.rs | 4 ++-- frame/ddc-validator/src/shm.rs | 33 +++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/frame/ddc-validator/src/lib.rs b/frame/ddc-validator/src/lib.rs index b48adc717a990..7440512a791ef 100644 --- a/frame/ddc-validator/src/lib.rs +++ b/frame/ddc-validator/src/lib.rs @@ -758,9 +758,9 @@ pub mod pallet { if let Ok(res) = response { let edge = utils::account_to_string::(assigned_edge.clone()); - let quorum = Self::find_validators_from_quorum(&validator, ¤t_era); let prev_era = (current_era - 1) as EraIndex; - let validations_res = shm::get_intermediate_decisions(&data_provider_url, &prev_era, quorum); + let quorum = Self::find_validators_from_quorum(&validator, &prev_era); + let validations_res = shm::get_intermediate_decisions(&data_provider_url, &edge_str, &prev_era, quorum); log::info!("get_intermediate_decisions result: {:?}", validations_res); diff --git a/frame/ddc-validator/src/shm.rs b/frame/ddc-validator/src/shm.rs index b12266fdcd72e..1fe38e1cca750 100644 --- a/frame/ddc-validator/src/shm.rs +++ b/frame/ddc-validator/src/shm.rs @@ -15,9 +15,18 @@ use sp_staking::EraIndex; use sp_std::prelude::*; use crate::{dac, utils, ValidationDecision}; use alt_serde::{de::DeserializeOwned, Deserialize, Serialize}; +use log::info; const HTTP_TIMEOUT_MS: u64 = 30_000; +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "alt_serde")] +#[serde(rename_all = "camelCase")] +pub struct IntermediateDecisionsWrapper { + #[serde(rename = "JSON.GET")] + json: String, +} + #[derive(Debug, Deserialize, Serialize, Clone)] #[serde(crate = "alt_serde")] pub(crate) struct IntermediateDecisions { @@ -32,7 +41,7 @@ struct IntermediateDecision { } pub fn base64_decode(input: &String) -> Vec { - let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity + let mut buf = Vec::with_capacity(392); // ToDo: calculate capacity buf.resize(392, 0); BASE64_STANDARD.decode_slice(input, &mut buf).unwrap(); // ToDo: handle error buf.iter().map(|&char| char as u8).collect() @@ -40,7 +49,7 @@ pub fn base64_decode(input: &String) -> Vec { /// Encodes a vector of bytes into a vector of characters using base64 encoding. pub fn base64_encode(input: &Vec) -> Vec { - let mut buf = Vec::with_capacity(1024); // ToDo: calculate capacity + let mut buf = Vec::with_capacity(392); // ToDo: calculate capacity buf.resize(392, 0); BASE64_STANDARD.encode_slice(input, &mut buf).unwrap(); // ToDo: handle error buf.iter().map(|&byte| byte as char).collect() @@ -56,10 +65,9 @@ pub fn share_intermediate_validation_result( validation_decision_encoded: &String, ) -> Result { let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(HTTP_TIMEOUT_MS)); - let validation_result_string = String::from(if validation_result { "true" } else { "false" }); let validation_decision_string = String::from(validation_decision_encoded); let json = serde_json::json!({ - "result": validation_result_string, + "result": validation_result, "data": validation_decision_string, }); let json_str = serde_json::to_string(&json).unwrap(); @@ -100,11 +108,16 @@ pub fn share_intermediate_validation_result( Ok(json) } -pub(crate) fn get_intermediate_decisions(data_provider_url: &String, era: &EraIndex, quorum: Vec) -> Vec { +pub(crate) fn get_intermediate_decisions(data_provider_url: &String, edge: &str, era: &EraIndex, quorum: Vec) -> Vec { let url = format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", data_provider_url, era); - let all_decisions: IntermediateDecisions = dac::http_get_json(url.as_str()).unwrap(); - let quorum_decisions = find_quorum_decisions(all_decisions, quorum); + let response: IntermediateDecisionsWrapper = dac::http_get_json(url.as_str()).unwrap(); + let mut edges_to_validators_decisions: BTreeMap> = serde_json::from_str(&response.json).unwrap(); + let decisions_for_edge = IntermediateDecisions { + validators_to_decisions: edges_to_validators_decisions.remove(edge).unwrap() + }; + + let quorum_decisions = find_quorum_decisions(decisions_for_edge, quorum); let decoded_decisions = decode_intermediate_decisions(quorum_decisions); decoded_decisions @@ -117,7 +130,11 @@ pub(crate) fn decode_intermediate_decisions(decisions: IntermediateDecisions) -> let data = base64_decode(&decision.data); let data_str = String::from_utf8_lossy(&data); - let decoded_decision: ValidationDecision = serde_json::from_str(&data_str).unwrap(); + let data_trimmed = data_str.trim_end_matches('\0'); + + info!("data_str: {:?}", data_trimmed); + + let decoded_decision: ValidationDecision = serde_json::from_str(&data_trimmed).unwrap(); decoded_decisions.push(decoded_decision); }