Merge branch 'master' into phry/legacy-backend-testing

Cargo.toml

@@ -26,7 +26,7 @@ exclude = [
     "testing/wasm-lightclient-tests",
     "signer/wasm-tests",
     "examples/wasm-example",
-    "examples/parachain-example"
+    "examples/parachain-example",
 ]
 resolver = "2"
 
@@ -104,7 +104,7 @@ serde = { version = "1.0.209", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0.127", default-features = false }
 syn = { version = "2.0.15", features = ["full", "extra-traits"] }
 thiserror = "1.0.63"
-tokio = { version = "1.39", default-features = false }
+tokio = { version = "1.40", default-features = false }
 tracing = { version = "0.1.40", default-features = false }
 tracing-wasm = "0.2.1"
 tracing-subscriber = "0.3.18"
@@ -116,6 +116,8 @@ which = "5.0.0"
 strip-ansi-escapes = "0.2.0"
 proptest = "1.5.0"
 hex-literal = "0.4.1"
+sc-executor = "0.40.0"
+sc-executor-common = "0.35.0"
 
 # Light client support:
 smoldot = { version = "0.16.0", default-features = false }
@@ -140,6 +142,9 @@ sp-core = { version = "31.0.0", default-features = false }
 sp-crypto-hashing = { version = "0.1.0", default-features = false }
 sp-runtime = "34.0.0"
 sp-keyring = "34.0.0"
+sp-maybe-compressed-blob = "11.0.0"
+sp-state-machine = "0.43.0"
+sp-io = "38.0.0"
 
 # Subxt workspace crates:
 subxt = { version = "0.37.0", path = "subxt", default-features = false }
@@ -167,6 +172,7 @@ base64 = { version = "0.22.1", default-features = false }
 scrypt = { version = "0.11.0", default-features = false }
 crypto_secretbox = { version = "0.1.1", default-features = false }
 
+
 [profile.dev.package.smoldot-light]
 opt-level = 2
 [profile.test.package.smoldot-light]

codegen/src/error.rs

@@ -57,6 +57,9 @@ pub enum CodegenError {
     /// Cannot generate types.
     #[error("Type Generation failed: {0}")]
     TypeGeneration(#[from] TypegenError),
+    /// Error when generating metadata from Wasm-runtime
+    #[error("Failed to generate metadata from wasm file. reason: {0}")]
+    Wasm(String),
 }
 
 impl CodegenError {

macro/Cargo.toml

@@ -15,6 +15,7 @@ description = "Generate types and helpers for interacting with Substrate runtime
 
 [features]
 web = ["subxt-codegen/web"]
+runtime-path = ["sp-io", "sc-executor-common", "sp-state-machine", "sp-maybe-compressed-blob", "sc-executor"]
 
 [lib]
 proc-macro = true
@@ -27,6 +28,11 @@ syn = { workspace = true }
 quote = { workspace = true }
 subxt-codegen = { workspace = true, features = ["fetch-metadata"] }
 scale-typegen = { workspace = true }
+sc-executor = { workspace = true, optional = true }
+sp-maybe-compressed-blob = { workspace = true, optional = true }
+sp-state-machine = { workspace = true, optional = true }
+sp-io = { workspace = true, optional = true }
+sc-executor-common = { workspace = true, optional = true }
 
 [lints]
 workspace = true

macro/src/lib.rs

@@ -23,6 +23,9 @@ use subxt_codegen::{
 };
 use syn::{parse_macro_input, punctuated::Punctuated};
 
+#[cfg(feature = "runtime-path")]
+mod wasm_loader;
+
 #[derive(Clone, Debug)]
 struct OuterAttribute(syn::Attribute);
 
@@ -60,6 +63,9 @@ struct RuntimeMetadataArgs {
     no_default_substitutions: bool,
     #[darling(default)]
     unstable_metadata: darling::util::Flag,
+    #[cfg(feature = "runtime-path")]
+    #[darling(default)]
+    runtime_path: Option<String>,
 }
 
 #[derive(Debug, FromMeta)]
@@ -206,6 +212,22 @@ fn validate_type_path(path: &syn::Path, metadata: &Metadata) {
 fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata, TokenStream> {
     // Do we want to fetch unstable metadata? This only works if fetching from a URL.
     let unstable_metadata = args.unstable_metadata.is_present();
+
+    #[cfg(feature = "runtime-path")]
+    if let Some(path) = &args.runtime_path {
+        if args.runtime_metadata_insecure_url.is_some() || args.runtime_metadata_path.is_some() {
+            abort_call_site!(
+                "Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' or `runtime_path` must be provided"
+            );
+        };
+        let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
+        let root_path = std::path::Path::new(&root);
+        let path = root_path.join(path);
+
+        let metadata = wasm_loader::from_wasm_file(&path).map_err(|e| e.into_compile_error())?;
+        return Ok(metadata);
+    };
+
     let metadata = match (
         &args.runtime_metadata_path,
         &args.runtime_metadata_insecure_url,
@@ -239,12 +261,26 @@ fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata,
                 .and_then(|b| subxt_codegen::Metadata::decode(&mut &*b).map_err(Into::into))
                 .map_err(|e| e.into_compile_error())?
         }
+        #[cfg(feature = "runtime-path")]
         (None, None) => {
             abort_call_site!(
-                "One of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' must be provided"
+                    "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
+                )
+        }
+        #[cfg(not(feature = "runtime-path"))]
+        (None, None) => {
+            abort_call_site!(
+                "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' can be provided"
             )
         }
-        (Some(_), Some(_)) => {
+        #[cfg(feature = "runtime-path")]
+        _ => {
+            abort_call_site!(
+                    "Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
+                )
+        }
+        #[cfg(not(feature = "runtime-path"))]
+        _ => {
             abort_call_site!(
                 "Only one of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' can be provided"
             )

macro/src/wasm_loader.rs

@@ -0,0 +1,55 @@
+// Copyright 2024 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use std::{borrow::Cow, path::Path};
+
+use codec::Decode;
+use sc_executor::{WasmExecutionMethod, WasmExecutor};
+use sc_executor_common::runtime_blob::RuntimeBlob;
+use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
+use subxt_codegen::{fetch_metadata::fetch_metadata_from_file_blocking, CodegenError, Metadata};
+
+/// Result type shorthand
+pub type WasmMetadataResult<A> = Result<A, CodegenError>;
+
+/// Uses wasm artifact produced by compiling the runtime to generate metadata
+pub fn from_wasm_file(wasm_file_path: &Path) -> WasmMetadataResult<Metadata> {
+    let wasm_file = fetch_metadata_from_file_blocking(wasm_file_path)
+        .map_err(Into::<CodegenError>::into)
+        .and_then(maybe_decompress)?;
+    call_and_decode(wasm_file)
+}
+
+fn call_and_decode(wasm_file: Vec<u8>) -> WasmMetadataResult<Metadata> {
+    let mut ext: sp_state_machine::BasicExternalities = Default::default();
+
+    let executor: WasmExecutor<sp_io::SubstrateHostFunctions> = WasmExecutor::builder()
+        .with_execution_method(WasmExecutionMethod::default())
+        .with_offchain_heap_alloc_strategy(sc_executor::HeapAllocStrategy::Dynamic {
+            maximum_pages: Some(64),
+        })
+        .with_max_runtime_instances(1)
+        .with_runtime_cache_size(1)
+        .build();
+
+    let runtime_blob =
+        RuntimeBlob::new(&wasm_file).map_err(|e| CodegenError::Wasm(e.to_string()))?;
+    let metadata_encoded = executor
+        .uncached_call(runtime_blob, &mut ext, true, "Metadata_metadata", &[])
+        .map_err(|_| CodegenError::Wasm("method \"Metadata_metadata\" doesnt exist".to_owned()))?;
+
+    let metadata = <Vec<u8>>::decode(&mut &metadata_encoded[..]).map_err(CodegenError::Decode)?;
+
+    decode(metadata)
+}
+
+fn decode(encoded_metadata: Vec<u8>) -> WasmMetadataResult<Metadata> {
+    Metadata::decode(&mut encoded_metadata.as_ref()).map_err(Into::into)
+}
+
+fn maybe_decompress(file_contents: Vec<u8>) -> WasmMetadataResult<Vec<u8>> {
+    sp_maybe_compressed_blob::decompress(file_contents.as_ref(), CODE_BLOB_BOMB_LIMIT)
+        .map_err(|e| CodegenError::Wasm(e.to_string()))
+        .map(Cow::into_owned)
+}

subxt/Cargo.toml

@@ -70,6 +70,9 @@ unstable-metadata = []
 # Note that this feature is experimental and things may break or not work as expected.
 unstable-light-client = ["subxt-lightclient"]
 
+# Activate this to expose the ability to generate metadata from Wasm runtime files.
+runtime-path = ["subxt-macro/runtime-path"]
+
 [dependencies]
 async-trait = { workspace = true }
 codec = { package = "parity-scale-codec", workspace = true, features = ["derive"] }

subxt/examples/setup_reconnecting_rpc_client.rs

@@ -2,7 +2,7 @@
 //! which hidden behind behind `--feature unstable-reconnecting-rpc-client`
 //!
 //! To utilize full logs from the RPC client use:
-//! `RUST_LOG="jsonrpsee=trace,reconnecting_jsonrpsee_ws_client=trace"`
+//! `RUST_LOG="jsonrpsee=trace,subxt-reconnecting-rpc-client=trace"`
 
 #![allow(missing_docs)]
 

subxt/src/backend/rpc/reconnecting_rpc_client/tests.rs

@@ -13,15 +13,13 @@ use jsonrpsee::server::{
 
 #[tokio::test]
 async fn call_works() {
-    tracing_subscriber::fmt::init();
     let (_handle, addr) = run_server().await.unwrap();
     let client = RpcClient::builder().build(addr).await.unwrap();
     assert!(client.request("say_hello".to_string(), None).await.is_ok(),)
 }
 
 #[tokio::test]
 async fn sub_works() {
-    tracing_subscriber::fmt::init();
     let (_handle, addr) = run_server().await.unwrap();
 
     let client = RpcClient::builder()
@@ -44,7 +42,6 @@ async fn sub_works() {
 
 #[tokio::test]
 async fn sub_with_reconnect() {
-    tracing_subscriber::fmt::init();
     let (handle, addr) = run_server().await.unwrap();
     let client = RpcClient::builder().build(addr.clone()).await.unwrap();
 
@@ -89,7 +86,6 @@ async fn sub_with_reconnect() {
 
 #[tokio::test]
 async fn call_with_reconnect() {
-    tracing_subscriber::fmt::init();
     let (handle, addr) = run_server_with_settings(None, true).await.unwrap();
 
     let client = Arc::new(RpcClient::builder().build(addr.clone()).await.unwrap());

testing/integration-tests/Cargo.toml

@@ -36,7 +36,7 @@ serde = { workspace = true }
 scale-info = { workspace = true, features = ["bit-vec"] }
 sp-core = { workspace = true }
 syn = { workspace = true }
-subxt = { workspace = true, features = ["unstable-metadata", "native", "jsonrpsee", "substrate-compat"] }
+subxt = { workspace = true, features = ["unstable-metadata", "native", "jsonrpsee", "substrate-compat", "unstable-reconnecting-rpc-client"] }
 subxt-signer = { workspace = true, features = ["default"] }
 subxt-codegen = { workspace = true }
 subxt-metadata = { workspace = true }

testing/integration-tests/src/full_client/client/mod.rs

@@ -2,8 +2,10 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+use std::collections::HashSet;
+
 use crate::{
-    subxt_test, test_context,
+    subxt_test, test_context, test_context_reconnecting_rpc_client,
     utils::{node_runtime, wait_for_blocks},
 };
 use codec::{Decode, Encode};
@@ -409,3 +411,50 @@ async fn partial_fee_estimate_correct() {
     // Both methods should yield the same fee
     assert_eq!(partial_fee_1, partial_fee_2);
 }
+
+#[subxt_test]
+async fn legacy_and_unstable_block_subscription_reconnect() {
+    let ctx = test_context_reconnecting_rpc_client().await;
+    let api = ctx.unstable_client().await;
+
+    let unstable_client_blocks = move |num: usize| {
+        let api = api.clone();
+        async move {
+            api.blocks()
+                .subscribe_finalized()
+                .await
+                .unwrap()
+                // Ignore `disconnected events`.
+                // This will be emitted by the legacy backend for every reconnection.
+                .filter(|item| {
+                    let disconnected = match item {
+                        Ok(_) => false,
+                        Err(e) => e.is_disconnected_will_reconnect(),
+                    };
+
+                    futures::future::ready(!disconnected)
+                })
+                .take(num)
+                .map(|x| x.unwrap().hash().to_string())
+                .collect::<Vec<String>>()
+                .await
+        }
+    };
+
+    let blocks = unstable_client_blocks(3).await;
+    let blocks: HashSet<String> = HashSet::from_iter(blocks.into_iter());
+
+    assert!(blocks.len() == 3);
+
+    let ctx = ctx.restart().await;
+
+    // Make  client aware that connection was dropped and force them to reconnect
+    let _ = ctx.unstable_client().await.backend().genesis_hash().await;
+
+    let unstable_blocks = unstable_client_blocks(6).await;
+
+    let unstable_blocks: HashSet<String> = HashSet::from_iter(unstable_blocks.into_iter());
+    let intersection = unstable_blocks.intersection(&blocks).count();
+
+    assert!(intersection == 3);
+}