merge queue: embarking main (d28386c) and [#8024 + #8034] together

.github/workflows/ci-build-crates.patch.yml

@@ -33,11 +33,11 @@ jobs:
       # This step is meant to dynamically create a JSON containing the values of each crate
       # available in this repo in the root directory. We use `cargo tree` to accomplish this task.
       #
-      # The result from `cargo tree` is then transform to JSON values between double quotes, 
+      # The result from `cargo tree` is then transform to JSON values between double quotes,
       # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable.
       #
       # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output
-      # named `matrix`, which is then used as the input in following steps, 
+      # named `matrix`, which is then used as the input in following steps,
       # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}`
       - id: set-matrix
         name: Dynamically build crates JSON

.github/workflows/ci-build-crates.yml

@@ -101,7 +101,8 @@
     name: Build ${{ matrix.crate }} crate
     timeout-minutes: 90
     needs: [ matrix, check-matrix ]
-    runs-on: ubuntu-latest
+    # Some of these builds take more than 14GB disk space
+    runs-on: ubuntu-latest-m
     strategy:
       # avoid rate-limit errors by only launching a few of these jobs at a time
       max-parallel: 2
@@ -132,15 +133,18 @@
         # Some Zebra crates do not have any features, and most don't have any default features.
       - name: Build ${{ matrix.crate }} crate with no default features
         run: |
-          cargo build --package ${{ matrix.crate }} --no-default-features
+          cargo clippy --package ${{ matrix.crate }} --no-default-features --all-targets -- -D warnings
+          cargo build --package ${{ matrix.crate }} --no-default-features --all-targets
 
-      - name: Build ${{ matrix.crate }} crate normally
+      - name: Build ${{ matrix.crate }} crate with default features
         run: |
-          cargo build --package ${{ matrix.crate }}
+          cargo clippy --package ${{ matrix.crate }} --all-targets -- -D warnings
+          cargo build --package ${{ matrix.crate }} --all-targets
 
       - name: Build ${{ matrix.crate }} crate with all features
         run: |
-          cargo build --package ${{ matrix.crate }} --all-features
+          cargo clippy --package ${{ matrix.crate }} --all-features --all-targets -- -D warnings
+          cargo build --package ${{ matrix.crate }} --all-features --all-targets
 
   failure-issue:
     name: Open or update issues for building crates individually failures

.github/workflows/ci-lint.yml

@@ -98,11 +98,11 @@ jobs:
           # GitHub displays the clippy job and its results as separate entries
           name: Clippy (stable) Results
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features --all-targets -- -D warnings
+          args: --workspace --all-features --all-targets -- -D warnings
 
       - name: Run clippy manually without annotations
         if: ${{ !steps.check_permissions.outputs.has-permission }}
-        run: cargo clippy --all-features --all-targets -- -D warnings
+        run: cargo clippy --workspace --all-features --all-targets -- -D warnings
 
   fmt:
     name: Rustfmt

zebra-scan/Cargo.toml

@@ -19,26 +19,30 @@ categories = ["cryptography::cryptocurrencies"]
 # Production features that activate extra dependencies, or extra features in dependencies
 
 [dependencies]
-zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.31" }
-zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31" }
-
-zcash_primitives = "0.13.0-rc.1"
-zcash_client_backend = "0.10.0-rc.1"
 
 color-eyre = "0.6.2"
 indexmap = { version = "2.0.1", features = ["serde"] }
 serde = { version = "1.0.193", features = ["serde_derive"] }
-tokio = { version = "1.34.0", features = ["test-util"] }
+tokio = "1.34.0"
 tower = "0.4.13"
 tracing = "0.1.39"
 
+zcash_client_backend = "0.10.0-rc.1"
+zcash_primitives = "0.13.0-rc.1"
+
+zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.31" }
+zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31" }
+
 [dev-dependencies]
-zcash_note_encryption = "0.4.0"
 
-rand = "0.8.5"
 bls12_381 = "0.8.0"
-jubjub = "0.10.0"
 ff = "0.13.0"
 group = "0.13.0"
+jubjub = "0.10.0"
+rand = "0.8.5"
+tokio = { version = "1.34.0", features = ["test-util"] }
+
+zcash_note_encryption = "0.4.0"
 
+zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["proptest-impl"] }
 zebra-test = { path = "../zebra-test" }

zebra-scan/src/scan.rs

@@ -1,10 +1,11 @@
-//! The scan task.
+//! The scanner task and scanning APIs.
 
 use std::{sync::Arc, time::Duration};
 
 use color_eyre::{eyre::eyre, Report};
 use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt};
 use tracing::info;
+
 use zcash_client_backend::{
     data_api::ScannedBlock,
     proto::compact_formats::{
@@ -13,6 +14,7 @@ use zcash_client_backend::{
     scanning::{ScanError, ScanningKey},
 };
 use zcash_primitives::zip32::AccountId;
+
 use zebra_chain::{
     block::Block, parameters::Network, serialization::ZcashSerialize, transaction::Transaction,
 };
@@ -68,7 +70,7 @@ pub async fn start(mut state: State, storage: Storage) -> Result<(), Report> {
     }
 }
 
-/// Returns transactions belonging to any of the given [`ScanningKey`]s.
+/// Returns transactions belonging to the given `ScanningKey`.
 ///
 /// TODO:
 /// - Remove the `sapling_tree_size` parameter or turn it into an `Option` once we have access to
@@ -78,7 +80,7 @@ pub fn scan_block<K: ScanningKey>(
     network: Network,
     block: Arc<Block>,
     sapling_tree_size: u32,
-    scanning_keys: &[&K],
+    scanning_key: &K,
 ) -> Result<ScannedBlock<K::Nf>, ScanError> {
     // TODO: Implement a check that returns early when the block height is below the Sapling
     // activation height.
@@ -94,10 +96,8 @@ pub fn scan_block<K: ScanningKey>(
     // Use a dummy `AccountId` as we don't use accounts yet.
     let dummy_account = AccountId::from(0);
 
-    let scanning_keys: Vec<_> = scanning_keys
-        .iter()
-        .map(|key| (&dummy_account, key))
-        .collect();
+    // We only support scanning one key and one block per function call for now.
+    let scanning_keys = vec![(&dummy_account, scanning_key)];
 
     zcash_client_backend::scanning::scan_block(
         &network,

zebra-scan/src/tests.rs

@@ -5,6 +5,11 @@
 
 use std::sync::Arc;
 
+use color_eyre::Result;
+use ff::{Field, PrimeField};
+use group::GroupEncoding;
+use rand::{rngs::OsRng, RngCore};
+
 use zcash_client_backend::{
     encoding::decode_extended_full_viewing_key,
     proto::compact_formats::{
@@ -26,96 +31,20 @@ use zcash_primitives::{
     zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey},
 };
 
-use color_eyre::Result;
-
-use rand::{rngs::OsRng, RngCore};
-
-use ff::{Field, PrimeField};
-use group::GroupEncoding;
-
 use zebra_chain::{
-    block::Block, chain_tip::ChainTip, parameters::Network, serialization::ZcashDeserializeInto,
-    transaction::Hash,
+    block::Block, chain_tip::ChainTip, serialization::ZcashDeserializeInto, transaction::Hash,
 };
 
 use crate::scan::{block_to_compact, scan_block};
 
-/// Scans a continuous chain of Mainnet blocks from tip to genesis.
-///
-/// Also verifies that no relevant transaction is found in the chain when scanning for a fake
-/// account's nullifier.
-#[tokio::test]
-async fn scanning_from_populated_zebra_state() -> Result<()> {
-    let network = Network::default();
-
-    // Create a continuous chain of mainnet blocks from genesis
-    let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
-        .iter()
-        .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap())
-        .collect();
-
-    // Create a populated state service.
-    let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) =
-        zebra_state::populated_state(blocks.clone(), network).await;
-
-    let db = read_only_state_service.db();
-
-    // use the tip as starting height
-    let mut height = latest_chain_tip.best_tip_height().unwrap();
-
-    let mut transactions_found = 0;
-    let mut transactions_scanned = 0;
-    let mut blocks_scanned = 0;
-    // TODO: Accessing the state database directly is ok in the tests, but not in production code.
-    // Use `Request::Block` if the code is copied to production.
-    while let Some(block) = db.block(height.into()) {
-        // We use a dummy size of the Sapling note commitment tree. We can't set the size to zero
-        // because the underlying scanning function would return
-        // `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`.
-        let sapling_commitment_tree_size = 1;
-
-        let orchard_commitment_tree_size = 0;
-
-        let chain_metadata = ChainMetadata {
-            sapling_commitment_tree_size,
-            orchard_commitment_tree_size,
-        };
-
-        let compact_block = block_to_compact(block.clone(), chain_metadata);
-
-        let res =
-            scan_block::<SaplingIvk>(network, block, sapling_commitment_tree_size, &[]).unwrap();
-
-        transactions_found += res.transactions().len();
-        transactions_scanned += compact_block.vtx.len();
-        blocks_scanned += 1;
-
-        if height.is_min() {
-            break;
-        }
-
-        // scan backwards
-        height = height.previous()?;
-    }
-
-    // make sure all blocks and transactions were scanned
-    assert_eq!(blocks_scanned, 11);
-    assert_eq!(transactions_scanned, 11);
-
-    // no relevant transactions should be found
-    assert_eq!(transactions_found, 0);
-
-    Ok(())
-}
-
 /// Prove that we can create fake blocks with fake notes and scan them using the
 /// `zcash_client_backend::scanning::scan_block` function:
 /// - Function `fake_compact_block` will generate 1 block with one pre created fake nullifier in
 /// the transaction and one additional random transaction without it.
 /// - Verify one relevant transaction is found in the chain when scanning for the pre created fake
 /// account's nullifier.
-#[tokio::test]
-async fn scanning_from_fake_generated_blocks() -> Result<()> {
+#[test]
+fn scanning_from_fake_generated_blocks() -> Result<()> {
     let account = AccountId::from(12);
     let extsk = ExtendedSpendingKey::master(&[]);
     let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
@@ -209,7 +138,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
 
         let compact_block = block_to_compact(block.clone(), chain_metadata);
 
-        let res = scan_block(network, block, sapling_commitment_tree_size, &[&ivk])
+        let res = scan_block(network, block, sapling_commitment_tree_size, &ivk)
             .expect("scanning block for the ZECpages viewing key should work");
 
         transactions_found += res.transactions().len();
@@ -235,9 +164,9 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
 
 /// In this test we generate a viewing key and manually add it to the database. Also we send results to the Storage database.
 /// The purpose of this test is to check if our database and our scanning code are compatible.
-#[tokio::test]
+#[test]
 #[allow(deprecated)]
-async fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
+fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
     // Generate a key
     let account = AccountId::from(12);
     let extsk = ExtendedSpendingKey::master(&[]);

zebrad/tests/common/config.rs

@@ -15,7 +15,7 @@ use std::{
 use color_eyre::eyre::Result;
 use tempfile::TempDir;
 
-use zebra_chain::parameters::Network::{self, *};
+use zebra_chain::parameters::Network;
 use zebra_test::net::random_known_port;
 use zebrad::{
     components::{mempool, sync, tracing},
@@ -70,10 +70,11 @@ pub fn default_test_config(net: Network) -> Result<ZebradConfig> {
 
     #[cfg(feature = "getblocktemplate-rpcs")]
     {
-        let miner_address = if network.network == Mainnet {
-            "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1"
-        } else {
+        let miner_address = if network.network.is_a_test_network() {
+            // Assume test networks all use the same address prefix and format
             "t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v"
+        } else {
+            "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1"
         };
 
         mining.miner_address = Some(miner_address.parse().expect("hard-coded address is valid"));