From 323e434a17cf4d337f9971282057ac815c482a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Tue, 13 Feb 2024 12:09:38 +0100 Subject: [PATCH 01/20] feat: Implement `BlockStore` for derefs, too --- wnfs-common/src/blockstore.rs | 28 ++++++++++++++++++++++++++-- wnfs/src/public/link.rs | 14 ++++---------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 8f139d28..1a2a7424 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -14,7 +14,7 @@ use libipld::{ }; use parking_lot::Mutex; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use std::collections::HashMap; +use std::{collections::HashMap, ops::Deref}; //-------------------------------------------------------------------------------------------------- // Constants @@ -51,7 +51,7 @@ pub const CODEC_RAW: u64 = 0x55; /// For types that implement block store operations like adding, getting content from the store. #[cfg_attr(not(target_arch = "wasm32"), async_trait)] #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait BlockStore: Sized + CondSync { +pub trait BlockStore: CondSync { async fn get_block(&self, cid: &Cid) -> Result; async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result; @@ -89,6 +89,30 @@ pub trait BlockStore: Sized + CondSync { } } +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +impl + CondSync> BlockStore for T { + async fn get_block(&self, cid: &Cid) -> Result { + self.deref().get_block(cid).await + } + + async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + self.deref().put_block(bytes, codec).await + } + + async fn get_deserializable(&self, cid: &Cid) -> Result { + self.deref().get_deserializable(cid).await + } + + async fn put_serializable(&self, value: &V) -> Result { + self.deref().put_serializable(value).await + } + + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + self.deref().create_cid(bytes, codec) + } +} + //-------------------------------------------------------------------------------------------------- // Implementations //-------------------------------------------------------------------------------------------------- diff --git a/wnfs/src/public/link.rs b/wnfs/src/public/link.rs index 34048e65..234f094e 100644 --- a/wnfs/src/public/link.rs +++ b/wnfs/src/public/link.rs @@ -52,31 +52,25 @@ impl PublicLink { /// Gets the Cid stored in type. It attempts to get it from the store if it is not present in type. #[inline] - pub async fn resolve_cid(&self, store: &(impl BlockStore + ?Sized)) -> Result { + pub async fn resolve_cid(&self, store: &impl BlockStore) -> Result { self.0.resolve_cid(store).await } /// Gets the value stored in link. It attempts to get it from the store if it is not present in link. #[inline] - pub async fn resolve_value(&self, store: &(impl BlockStore + ?Sized)) -> Result<&PublicNode> { + pub async fn resolve_value(&self, store: &impl BlockStore) -> Result<&PublicNode> { self.0.resolve_value(store).await } /// Gets mut value stored in link. It attempts to get it from the store if it is not present in link. #[inline] - pub async fn resolve_value_mut( - &mut self, - store: &(impl BlockStore + ?Sized), - ) -> Result<&mut PublicNode> { + pub async fn resolve_value_mut(&mut self, store: &impl BlockStore) -> Result<&mut PublicNode> { self.0.resolve_value_mut(store).await } /// Gets an owned value from type. It attempts to it get from the store if it is not present in type. #[inline] - pub async fn resolve_owned_value( - self, - store: &(impl BlockStore + ?Sized), - ) -> Result { + pub async fn resolve_owned_value(self, store: &impl BlockStore) -> Result { self.0.resolve_owned_value(store).await } From e8184827a9e7d59d85eea6730e5015c689c305c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Tue, 13 Feb 2024 12:23:39 +0100 Subject: [PATCH 02/20] refactor: Use new RPITIT feature for `trait BlockStore` --- wnfs-common/Cargo.toml | 1 + wnfs-common/src/blockstore.rs | 49 +++++++++++++++-------------- wnfs-common/src/utils/test.rs | 3 -- wnfs-hamt/Cargo.toml | 1 + wnfs-unixfs-file/Cargo.toml | 1 + wnfs-wasm/Cargo.toml | 1 + wnfs-wasm/src/fs/blockstore.rs | 1 - wnfs-wasm/yarn.lock | 5 --- wnfs/Cargo.toml | 1 + wnfs/examples/tiered_blockstores.rs | 3 -- 10 files changed, 31 insertions(+), 35 deletions(-) diff --git a/wnfs-common/Cargo.toml b/wnfs-common/Cargo.toml index be7cea25..82c1f10b 100644 --- a/wnfs-common/Cargo.toml +++ b/wnfs-common/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-common" homepage = "https://fission.codes" authors = ["The Fission Authors"] diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 1a2a7424..eeca51f7 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -4,8 +4,8 @@ use crate::{ BlockStoreError, MAX_BLOCK_SIZE, }; use anyhow::{bail, Result}; -use async_trait::async_trait; use bytes::Bytes; +use futures::Future; use libipld::{ cbor::DagCborCodec, cid::Version, @@ -49,27 +49,34 @@ pub const CODEC_RAW: u64 = 0x55; //-------------------------------------------------------------------------------------------------- /// For types that implement block store operations like adding, getting content from the store. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait BlockStore: CondSync { - async fn get_block(&self, cid: &Cid) -> Result; - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result; - - async fn get_deserializable(&self, cid: &Cid) -> Result - where - V: DeserializeOwned, - { - let bytes = self.get_block(cid).await?; - let ipld = decode(bytes.as_ref(), DagCborCodec)?; - Ok(ipld_serde::from_ipld::(ipld)?) + fn get_block(&self, cid: &Cid) -> impl Future> + CondSend; + + fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> impl Future> + CondSend; + + fn get_deserializable( + &self, + cid: &Cid, + ) -> impl Future> + CondSend { + async { + let bytes = self.get_block(cid).await?; + let ipld = decode(bytes.as_ref(), DagCborCodec)?; + Ok(ipld_serde::from_ipld::(ipld)?) + } } - async fn put_serializable(&self, value: &V) -> Result - where - V: Serialize + CondSync, - { - let bytes = encode(&ipld_serde::to_ipld(value)?, DagCborCodec)?; - self.put_block(bytes, CODEC_DAG_CBOR).await + fn put_serializable( + &self, + value: &V, + ) -> impl Future> + CondSend { + async move { + let bytes = encode(&ipld_serde::to_ipld(value)?, DagCborCodec)?; + self.put_block(bytes, CODEC_DAG_CBOR).await + } } // This should be the same in all implementations of BlockStore @@ -89,8 +96,6 @@ pub trait BlockStore: CondSync { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl + CondSync> BlockStore for T { async fn get_block(&self, cid: &Cid) -> Result { self.deref().get_block(cid).await @@ -135,8 +140,6 @@ impl MemoryBlockStore { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for MemoryBlockStore { /// Retrieves an array of bytes from the block store with given CID. async fn get_block(&self, cid: &Cid) -> Result { diff --git a/wnfs-common/src/utils/test.rs b/wnfs-common/src/utils/test.rs index f0f429f8..44d8a3b5 100644 --- a/wnfs-common/src/utils/test.rs +++ b/wnfs-common/src/utils/test.rs @@ -1,7 +1,6 @@ use super::{Arc, CondSend, CondSync}; use crate::{BlockStore, MemoryBlockStore, CODEC_DAG_CBOR, CODEC_RAW}; use anyhow::Result; -use async_trait::async_trait; use base64_serde::base64_serde_type; use bytes::Bytes; use libipld::{ @@ -112,8 +111,6 @@ impl SnapshotBlockStore { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for SnapshotBlockStore { #[inline] async fn get_block(&self, cid: &Cid) -> Result { diff --git a/wnfs-hamt/Cargo.toml b/wnfs-hamt/Cargo.toml index b61c35fa..28fff289 100644 --- a/wnfs-hamt/Cargo.toml +++ b/wnfs-hamt/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-hamt" homepage = "https://fission.codes" authors = ["The Fission Authors"] diff --git a/wnfs-unixfs-file/Cargo.toml b/wnfs-unixfs-file/Cargo.toml index 83c521a3..a0cee00a 100644 --- a/wnfs-unixfs-file/Cargo.toml +++ b/wnfs-unixfs-file/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file" homepage = "https://fission.codes" authors = ["The Fission Authors"] diff --git a/wnfs-wasm/Cargo.toml b/wnfs-wasm/Cargo.toml index 24014f78..28582337 100644 --- a/wnfs-wasm/Cargo.toml +++ b/wnfs-wasm/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm" homepage = "https://fission.codes" authors = ["The Fission Authors"] diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index b01d9f84..0e59c6ba 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -38,7 +38,6 @@ pub struct ForeignBlockStore(pub(crate) BlockStore); // Implementations //-------------------------------------------------------------------------------------------------- -#[async_trait(?Send)] impl WnfsBlockStore for ForeignBlockStore { /// Stores an array of bytes in the block store. async fn put_block(&self, bytes: impl Into, codec: u64) -> Result { diff --git a/wnfs-wasm/yarn.lock b/wnfs-wasm/yarn.lock index 73cbeda5..3ee07444 100644 --- a/wnfs-wasm/yarn.lock +++ b/wnfs-wasm/yarn.lock @@ -2009,11 +2009,6 @@ multicast-dns@^7.2.5: dns-packet "^5.2.2" thunky "^1.0.2" -multiformats@^12.0.1: - version "12.1.3" - resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-12.1.3.tgz#cbf7a9861e11e74f8228b21376088cb43ba8754e" - integrity sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw== - multiformats@^13.0.0: version "13.0.0" resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-13.0.0.tgz#97f3341b16c34716a14518d178ea0c190e987c32" diff --git a/wnfs/Cargo.toml b/wnfs/Cargo.toml index c8386907..d532b41b 100644 --- a/wnfs/Cargo.toml +++ b/wnfs/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs" homepage = "https://fission.codes" authors = ["The Fission Authors"] diff --git a/wnfs/examples/tiered_blockstores.rs b/wnfs/examples/tiered_blockstores.rs index 3e36bb0a..a46976bf 100644 --- a/wnfs/examples/tiered_blockstores.rs +++ b/wnfs/examples/tiered_blockstores.rs @@ -4,7 +4,6 @@ //! work with high latency. use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; use libipld_core::cid::Cid; @@ -96,8 +95,6 @@ struct TieredBlockStore { cold: C, } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for TieredBlockStore { async fn get_block(&self, cid: &Cid) -> Result { match self.hot.get_block(cid).await { From 5e44fdc22e61ed271ec3c410fb686f9ca0ae7c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Tue, 13 Feb 2024 15:24:33 +0100 Subject: [PATCH 03/20] refactor: Move `trait PrivateForest` to RPITIT --- wnfs/src/private/forest/hamt.rs | 4 ---- wnfs/src/private/forest/proofs.rs | 3 --- wnfs/src/private/forest/traits.rs | 32 ++++++++++++++++++------------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/wnfs/src/private/forest/hamt.rs b/wnfs/src/private/forest/hamt.rs index a93796fb..4f34c0fd 100644 --- a/wnfs/src/private/forest/hamt.rs +++ b/wnfs/src/private/forest/hamt.rs @@ -231,8 +231,6 @@ impl HamtForest { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for HamtForest { fn empty_name(&self) -> Name { Name::empty(&self.accumulator) @@ -336,8 +334,6 @@ impl PrivateForest for HamtForest { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for Arc { fn empty_name(&self) -> Name { (**self).empty_name() diff --git a/wnfs/src/private/forest/proofs.rs b/wnfs/src/private/forest/proofs.rs index 475a4b15..3a3d7990 100644 --- a/wnfs/src/private/forest/proofs.rs +++ b/wnfs/src/private/forest/proofs.rs @@ -1,7 +1,6 @@ use super::{hamt::HamtForest, traits::PrivateForest}; use crate::error::{FsError, VerificationError}; use anyhow::{bail, Result}; -use async_trait::async_trait; use libipld_core::cid::Cid; use std::collections::{BTreeSet, HashMap}; use wnfs_common::{ @@ -150,8 +149,6 @@ impl Default for ForestProofs { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for ProvingHamtForest { fn empty_name(&self) -> Name { self.forest.empty_name() diff --git a/wnfs/src/private/forest/traits.rs b/wnfs/src/private/forest/traits.rs index 85c243cf..cc9833eb 100644 --- a/wnfs/src/private/forest/traits.rs +++ b/wnfs/src/private/forest/traits.rs @@ -4,7 +4,7 @@ use crate::{ }; use anyhow::Result; use async_stream::stream; -use async_trait::async_trait; +use futures::Future; use libipld_core::cid::Cid; use std::collections::BTreeSet; use wnfs_common::{ @@ -20,8 +20,6 @@ use wnfs_nameaccumulator::{AccumulatorSetup, ElementsProof, Name, NameAccumulato /// It also stores the accumulator setup information for running /// name accumulator operations. Upon put or remove, it'll run /// these operations for the caller. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait PrivateForest: CondSync { /// Construct what represents the empty name in this forest. /// @@ -86,42 +84,50 @@ pub trait PrivateForest: CondSync { /// assert!(forest.has_by_hash(access_key.get_label(), store).await.unwrap()); /// } /// ``` - async fn has_by_hash(&self, name_hash: &HashOutput, store: &impl BlockStore) -> Result; + fn has_by_hash( + &self, + name_hash: &HashOutput, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Check whether a certain name has any values. - async fn has(&self, name: &Name, store: &impl BlockStore) -> Result; + fn has( + &self, + name: &Name, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Adds new encrypted values at the given key. - async fn put_encrypted( + fn put_encrypted( &mut self, name: &Name, values: I, store: &impl BlockStore, - ) -> Result + ) -> impl Future> + CondSend where I: IntoIterator + CondSend, I::IntoIter: CondSend; /// Gets the CIDs to blocks of ciphertext by hash of name. - async fn get_encrypted_by_hash<'b>( + fn get_encrypted_by_hash<'b>( &'b self, name_hash: &HashOutput, store: &impl BlockStore, - ) -> Result>>; + ) -> impl Future>>> + CondSend; /// Gets the CIDs to blocks of ciphertext by name. - async fn get_encrypted( + fn get_encrypted( &self, name: &Name, store: &impl BlockStore, - ) -> Result>>; + ) -> impl Future>>> + CondSend; /// Removes the CIDs to blocks of ciphertext by name. - async fn remove_encrypted( + fn remove_encrypted( &mut self, name: &Name, store: &impl BlockStore, - ) -> Result>>>; + ) -> impl Future>>>> + CondSend; /// Returns a stream of all private nodes that could be decrypted at given revision. /// From 29917b8fd94ca5a76ca57c5fcafb396a6711dcee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Tue, 13 Feb 2024 15:28:02 +0100 Subject: [PATCH 04/20] refactor: Use RPITIT in `trait PrivateKey` & `trait ExchangeKey` --- wnfs-wasm/src/fs/blockstore.rs | 1 - wnfs-wasm/src/fs/private/exchange_key.rs | 3 --- wnfs-wasm/src/fs/public/file.rs | 2 +- wnfs/examples/mnemonic_based.rs | 5 ----- wnfs/src/private/keys/exchange.rs | 17 +++++------------ 5 files changed, 6 insertions(+), 22 deletions(-) diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index 0e59c6ba..a29dec81 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -2,7 +2,6 @@ use super::utils::anyhow_error; use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use js_sys::{Promise, Uint8Array}; use libipld_core::cid::Cid; diff --git a/wnfs-wasm/src/fs/private/exchange_key.rs b/wnfs-wasm/src/fs/private/exchange_key.rs index 03b92ed4..71a088ca 100644 --- a/wnfs-wasm/src/fs/private/exchange_key.rs +++ b/wnfs-wasm/src/fs/private/exchange_key.rs @@ -1,6 +1,5 @@ use crate::fs::utils::anyhow_error; use anyhow::Result; -use async_trait::async_trait; use js_sys::{Promise, Uint8Array}; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen_futures::JsFuture; @@ -45,7 +44,6 @@ pub struct ForeignPrivateKey(pub(crate) PrivateKey); // Implementations //-------------------------------------------------------------------------------------------------- -#[async_trait(?Send)] impl WnfsExchangeKey for ForeignExchangeKey { async fn from_modulus(modulus: &[u8]) -> Result where @@ -67,7 +65,6 @@ impl WnfsExchangeKey for ForeignExchangeKey { } } -#[async_trait(?Send)] impl WnfsPrivateKey for ForeignPrivateKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let v = JsFuture::from(self.0.decrypt(ciphertext)) diff --git a/wnfs-wasm/src/fs/public/file.rs b/wnfs-wasm/src/fs/public/file.rs index 0312cc78..e675d9fd 100644 --- a/wnfs-wasm/src/fs/public/file.rs +++ b/wnfs-wasm/src/fs/public/file.rs @@ -103,7 +103,7 @@ impl PublicFile { /// Gets the content cid of the file. #[wasm_bindgen(js_name = "getRawContentCid")] pub fn get_raw_content_cid(&self, store: BlockStore) -> JsResult { - let mut file = Rc::clone(&self.0); + let file = Rc::clone(&self.0); let store = ForeignBlockStore(store); Ok(future_to_promise(async move { diff --git a/wnfs/examples/mnemonic_based.rs b/wnfs/examples/mnemonic_based.rs index 00ab711a..cd2088ba 100644 --- a/wnfs/examples/mnemonic_based.rs +++ b/wnfs/examples/mnemonic_based.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use async_trait::async_trait; use bip39::{Language, Mnemonic, MnemonicType, Seed}; use chrono::Utc; use rand_chacha::ChaCha12Rng; @@ -187,8 +186,6 @@ impl SeededExchangeKey { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateKey for SeededExchangeKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let padding = Oaep::new::(); @@ -196,8 +193,6 @@ impl PrivateKey for SeededExchangeKey { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl ExchangeKey for PublicExchangeKey { async fn encrypt(&self, data: &[u8]) -> Result> { let padding = Oaep::new::(); diff --git a/wnfs/src/private/keys/exchange.rs b/wnfs/src/private/keys/exchange.rs index 4c589707..f8390bf0 100644 --- a/wnfs/src/private/keys/exchange.rs +++ b/wnfs/src/private/keys/exchange.rs @@ -3,11 +3,12 @@ use crate::error::RsaError; #[cfg(test)] use anyhow::anyhow; use anyhow::Result; -use async_trait::async_trait; +use futures::Future; #[cfg(test)] use rsa::{traits::PublicKeyParts, BigUint, Oaep}; #[cfg(test)] use sha2::Sha256; +use wnfs_common::utils::CondSend; //-------------------------------------------------------------------------------------------------- // Constants @@ -28,26 +29,22 @@ pub const PUBLIC_KEY_EXPONENT: u64 = 65537; /// More on exchange keys [here][key]. /// /// [key]: https://github.com/wnfs-wg/spec/blob/main/spec/shared-private-data.md#2-exchange-keys-partition -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait ExchangeKey { /// Creates an RSA public key from the public key modulus. /// /// The exponent is expected to be of the value [`PUBLIC_KEY_EXPONENT`](constant.PUBLIC_KEY_EXPONENT.html) constant. - async fn from_modulus(modulus: &[u8]) -> Result + fn from_modulus(modulus: &[u8]) -> impl Future> + CondSend where Self: Sized; /// Encrypts data with the public key. - async fn encrypt(&self, data: &[u8]) -> Result>; + fn encrypt(&self, data: &[u8]) -> impl Future>> + CondSend; } /// The `PrivateKey` trait represents a RSA private key type that can be used to decrypt data encrypted with corresponding public key. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait PrivateKey { /// Decrypts ciphertext with the private key. - async fn decrypt(&self, ciphertext: &[u8]) -> Result>; + fn decrypt(&self, ciphertext: &[u8]) -> impl Future>> + CondSend; } pub type PublicKeyModulus = Vec; @@ -89,8 +86,6 @@ impl RsaPrivateKey { } #[cfg(test)] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl ExchangeKey for RsaPublicKey { async fn encrypt(&self, data: &[u8]) -> Result> { let padding = Oaep::new::(); @@ -110,8 +105,6 @@ impl ExchangeKey for RsaPublicKey { } #[cfg(test)] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateKey for RsaPrivateKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let padding = Oaep::new::(); From c0cbc5d8fdd74a96cec2453937571a83db6e89b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 17:18:34 +0100 Subject: [PATCH 05/20] refactor: Completely remove `async_trait` --- wnfs-common/Cargo.toml | 1 - wnfs-common/src/link.rs | 5 ---- wnfs-common/src/storable.rs | 46 ++++++++++++++++++-------------- wnfs-common/src/traits.rs | 12 +++++---- wnfs-hamt/Cargo.toml | 1 - wnfs-hamt/src/hamt.rs | 3 --- wnfs-hamt/src/node.rs | 7 ++--- wnfs-hamt/src/pointer.rs | 5 ---- wnfs-nameaccumulator/Cargo.toml | 1 - wnfs-nameaccumulator/src/name.rs | 2 -- wnfs-unixfs-file/Cargo.toml | 1 - wnfs-unixfs-file/src/unixfs.rs | 3 --- wnfs-wasm/Cargo.toml | 1 - wnfs/Cargo.toml | 1 - wnfs/src/private/forest/hamt.rs | 3 --- wnfs/src/public/directory.rs | 7 ++--- wnfs/src/public/file.rs | 3 --- wnfs/src/public/node/node.rs | 3 --- 18 files changed, 37 insertions(+), 68 deletions(-) diff --git a/wnfs-common/Cargo.toml b/wnfs-common/Cargo.toml index 82c1f10b..88755ea7 100644 --- a/wnfs-common/Cargo.toml +++ b/wnfs-common/Cargo.toml @@ -20,7 +20,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" async-once-cell = "0.5" -async-trait = "0.1" base64 = { version = "0.21", optional = true } base64-serde = { version = "0.7", optional = true } bytes = { version = "1.4", features = ["serde"] } diff --git a/wnfs-common/src/link.rs b/wnfs-common/src/link.rs index d6fb8cec..5e3943c9 100644 --- a/wnfs-common/src/link.rs +++ b/wnfs-common/src/link.rs @@ -1,7 +1,6 @@ use crate::{traits::IpldEq, utils::CondSync, BlockStore, Storable}; use anyhow::Result; use async_once_cell::OnceCell; -use async_trait::async_trait; use libipld::Cid; use std::fmt::{self, Debug, Formatter}; @@ -137,8 +136,6 @@ impl Link { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl IpldEq for Link { async fn eq(&self, other: &Link, store: &impl BlockStore) -> Result { if self == other { @@ -227,7 +224,6 @@ mod tests { use crate::{BlockStore, Link, MemoryBlockStore, Storable}; use anyhow::Result; use async_once_cell::OnceCell; - use async_trait::async_trait; use libipld::Cid; use serde::{Deserialize, Serialize}; @@ -238,7 +234,6 @@ mod tests { persisted_as: OnceCell, } - #[async_trait] impl Storable for Example { type Serializable = Example; diff --git a/wnfs-common/src/storable.rs b/wnfs-common/src/storable.rs index 79854cae..7ecb4141 100644 --- a/wnfs-common/src/storable.rs +++ b/wnfs-common/src/storable.rs @@ -2,13 +2,13 @@ //! that are implemented for most WNFS structures, such as `PublicFile`, `PublicDirectory`, //! `PublicNode`, `HamtForest` etc. use crate::{ - utils::{Arc, CondSync}, + utils::{Arc, CondSend, CondSync}, BlockStore, }; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use bytes::Bytes; +use futures::Future; use libipld::{cbor::DagCborCodec, Cid}; use serde::{de::DeserializeOwned, Serialize}; @@ -20,8 +20,6 @@ use serde::{de::DeserializeOwned, Serialize}; macro_rules! impl_storable_from_serde { ( $( $ty:ty $( : < $( $generics:ident ),+ > )? ),+ ) => { $( - #[cfg_attr(not(target_arch = "wasm32"), ::async_trait::async_trait)] - #[cfg_attr(target_arch = "wasm32", ::async_trait::async_trait(?Send))] impl $( < $( $generics ),+ > )? $crate::Storable for $ty $( where $( $generics: ::serde::Serialize + ::serde::de::DeserializeOwned + Clone + $crate::utils::CondSync ),+ )?{ type Serializable = $ty; @@ -62,19 +60,22 @@ pub use impl_storable_from_serde; /// If you do so, remember to initialize the `OnceCell` if a `Cid` is passed in the /// `from_serializable` call, such that a `store` call right after a `load` call is practically /// free. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait Storable: Sized { /// The at-rest representation of this storable type. type Serializable: StoreIpld + LoadIpld + CondSync; /// Turn the current type into the at-rest representation of this type. - async fn to_serializable(&self, store: &impl BlockStore) -> Result; + fn to_serializable( + &self, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Take an at-rest representation of this type and turn it into the in-memory representation. /// You can use the `cid` parameter to populate a cache. - async fn from_serializable(cid: Option<&Cid>, serializable: Self::Serializable) - -> Result; + fn from_serializable( + cid: Option<&Cid>, + serializable: Self::Serializable, + ) -> impl Future> + CondSend; /// Return a serialization cache, if it exists. /// By default, this always returns `None`. @@ -85,16 +86,21 @@ pub trait Storable: Sized { /// Store this data type in a given `BlockStore`. /// /// This will short-circuit by using the `persisted_as` once-cell, if available. - async fn store(&self, store: &impl BlockStore) -> Result { + fn store(&self, store: &impl BlockStore) -> impl Future> + CondSend + where + Self: CondSync, + { let store_future = async { let (bytes, codec) = self.to_serializable(store).await?.encode_ipld()?; store.put_block(bytes, codec).await }; - if let Some(persisted_as) = self.persisted_as() { - persisted_as.get_or_try_init(store_future).await.cloned() - } else { - store_future.await + async { + if let Some(persisted_as) = self.persisted_as() { + persisted_as.get_or_try_init(store_future).await.cloned() + } else { + store_future.await + } } } @@ -102,10 +108,12 @@ pub trait Storable: Sized { /// /// This will pass on the CID to the `from_serializable` function so it can /// populate a cache in some cases. - async fn load(cid: &Cid, store: &impl BlockStore) -> Result { - let bytes = store.get_block(cid).await?; - let serializable = Self::Serializable::decode_ipld(cid, bytes)?; - Self::from_serializable(Some(cid), serializable).await + fn load(cid: &Cid, store: &impl BlockStore) -> impl Future> + CondSend { + async { + let bytes = store.get_block(cid).await?; + let serializable = Self::Serializable::decode_ipld(cid, bytes)?; + Self::from_serializable(Some(cid), serializable).await + } } } @@ -158,8 +166,6 @@ impl LoadIpld for T { // } // } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Arc { type Serializable = T::Serializable; diff --git a/wnfs-common/src/traits.rs b/wnfs-common/src/traits.rs index 5cd3afff..23aab5df 100644 --- a/wnfs-common/src/traits.rs +++ b/wnfs-common/src/traits.rs @@ -1,15 +1,17 @@ -use crate::BlockStore; +use crate::{utils::CondSend, BlockStore}; use anyhow::Result; -use async_trait::async_trait; +use futures::Future; //-------------------------------------------------------------------------------------------------- // Traits //-------------------------------------------------------------------------------------------------- /// Implements deep equality check for two types. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait IpldEq { /// Checks if the two items are deeply equal. - async fn eq(&self, other: &Self, store: &impl BlockStore) -> Result; + fn eq( + &self, + other: &Self, + store: &impl BlockStore, + ) -> impl Future> + CondSend; } diff --git a/wnfs-hamt/Cargo.toml b/wnfs-hamt/Cargo.toml index 28fff289..431949a5 100644 --- a/wnfs-hamt/Cargo.toml +++ b/wnfs-hamt/Cargo.toml @@ -21,7 +21,6 @@ authors = ["The Fission Authors"] anyhow = "1.0" async-once-cell = "0.5" async-recursion = "1.0" -async-trait = "0.1" bitvec = { version = "1.0", features = ["serde"] } blake3 = { version = "1.4", features = ["traits-preview"] } chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } diff --git a/wnfs-hamt/src/hamt.rs b/wnfs-hamt/src/hamt.rs index 19980295..2211c96f 100644 --- a/wnfs-hamt/src/hamt.rs +++ b/wnfs-hamt/src/hamt.rs @@ -1,7 +1,6 @@ use super::{KeyValueChange, Node, HAMT_VERSION}; use crate::{serializable::HamtSerializable, Hasher}; use anyhow::Result; -use async_trait::async_trait; use libipld::Cid; use semver::Version; use serde::{de::DeserializeOwned, Serialize}; @@ -129,8 +128,6 @@ impl Hamt { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Hamt where K: Storable + CondSync, diff --git a/wnfs-hamt/src/node.rs b/wnfs-hamt/src/node.rs index 7624be6e..43baa161 100644 --- a/wnfs-hamt/src/node.rs +++ b/wnfs-hamt/src/node.rs @@ -7,7 +7,6 @@ use crate::{serializable::NodeSerializable, HAMT_VALUES_BUCKET_SIZE}; use anyhow::{bail, Result}; use async_once_cell::OnceCell; use async_recursion::async_recursion; -use async_trait::async_trait; use bitvec::array::BitArray; use either::{Either, Either::*}; use libipld::Cid; @@ -22,7 +21,7 @@ use std::{ marker::PhantomData, }; use wnfs_common::{ - utils::{Arc, BoxFuture, CondSend, CondSync}, + utils::{boxed_fut, Arc, BoxFuture, CondSend, CondSync}, BlockStore, HashOutput, Link, Storable, }; @@ -838,8 +837,6 @@ where } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Node where K: Storable + CondSync, @@ -855,7 +852,7 @@ where let mut pointers = Vec::with_capacity(self.pointers.len()); for pointer in self.pointers.iter() { - pointers.push(pointer.to_serializable(store).await?); + pointers.push(boxed_fut(pointer.to_serializable(store)).await?); } Ok(NodeSerializable(bitmask, pointers)) diff --git a/wnfs-hamt/src/pointer.rs b/wnfs-hamt/src/pointer.rs index e115b155..042589d9 100644 --- a/wnfs-hamt/src/pointer.rs +++ b/wnfs-hamt/src/pointer.rs @@ -1,7 +1,6 @@ use super::{error::HamtError, hash::Hasher, Node, HAMT_VALUES_BUCKET_SIZE}; use crate::serializable::PointerSerializable; use anyhow::Result; -use async_trait::async_trait; use libipld::Cid; use serde::{de::DeserializeOwned, Serialize}; use std::fmt::Debug; @@ -110,8 +109,6 @@ impl Pointer { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Pointer where K: Storable + CondSync, @@ -155,8 +152,6 @@ where } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Pair where K: Storable + CondSync, diff --git a/wnfs-nameaccumulator/Cargo.toml b/wnfs-nameaccumulator/Cargo.toml index 6732aec7..ea449b70 100644 --- a/wnfs-nameaccumulator/Cargo.toml +++ b/wnfs-nameaccumulator/Cargo.toml @@ -18,7 +18,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" -async-trait = "0.1" blake3 = { version = "1.4", features = ["traits-preview"] } libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] } num-bigint-dig = { version = "0.8.2", features = ["prime", "zeroize"], optional = true } diff --git a/wnfs-nameaccumulator/src/name.rs b/wnfs-nameaccumulator/src/name.rs index 81a8eef7..eee9c89b 100644 --- a/wnfs-nameaccumulator/src/name.rs +++ b/wnfs-nameaccumulator/src/name.rs @@ -443,8 +443,6 @@ impl<'a, B: Big> BatchedProofVerification<'a, B> { macro_rules! impl_storable { ( $ty:ty ) => { - #[cfg_attr(not(target_arch = "wasm32"), ::async_trait::async_trait)] - #[cfg_attr(target_arch = "wasm32", ::async_trait::async_trait(?Send))] impl Storable for $ty { type Serializable = $ty; diff --git a/wnfs-unixfs-file/Cargo.toml b/wnfs-unixfs-file/Cargo.toml index a0cee00a..7bc0b70a 100644 --- a/wnfs-unixfs-file/Cargo.toml +++ b/wnfs-unixfs-file/Cargo.toml @@ -20,7 +20,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" async-stream = "0.3" -async-trait = "0.1" bytes = "1.5" futures = "0.3" libipld = { version = "0.16", features = [] } diff --git a/wnfs-unixfs-file/src/unixfs.rs b/wnfs-unixfs-file/src/unixfs.rs index 74b02658..2c3f9db2 100644 --- a/wnfs-unixfs-file/src/unixfs.rs +++ b/wnfs-unixfs-file/src/unixfs.rs @@ -5,7 +5,6 @@ use crate::{ types::{Block, Link, LinkRef, Links, PbLinks}, }; use anyhow::{anyhow, bail, ensure, Result}; -use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; use libipld::Cid; @@ -223,8 +222,6 @@ impl UnixFsFile { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for UnixFsFile { type Serializable = UnixFsFile; diff --git a/wnfs-wasm/Cargo.toml b/wnfs-wasm/Cargo.toml index 28582337..a850c4f7 100644 --- a/wnfs-wasm/Cargo.toml +++ b/wnfs-wasm/Cargo.toml @@ -19,7 +19,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" -async-trait = "0.1" bytes = "1.4.0" cfg-if = "1.0" chrono = { version = "0.4", default-features = false, features = ["clock", "std", "wasmbind"] } diff --git a/wnfs/Cargo.toml b/wnfs/Cargo.toml index d532b41b..0fd9bd0f 100644 --- a/wnfs/Cargo.toml +++ b/wnfs/Cargo.toml @@ -23,7 +23,6 @@ anyhow = "1.0" async-once-cell = "0.5" async-recursion = "1.0" async-stream = "0.3" -async-trait = "0.1" blake3 = { version = "1.4", features = ["traits-preview"] } bytes = "1.4.0" chacha20poly1305 = "0.10" diff --git a/wnfs/src/private/forest/hamt.rs b/wnfs/src/private/forest/hamt.rs index 4f34c0fd..cb1fed72 100644 --- a/wnfs/src/private/forest/hamt.rs +++ b/wnfs/src/private/forest/hamt.rs @@ -1,7 +1,6 @@ use super::traits::PrivateForest; use crate::error::FsError; use anyhow::Result; -use async_trait::async_trait; use libipld_core::cid::Cid; use quick_cache::sync::Cache; use rand_core::CryptoRngCore; @@ -393,8 +392,6 @@ impl PrivateForest for Arc { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for HamtForest { type Serializable = HamtForestSerializable; diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 8c534e8f..9bf9762f 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -8,12 +8,11 @@ use crate::{ }; use anyhow::{bail, ensure, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use libipld_core::cid::Cid; use std::collections::{BTreeMap, BTreeSet}; use wnfs_common::{ - utils::{error, Arc}, + utils::{boxed_fut, error, Arc}, BlockStore, Metadata, NodeType, Storable, }; @@ -827,8 +826,6 @@ impl Clone for PublicDirectory { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicDirectory { type Serializable = PublicNodeSerializable; @@ -836,7 +833,7 @@ impl Storable for PublicDirectory { let userland = { let mut map = BTreeMap::new(); for (name, link) in self.userland.iter() { - map.insert(name.clone(), link.resolve_cid(store).await?); + map.insert(name.clone(), boxed_fut(link.resolve_cid(store)).await?); } map }; diff --git a/wnfs/src/public/file.rs b/wnfs/src/public/file.rs index 8720a6da..b5f4108d 100644 --- a/wnfs/src/public/file.rs +++ b/wnfs/src/public/file.rs @@ -4,7 +4,6 @@ use super::{PublicFileSerializable, PublicNodeSerializable}; use crate::{error::FsError, is_readable_wnfs_version, traits::Id, WNFS_VERSION}; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::{AsyncRead, AsyncReadExt}; use libipld_core::cid::Cid; @@ -422,8 +421,6 @@ impl PublicFile { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicFile { type Serializable = PublicNodeSerializable; diff --git a/wnfs/src/public/node/node.rs b/wnfs/src/public/node/node.rs index 4eb9ea81..fb0bc4e1 100644 --- a/wnfs/src/public/node/node.rs +++ b/wnfs/src/public/node/node.rs @@ -8,7 +8,6 @@ use crate::{ }; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use libipld_core::cid::Cid; use std::collections::BTreeSet; @@ -267,8 +266,6 @@ impl From for PublicNode { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicNode { type Serializable = PublicNodeSerializable; From 92dc8cd78beb65654b504f7474a8fd487d1a0329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 17:20:55 +0100 Subject: [PATCH 06/20] chore: Fix warnings, remove unused `IpldEq` trait --- wnfs-common/src/lib.rs | 1 - wnfs-common/src/link.rs | 12 +----------- wnfs-common/src/traits.rs | 17 ----------------- wnfs/src/lib.rs | 2 +- 4 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 wnfs-common/src/traits.rs diff --git a/wnfs-common/src/lib.rs b/wnfs-common/src/lib.rs index b3ac7b9a..74302b54 100644 --- a/wnfs-common/src/lib.rs +++ b/wnfs-common/src/lib.rs @@ -6,7 +6,6 @@ mod link; mod metadata; mod pathnodes; mod storable; -mod traits; pub mod utils; pub use blockstore::*; diff --git a/wnfs-common/src/link.rs b/wnfs-common/src/link.rs index 5e3943c9..8d093301 100644 --- a/wnfs-common/src/link.rs +++ b/wnfs-common/src/link.rs @@ -1,4 +1,4 @@ -use crate::{traits::IpldEq, utils::CondSync, BlockStore, Storable}; +use crate::{utils::CondSync, BlockStore, Storable}; use anyhow::Result; use async_once_cell::OnceCell; use libipld::Cid; @@ -136,16 +136,6 @@ impl Link { } } -impl IpldEq for Link { - async fn eq(&self, other: &Link, store: &impl BlockStore) -> Result { - if self == other { - return Ok(true); - } - - Ok(self.resolve_cid(store).await? == other.resolve_cid(store).await?) - } -} - impl From for Link { fn from(value: T) -> Self { Self::Decoded { value } diff --git a/wnfs-common/src/traits.rs b/wnfs-common/src/traits.rs deleted file mode 100644 index 23aab5df..00000000 --- a/wnfs-common/src/traits.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::{utils::CondSend, BlockStore}; -use anyhow::Result; -use futures::Future; - -//-------------------------------------------------------------------------------------------------- -// Traits -//-------------------------------------------------------------------------------------------------- - -/// Implements deep equality check for two types. -pub trait IpldEq { - /// Checks if the two items are deeply equal. - fn eq( - &self, - other: &Self, - store: &impl BlockStore, - ) -> impl Future> + CondSend; -} diff --git a/wnfs/src/lib.rs b/wnfs/src/lib.rs index bbb7c8c8..bef16225 100644 --- a/wnfs/src/lib.rs +++ b/wnfs/src/lib.rs @@ -161,7 +161,7 @@ pub const WNFS_VERSION: semver::Version = semver::Version::new(1, 0, 0); /// The result of an basic get operation. pub(crate) enum SearchResult { Missing(T, usize), - NotADir(T, usize), + NotADir(T, #[allow(unused)] usize), Found(T), } From 80b90f3ee0ecd8f97d3432d48892fae3386ca669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 17:26:12 +0100 Subject: [PATCH 07/20] fix: Update rug & enable std feature --- wnfs-nameaccumulator/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wnfs-nameaccumulator/Cargo.toml b/wnfs-nameaccumulator/Cargo.toml index ea449b70..a9fa265a 100644 --- a/wnfs-nameaccumulator/Cargo.toml +++ b/wnfs-nameaccumulator/Cargo.toml @@ -25,7 +25,7 @@ num-integer = "0.1.45" num-traits = "0.2.15" once_cell = "1.0" rand_core = "0.6" -rug = { version = "1.22", optional = true, default-features = false, features = ["rand", "integer", "num-traits"] } +rug = { version = "1.24", optional = true, default-features = false, features = ["std", "rand", "integer", "num-traits"] } serde = { version = "1.0", features = ["rc"] } serde_bytes = "0.11.9" thiserror = "1.0" From 19ce5694d33c33a36309d0a6533b46d28f87a010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 17:36:12 +0100 Subject: [PATCH 08/20] feat: don't require `std` for `rug`, more efficient `to_bytes_be` --- wnfs-nameaccumulator/Cargo.toml | 2 +- wnfs-nameaccumulator/src/traits.rs | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/wnfs-nameaccumulator/Cargo.toml b/wnfs-nameaccumulator/Cargo.toml index a9fa265a..726f93aa 100644 --- a/wnfs-nameaccumulator/Cargo.toml +++ b/wnfs-nameaccumulator/Cargo.toml @@ -25,7 +25,7 @@ num-integer = "0.1.45" num-traits = "0.2.15" once_cell = "1.0" rand_core = "0.6" -rug = { version = "1.24", optional = true, default-features = false, features = ["std", "rand", "integer", "num-traits"] } +rug = { version = "1.24", optional = true, default-features = false, features = ["rand", "integer", "num-traits"] } serde = { version = "1.0", features = ["rc"] } serde_bytes = "0.11.9" thiserror = "1.0" diff --git a/wnfs-nameaccumulator/src/traits.rs b/wnfs-nameaccumulator/src/traits.rs index 3ff2d68d..8d80c984 100644 --- a/wnfs-nameaccumulator/src/traits.rs +++ b/wnfs-nameaccumulator/src/traits.rs @@ -238,10 +238,8 @@ impl Big for BigNumRug { } fn to_bytes_be(n: &Self::Num) -> [u8; N] { - let vec = n.to_digits(Order::MsfLe); let mut bytes = [0u8; N]; - let zero_bytes = N - vec.len(); - bytes[zero_bytes..].copy_from_slice(&vec); + n.write_digits(&mut bytes, Order::MsfLe); bytes } @@ -293,6 +291,8 @@ mod rug_tests { use crate::{Big, BigNumRug}; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; + use rug::Integer; + use std::str::FromStr; /// We need this property for snapshot testing #[test] @@ -314,4 +314,18 @@ mod rug_tests { let run_three = BigNumRug::rand_below(&ceiling, &mut ChaCha12Rng::seed_from_u64(1)); assert_ne!(run_one, run_three); } + + #[test] + fn test_to_bytes_be_snapshot() { + let modulus = Integer::from_str( + "25195908475657893494027183240048398571429282126204032027777137836043662020707595556264018525880784406918290641249515082189298559149176184502808489120072844992687392807287776735971418347270261896375014971824691165077613379859095700097330459748808428401797429100642458691817195118746121515172654632282216869987549182422433637259085141865462043576798423387184774447920739934236584823824281198163815010674810451660377306056201619676256133844143603833904414952634432190114657544454178424020924616515723350778707749817125772467962926386356373289912154831438167899885040445364023527381951378636564391212010397122822120720357", + ).expect("Can parse integer"); + + let modulus_hex = "c7970ceedcc3b0754490201a7aa613cd73911081c790f5f1a8726f463550bb5b7ff0db8e1ea1189ec72f93d1650011bd721aeeacc2acde32a04107f0648c2813a31f5b0b7765ff8b44b4b6ffc93384b646eb09c7cf5e8592d40ea33c80039f35b4f14a04b51f7bfd781be4d1673164ba8eb991c2c4d730bbbe35f592bdef524af7e8daefd26c66fc02c479af89d64d373f442709439de66ceb955f3ea37d5159f6135809f85334b5cb1813addc80cd05609f10ac6a95ad65872c909525bdad32bc729592642920f24c61dc5b3c3b7923e56b16a4d9d373d8721f24a3fc0f1b3131f55615172866bccc30f95054c824e733a5eb6817f7bc16399d48c6361cc7e5"; + + let bytes = BigNumRug::to_bytes_be::<256>(&modulus); + let hex_encoded = hex::encode(bytes); + + assert_eq!(hex_encoded, modulus_hex); + } } From 4b67fd502770c680aef7e29d065ae289cefe3dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 17:37:29 +0100 Subject: [PATCH 09/20] chore: Fix nightly warning --- wnfs-common/src/metadata.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/wnfs-common/src/metadata.rs b/wnfs-common/src/metadata.rs index 8325821b..b20d9291 100644 --- a/wnfs-common/src/metadata.rs +++ b/wnfs-common/src/metadata.rs @@ -7,7 +7,7 @@ use serde::{ de::{DeserializeOwned, Error as DeError}, Deserialize, Deserializer, Serialize, Serializer, }; -use std::{collections::BTreeMap, convert::TryInto}; +use std::{collections::BTreeMap, convert::TryInto, fmt::Display}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -24,17 +24,16 @@ pub enum NodeType { SnapshotSharePointer, } -impl ToString for NodeType { - fn to_string(&self) -> String { - match self { +impl Display for NodeType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { NodeType::PublicFile => "wnfs/pub/file", NodeType::PublicDirectory => "wnfs/pub/dir", NodeType::PrivateFile => "wnfs/priv/file", NodeType::PrivateDirectory => "wnfs/priv/dir", NodeType::TemporalSharePointer => "wnfs/share/temporal", NodeType::SnapshotSharePointer => "wnfs/share/snapshot", - } - .to_string() + }) } } From f121a794b5ca9fa868edc6e268ede0b64b0f14d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 18:18:22 +0100 Subject: [PATCH 10/20] refactor: Blanket-impl for `&B` and `Box` instead of `Deref` This way is recommended by dtolnay (https://github.com/rust-lang/api-guidelines/discussions/158) and the "Rust for Rustaceans" book (paragraph "Ergonomic Trait Implementations"). This leads to better compiler error messages when you pass something that doesn't `impl BlockStore` the right way. `rand_core` explicitly decided against a `DerefMut` blanket implementation for `trait RngCore`. --- wnfs-common/src/blockstore.rs | 36 ++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index eeca51f7..7793ef0d 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -14,7 +14,7 @@ use libipld::{ }; use parking_lot::Mutex; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use std::{collections::HashMap, ops::Deref}; +use std::collections::HashMap; //-------------------------------------------------------------------------------------------------- // Constants @@ -96,25 +96,47 @@ pub trait BlockStore: CondSync { } } -impl + CondSync> BlockStore for T { +impl BlockStore for &B { async fn get_block(&self, cid: &Cid) -> Result { - self.deref().get_block(cid).await + (**self).get_block(cid).await } async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { - self.deref().put_block(bytes, codec).await + (**self).put_block(bytes, codec).await } async fn get_deserializable(&self, cid: &Cid) -> Result { - self.deref().get_deserializable(cid).await + (**self).get_deserializable(cid).await } async fn put_serializable(&self, value: &V) -> Result { - self.deref().put_serializable(value).await + (**self).put_serializable(value).await } fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { - self.deref().create_cid(bytes, codec) + (**self).create_cid(bytes, codec) + } +} + +impl BlockStore for Box { + async fn get_block(&self, cid: &Cid) -> Result { + (**self).get_block(cid).await + } + + async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + (**self).put_block(bytes, codec).await + } + + async fn get_deserializable(&self, cid: &Cid) -> Result { + (**self).get_deserializable(cid).await + } + + async fn put_serializable(&self, value: &V) -> Result { + (**self).put_serializable(value).await + } + + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + (**self).create_cid(bytes, codec) } } From 73cb7926f2b0f5b200b8a9e75f1f2d2907709727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 19:14:04 +0100 Subject: [PATCH 11/20] refactor: Remove serializable things from `BlockStore` Use the `Storable` trait and its `store` and `load` functions instead. --- wnfs-common/src/blockstore.rs | 89 +++++-------------- wnfs-common/src/link.rs | 2 +- wnfs-nameaccumulator/src/name.rs | 4 +- wnfs/src/private/keys/access.rs | 27 ++++-- ...s__access__snapshot_tests__access_key.snap | 30 +++---- wnfs/src/public/directory.rs | 14 +-- wnfs/src/root_tree.rs | 9 +- 7 files changed, 74 insertions(+), 101 deletions(-) diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 7793ef0d..81ec3c45 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -10,10 +10,10 @@ use libipld::{ cbor::DagCborCodec, cid::Version, multihash::{Code, MultihashDigest}, - serde as ipld_serde, Cid, + Cid, }; use parking_lot::Mutex; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; //-------------------------------------------------------------------------------------------------- @@ -45,7 +45,7 @@ pub const CODEC_DAG_PB: u64 = 0x70; pub const CODEC_RAW: u64 = 0x55; //-------------------------------------------------------------------------------------------------- -// Type Definitions +// Traits //-------------------------------------------------------------------------------------------------- /// For types that implement block store operations like adding, getting content from the store. @@ -58,27 +58,6 @@ pub trait BlockStore: CondSync { codec: u64, ) -> impl Future> + CondSend; - fn get_deserializable( - &self, - cid: &Cid, - ) -> impl Future> + CondSend { - async { - let bytes = self.get_block(cid).await?; - let ipld = decode(bytes.as_ref(), DagCborCodec)?; - Ok(ipld_serde::from_ipld::(ipld)?) - } - } - - fn put_serializable( - &self, - value: &V, - ) -> impl Future> + CondSend { - async move { - let bytes = encode(&ipld_serde::to_ipld(value)?, DagCborCodec)?; - self.put_block(bytes, CODEC_DAG_CBOR).await - } - } - // This should be the same in all implementations of BlockStore fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { // If there are too many bytes, abandon this task @@ -96,6 +75,10 @@ pub trait BlockStore: CondSync { } } +//-------------------------------------------------------------------------------------------------- +// Implementations +//-------------------------------------------------------------------------------------------------- + impl BlockStore for &B { async fn get_block(&self, cid: &Cid) -> Result { (**self).get_block(cid).await @@ -105,14 +88,6 @@ impl BlockStore for &B { (**self).put_block(bytes, codec).await } - async fn get_deserializable(&self, cid: &Cid) -> Result { - (**self).get_deserializable(cid).await - } - - async fn put_serializable(&self, value: &V) -> Result { - (**self).put_serializable(value).await - } - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } @@ -127,23 +102,11 @@ impl BlockStore for Box { (**self).put_block(bytes, codec).await } - async fn get_deserializable(&self, cid: &Cid) -> Result { - (**self).get_deserializable(cid).await - } - - async fn put_serializable(&self, value: &V) -> Result { - (**self).put_serializable(value).await - } - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } } -//-------------------------------------------------------------------------------------------------- -// Implementations -//-------------------------------------------------------------------------------------------------- - /// An in-memory block store to simulate IPFS. /// /// IPFS is basically a glorified HashMap. @@ -195,21 +158,18 @@ impl BlockStore for MemoryBlockStore { //-------------------------------------------------------------------------------------------------- /// Tests the retrieval property of a BlockStore-conforming type. -pub async fn bs_retrieval_test(store: &T) -> Result<()> -where - T: BlockStore + 'static, -{ +pub async fn bs_retrieval_test(store: impl BlockStore) -> Result<()> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = b"hello world".to_vec(); // Insert the objects into the blockstore - let first_cid = store.put_serializable(&first_bytes).await?; - let second_cid = store.put_serializable(&second_bytes).await?; + let first_cid = store.put_block(first_bytes.clone(), CODEC_RAW).await?; + let second_cid = store.put_block(second_bytes.clone(), CODEC_RAW).await?; // Retrieve the objects from the blockstore - let first_loaded: Vec = store.get_deserializable(&first_cid).await?; - let second_loaded: Vec = store.get_deserializable(&second_cid).await?; + let first_loaded = store.get_block(&first_cid).await?; + let second_loaded = store.get_block(&second_cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(first_loaded, first_bytes); @@ -219,24 +179,21 @@ where } /// Tests the duplication of a BlockStore-conforming type. -pub async fn bs_duplication_test(store: &T) -> Result<()> -where - T: BlockStore + 'static, -{ +pub async fn bs_duplication_test(store: impl BlockStore) -> Result<()> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = first_bytes.clone(); // Insert the objects into the blockstore - let first_cid = store.put_serializable(&first_bytes).await?; - let second_cid = store.put_serializable(&second_bytes).await?; + let first_cid = store.put_block(first_bytes.clone(), CODEC_RAW).await?; + let second_cid = store.put_block(second_bytes.clone(), CODEC_RAW).await?; // Assert that the two vecs produced the same CID assert_eq!(first_cid, second_cid); // Retrieve the objects from the blockstore - let first_loaded: Vec = store.get_deserializable(&first_cid).await?; - let second_loaded: Vec = store.get_deserializable(&second_cid).await?; + let first_loaded = store.get_block(&first_cid).await?; + let second_loaded = store.get_block(&second_cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(first_loaded, first_bytes); @@ -251,20 +208,20 @@ where /// Tests the serialization of a BlockStore-conforming type. pub async fn bs_serialization_test(store: &T) -> Result<()> where - T: BlockStore + Serialize + 'static + for<'de> Deserialize<'de>, + T: BlockStore + Serialize + for<'de> Deserialize<'de>, { // Example objects to insert and remove from the blockstore let bytes = vec![1, 2, 3, 4, 5]; // Insert the object into the blockstore - let cid = store.put_serializable(&bytes).await?; + let cid = store.put_block(bytes.clone(), CODEC_RAW).await?; // Serialize the BlockStore let serial_store: Vec = encode(&store, DagCborCodec)?; // Construct a new BlockStore from the Serialized object let deserial_store: T = decode(&serial_store, DagCborCodec)?; // Retrieve the object from the blockstore - let loaded: Vec = deserial_store.get_deserializable(&cid).await?; + let loaded = deserial_store.get_block(&cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(loaded, bytes); @@ -280,9 +237,9 @@ mod tests { #[async_std::test] async fn memory_blockstore() -> Result<()> { let store = &MemoryBlockStore::new(); - bs_retrieval_test(store).await?; - bs_duplication_test(store).await?; - bs_serialization_test(store).await?; + bs_retrieval_test::(store).await?; + bs_duplication_test::(store).await?; + bs_serialization_test::(store).await?; Ok(()) } } diff --git a/wnfs-common/src/link.rs b/wnfs-common/src/link.rs index 8d093301..1f3bb8b1 100644 --- a/wnfs-common/src/link.rs +++ b/wnfs-common/src/link.rs @@ -277,7 +277,7 @@ mod tests { async fn link_value_can_be_resolved() { let store = &MemoryBlockStore::default(); let example = Example::new(256); - let cid = store.put_serializable(&example).await.unwrap(); + let cid = example.store(store).await.unwrap(); let link = Link::::from_cid(cid); let value = link.resolve_value(store).await.unwrap(); diff --git a/wnfs-nameaccumulator/src/name.rs b/wnfs-nameaccumulator/src/name.rs index eee9c89b..1d73bfa1 100644 --- a/wnfs-nameaccumulator/src/name.rs +++ b/wnfs-nameaccumulator/src/name.rs @@ -821,7 +821,7 @@ mod snapshot_tests { use crate::{BigNumDig, NameSegment}; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; - use wnfs_common::{utils::SnapshotBlockStore, BlockStore}; + use wnfs_common::utils::SnapshotBlockStore; #[async_std::test] async fn test_name_accumulator() { @@ -839,7 +839,7 @@ mod snapshot_tests { setup, ); - let cid = store.put_serializable(&acc).await.unwrap(); + let cid = acc.store(store).await.unwrap(); let name = store.get_block_snapshot(&cid).await.unwrap(); insta::assert_json_snapshot!(name); diff --git a/wnfs/src/private/keys/access.rs b/wnfs/src/private/keys/access.rs index 3c5b1b2e..947b03b4 100644 --- a/wnfs/src/private/keys/access.rs +++ b/wnfs/src/private/keys/access.rs @@ -124,15 +124,19 @@ impl From<&AccessKey> for Vec { #[cfg(test)] mod snapshot_tests { use super::*; + use libipld_core::ipld::Ipld; use rand::Rng; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; - use wnfs_common::{utils::SnapshotBlockStore, BlockStore}; + use testresult::TestResult; + use wnfs_common::{ + decode, encode, + libipld::{cbor::DagCborCodec, json::DagJsonCodec}, + }; #[async_std::test] - async fn test_access_key() { + async fn test_access_key() -> TestResult { let rng = &mut ChaCha12Rng::seed_from_u64(0); - let store = &SnapshotBlockStore::default(); let private_ref = PrivateRef::with_temporal_key(rng.gen(), TemporalKey(rng.gen()), Cid::default()); @@ -140,13 +144,20 @@ mod snapshot_tests { let temporal_access_key = AccessKey::Temporal(TemporalAccessKey::from(&private_ref)); let snapshot_access_key = AccessKey::Snapshot(SnapshotAccessKey::from(&private_ref)); - let temp_cid = store.put_serializable(&temporal_access_key).await.unwrap(); - let snap_cid = store.put_serializable(&snapshot_access_key).await.unwrap(); - - let temp_key = store.get_block_snapshot(&temp_cid).await.unwrap(); - let snap_key = store.get_block_snapshot(&snap_cid).await.unwrap(); + let temp_key = as_dag_json_value(&temporal_access_key)?; + let snap_key = as_dag_json_value(&snapshot_access_key)?; insta::assert_json_snapshot!(temp_key); insta::assert_json_snapshot!(snap_key); + + Ok(()) + } + + fn as_dag_json_value(s: impl Serialize) -> Result { + let dag_cbor = encode(&s, DagCborCodec)?; + let ipld: Ipld = decode(&dag_cbor, DagCborCodec)?; + let dag_json = encode(&ipld, DagJsonCodec)?; + let value = serde_json::from_slice(&dag_json)?; + Ok(value) } } diff --git a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap index 79f2b6fe..fe57ba35 100644 --- a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap +++ b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap @@ -3,23 +3,19 @@ source: wnfs/src/private/keys/access.rs expression: temp_key --- { - "cid": "bafyr4ih2hblsecub5jvxxlbkhjgmabg2mnaxzq7jtzkchd4s4sacqxwbge", - "value": { - "wnfs/share/temporal": { - "contentCid": { - "/": "baeaaaaa" - }, - "label": { - "/": { - "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" - } - }, - "temporalKey": { - "/": { - "bytes": "mmMoPLrw/bzrH2R5sZfzqI3Q2Akv5yp8VigVOHOLB+I" - } + "wnfs/share/temporal": { + "contentCid": { + "/": "baeaaaaa" + }, + "label": { + "/": { + "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" + } + }, + "temporalKey": { + "/": { + "bytes": "mmMoPLrw/bzrH2R5sZfzqI3Q2Akv5yp8VigVOHOLB+I" } } - }, - "bytes": "oXN3bmZzL3NoYXJlL3RlbXBvcmFso2VsYWJlbFggf7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooRqY29udGVudENpZNgqRQABAAAAa3RlbXBvcmFsS2V5WCCaYyg8uvD9vOsfZHmxl/OojdDYCS/nKnxWKBU4c4sH4g==" + } } diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 9bf9762f..02cd6d80 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -891,7 +891,7 @@ mod tests { use chrono::Utc; use libipld_core::ipld::Ipld; use testresult::TestResult; - use wnfs_common::MemoryBlockStore; + use wnfs_common::{decode, libipld::cbor::DagCborCodec, MemoryBlockStore}; #[async_std::test] async fn look_up_can_fetch_file_added_to_directory() -> TestResult { @@ -1242,10 +1242,14 @@ mod tests { root_dir.mkdir(&["test".into()], time, store).await.unwrap(); - let ipld = store - .get_deserializable::(&root_dir.store(store).await.unwrap()) - .await - .unwrap(); + let ipld: Ipld = decode( + &store + .get_block(&root_dir.store(store).await.unwrap()) + .await + .unwrap(), + DagCborCodec, + ) + .unwrap(); match ipld { Ipld::Map(map) => match map.get("wnfs/pub/dir") { Some(Ipld::Map(content)) => match content.get("previous") { diff --git a/wnfs/src/root_tree.rs b/wnfs/src/root_tree.rs index e11b7e3b..71972c88 100644 --- a/wnfs/src/root_tree.rs +++ b/wnfs/src/root_tree.rs @@ -25,6 +25,8 @@ use std::collections::HashMap; #[cfg(test)] use wnfs_common::MemoryBlockStore; use wnfs_common::{ + decode, encode, + libipld::cbor::DagCborCodec, utils::{Arc, CondSend}, BlockStore, Metadata, Storable, }; @@ -309,7 +311,9 @@ where version: WNFS_VERSION, }; - store.put_serializable(&serializable).await + store + .put_block(encode(&serializable, DagCborCodec)?, DagCborCodec.into()) + .await } pub async fn load( @@ -318,7 +322,8 @@ where rng: R, private_map: HashMap, Arc>, ) -> Result> { - let deserialized: RootTreeSerializable = store.get_deserializable(cid).await?; + let deserialized: RootTreeSerializable = + decode(&store.get_block(cid).await?, DagCborCodec)?; let forest = Arc::new(HamtForest::load(&deserialized.forest, store).await?); let public_root = Arc::new(PublicDirectory::load(&deserialized.public, store).await?); let exchange_root = Arc::new(PublicDirectory::load(&deserialized.exchange, store).await?); From 11d7a673a7dcb214bcb273e1b33d6b1396e58adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Wed, 14 Feb 2024 19:26:13 +0100 Subject: [PATCH 12/20] feat: Add `has_block` to `trait BlockStore` --- wnfs-common/src/blockstore.rs | 34 +++++++++++++++++++++++++++-- wnfs-common/src/utils/test.rs | 5 +++++ wnfs/examples/tiered_blockstores.rs | 8 +++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 81ec3c45..c683ad8c 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -50,14 +50,34 @@ pub const CODEC_RAW: u64 = 0x55; /// For types that implement block store operations like adding, getting content from the store. pub trait BlockStore: CondSync { + /// Retrieve a block from this store via its hash (`Cid`). + /// + /// If this store can't find the block, it may raise an error like `BlockNotFound`. fn get_block(&self, cid: &Cid) -> impl Future> + CondSend; + /// Put some bytes into the blockstore. These bytes should be encoded with the given codec. + /// + /// E.g. `CODEC_RAW` for raw bytes blocks, `CODEC_DAG_CBOR` for dag-cbor, etc. + /// + /// This codec will determine the codec encoded in the final `Cid` that's returned. + /// + /// If the codec is incorrect, this function won't fail, but any tools that depend on the + /// correctness of the codec may fail. (E.g. tools that follow the links of blocks). + /// + /// This funciton allows the blockstore to choose the hashing function itself. + /// The hashing function that was chosen will be readable from the `Cid` metadata. fn put_block( &self, bytes: impl Into + CondSend, codec: u64, ) -> impl Future> + CondSend; + /// Find out whether a call to `get_block` would return with a result or not. + /// + /// This is useful for data exchange protocols to find out what needs to be fetched + /// externally and what doesn't. + fn has_block(&self, cid: &Cid) -> impl Future> + CondSend; + // This should be the same in all implementations of BlockStore fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { // If there are too many bytes, abandon this task @@ -88,6 +108,10 @@ impl BlockStore for &B { (**self).put_block(bytes, codec).await } + async fn has_block(&self, cid: &Cid) -> Result { + (**self).has_block(cid).await + } + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } @@ -102,6 +126,10 @@ impl BlockStore for Box { (**self).put_block(bytes, codec).await } + async fn has_block(&self, cid: &Cid) -> Result { + (**self).has_block(cid).await + } + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } @@ -126,7 +154,6 @@ impl MemoryBlockStore { } impl BlockStore for MemoryBlockStore { - /// Retrieves an array of bytes from the block store with given CID. async fn get_block(&self, cid: &Cid) -> Result { let bytes = self .0 @@ -138,7 +165,6 @@ impl BlockStore for MemoryBlockStore { Ok(bytes) } - /// Stores an array of bytes in the block store. async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { // Convert the bytes into a Bytes object let bytes: Bytes = bytes.into(); @@ -151,6 +177,10 @@ impl BlockStore for MemoryBlockStore { Ok(cid) } + + async fn has_block(&self, cid: &Cid) -> Result { + Ok(self.0.lock().contains_key(cid)) + } } //-------------------------------------------------------------------------------------------------- diff --git a/wnfs-common/src/utils/test.rs b/wnfs-common/src/utils/test.rs index 44d8a3b5..78994cc3 100644 --- a/wnfs-common/src/utils/test.rs +++ b/wnfs-common/src/utils/test.rs @@ -121,6 +121,11 @@ impl BlockStore for SnapshotBlockStore { async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { self.inner.put_block(bytes, codec).await } + + #[inline] + async fn has_block(&self, cid: &Cid) -> Result { + self.inner.has_block(cid).await + } } impl Sampleable for S diff --git a/wnfs/examples/tiered_blockstores.rs b/wnfs/examples/tiered_blockstores.rs index a46976bf..75d067db 100644 --- a/wnfs/examples/tiered_blockstores.rs +++ b/wnfs/examples/tiered_blockstores.rs @@ -108,4 +108,12 @@ impl BlockStore for TieredBlockStore { async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { self.hot.put_block(bytes.into(), codec).await } + + async fn has_block(&self, cid: &Cid) -> Result { + if self.hot.has_block(cid).await? { + return Ok(true); + } + + self.cold.has_block(cid).await + } } From 983ae3f823d1c988e39070c71d83006336563a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 10:47:40 +0100 Subject: [PATCH 13/20] fix: Update accesskey snapshot --- ..._access__snapshot_tests__access_key-2.snap | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap index 2976296c..361e1745 100644 --- a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap +++ b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap @@ -3,23 +3,19 @@ source: wnfs/src/private/keys/access.rs expression: snap_key --- { - "cid": "bafyr4ics27f5xqe7ryshkpoln5w7nkvvsdnoaw55r5gfb5earluczlil4a", - "value": { - "wnfs/share/snapshot": { - "contentCid": { - "/": "baeaaaaa" - }, - "label": { - "/": { - "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" - } - }, - "snapshotKey": { - "/": { - "bytes": "7SoxAmhQW3XHxn5UpettI89xokpcEjFXapTSkFRQleU" - } + "wnfs/share/snapshot": { + "contentCid": { + "/": "baeaaaaa" + }, + "label": { + "/": { + "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" + } + }, + "snapshotKey": { + "/": { + "bytes": "7SoxAmhQW3XHxn5UpettI89xokpcEjFXapTSkFRQleU" } } - }, - "bytes": "oXN3bmZzL3NoYXJlL3NuYXBzaG90o2VsYWJlbFggf7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooRqY29udGVudENpZNgqRQABAAAAa3NuYXBzaG90S2V5WCDtKjECaFBbdcfGflSl620jz3GiSlwSMVdqlNKQVFCV5Q==" + } } From e2a2e2f03a8d791e2b265d81d3c218307c24c22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 10:58:01 +0100 Subject: [PATCH 14/20] fix: Implement `has_block` for `ForeignBlockStore` --- wnfs-wasm/src/fs/blockstore.rs | 17 +++++++++++++---- wnfs-wasm/tests/mock.ts | 10 ++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index a29dec81..d19987a5 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -21,6 +21,9 @@ extern "C" { #[wasm_bindgen(method, js_name = "putBlock")] pub(crate) fn put_block(store: &BlockStore, bytes: Vec, codec: u32) -> Promise; + #[wasm_bindgen(method, js_name = "hasBlock")] + pub(crate) fn has_block(store: &BlockStore, cid: Vec) -> Promise; + #[wasm_bindgen(method, js_name = "getBlock")] pub(crate) fn get_block(store: &BlockStore, cid: Vec) -> Promise; } @@ -38,13 +41,12 @@ pub struct ForeignBlockStore(pub(crate) BlockStore); //-------------------------------------------------------------------------------------------------- impl WnfsBlockStore for ForeignBlockStore { - /// Stores an array of bytes in the block store. async fn put_block(&self, bytes: impl Into, codec: u64) -> Result { let bytes: Bytes = bytes.into(); let value = JsFuture::from(self.0.put_block(bytes.into(), codec.try_into()?)) .await - .map_err(anyhow_error("Cannot get block: {:?}"))?; + .map_err(anyhow_error("Cannot put block: {:?}"))?; // Convert the value to a vector of bytes. let bytes = Uint8Array::new(&value).to_vec(); @@ -53,8 +55,7 @@ impl WnfsBlockStore for ForeignBlockStore { Ok(Cid::try_from(&bytes[..])?) } - /// Retrieves an array of bytes from the block store with given CID. - async fn get_block<'a>(&'a self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { let value = JsFuture::from(self.0.get_block(cid.to_bytes())) .await .map_err(anyhow_error("Cannot get block: {:?}"))?; @@ -63,4 +64,12 @@ impl WnfsBlockStore for ForeignBlockStore { let bytes = Uint8Array::new(&value).to_vec(); Ok(Bytes::from(bytes)) } + + async fn has_block(&self, cid: &Cid) -> Result { + let value = JsFuture::from(self.0.has_block(cid.to_bytes())) + .await + .map_err(anyhow_error("Cannot run has_block: {:?}"))?; + + Ok(js_sys::Boolean::from(value).value_of()) + } } diff --git a/wnfs-wasm/tests/mock.ts b/wnfs-wasm/tests/mock.ts index fb5fa0a6..4a7423c5 100644 --- a/wnfs-wasm/tests/mock.ts +++ b/wnfs-wasm/tests/mock.ts @@ -30,8 +30,8 @@ class MemoryBlockStore { /** Stores an array of bytes in the block store. */ async getBlock(cid: Uint8Array): Promise { - const decoded_cid = CID.decode(cid); - return this.store.get(decoded_cid.toString()); + const decodedCid = CID.decode(cid); + return this.store.get(decodedCid.toString()); } /** Retrieves an array of bytes from the block store with given CID. */ @@ -41,6 +41,12 @@ class MemoryBlockStore { this.store.set(cid.toString(), bytes); return cid.bytes; } + + /** Finds out whether a block is retrievable from this blockstore */ + async hasBlock(cid: Uint8Array): Promise { + const decodedCid = CID.decode(cid); + return this.store.has(decodedCid.toString()); + } } /** A pseudo-random number generator */ From 32e352896bc8f9c1b477b24a648ea0f33d5b3c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 12:08:59 +0100 Subject: [PATCH 15/20] feat: Add `get_block_keyed` to `trait BlockStore`, fix wasm --- wnfs-common/src/blockstore.rs | 47 ++++++++++++++++++++++------- wnfs-common/src/utils/test.rs | 5 +++ wnfs-wasm/src/fs/blockstore.rs | 20 +++++------- wnfs-wasm/src/fs/types.rs | 3 +- wnfs-wasm/tests/mock.ts | 8 ++--- wnfs-wasm/tests/public.spec.ts | 2 +- wnfs/examples/tiered_blockstores.rs | 15 +++++---- 7 files changed, 64 insertions(+), 36 deletions(-) diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index c683ad8c..64adfbf9 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -66,11 +66,35 @@ pub trait BlockStore: CondSync { /// /// This funciton allows the blockstore to choose the hashing function itself. /// The hashing function that was chosen will be readable from the `Cid` metadata. + /// + /// If you need control over the concrete hashing function that's used, see `put_block_keyed`. fn put_block( &self, bytes: impl Into + CondSend, codec: u64, - ) -> impl Future> + CondSend; + ) -> impl Future> + CondSend { + let bytes = bytes.into(); + async move { + let cid = self.create_cid(&bytes, codec)?; + self.put_block_keyed(cid, bytes).await?; + Ok(cid) + } + } + + /// Put a block of data into this blockstore. The block's CID needs to match the CID given. + /// + /// It's up to the blockstore whether to check this fact or assume it when this function is called. + /// + /// The default implementation of `put_block` will use this function under the hood and use + /// the correct CID provided by the `create_cid` function. + /// + /// This is useful to be able to add blocks that were generated from other + /// clients with differently configured hashing functions to this blockstore. + fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> impl Future> + CondSend; /// Find out whether a call to `get_block` would return with a result or not. /// @@ -108,6 +132,10 @@ impl BlockStore for &B { (**self).put_block(bytes, codec).await } + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + (**self).put_block_keyed(cid, bytes).await + } + async fn has_block(&self, cid: &Cid) -> Result { (**self).has_block(cid).await } @@ -126,6 +154,10 @@ impl BlockStore for Box { (**self).put_block(bytes, codec).await } + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + (**self).put_block_keyed(cid, bytes).await + } + async fn has_block(&self, cid: &Cid) -> Result { (**self).has_block(cid).await } @@ -165,17 +197,10 @@ impl BlockStore for MemoryBlockStore { Ok(bytes) } - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { - // Convert the bytes into a Bytes object - let bytes: Bytes = bytes.into(); + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + self.0.lock().insert(cid, bytes.into()); - // Try to build the CID from the bytes and codec - let cid = self.create_cid(&bytes, codec)?; - - // Insert the bytes into the HashMap using the CID as the key - self.0.lock().insert(cid, bytes); - - Ok(cid) + Ok(()) } async fn has_block(&self, cid: &Cid) -> Result { diff --git a/wnfs-common/src/utils/test.rs b/wnfs-common/src/utils/test.rs index 78994cc3..c173da73 100644 --- a/wnfs-common/src/utils/test.rs +++ b/wnfs-common/src/utils/test.rs @@ -122,6 +122,11 @@ impl BlockStore for SnapshotBlockStore { self.inner.put_block(bytes, codec).await } + #[inline] + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + self.inner.put_block_keyed(cid, bytes).await + } + #[inline] async fn has_block(&self, cid: &Cid) -> Result { self.inner.has_block(cid).await diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index d19987a5..681dc7e5 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -18,14 +18,14 @@ extern "C" { #[wasm_bindgen(typescript_type = "BlockStore")] pub type BlockStore; - #[wasm_bindgen(method, js_name = "putBlock")] - pub(crate) fn put_block(store: &BlockStore, bytes: Vec, codec: u32) -> Promise; - - #[wasm_bindgen(method, js_name = "hasBlock")] - pub(crate) fn has_block(store: &BlockStore, cid: Vec) -> Promise; + #[wasm_bindgen(method, js_name = "putBlockKeyed")] + pub(crate) fn put_block_keyed(store: &BlockStore, cid: Vec, bytes: Vec) -> Promise; #[wasm_bindgen(method, js_name = "getBlock")] pub(crate) fn get_block(store: &BlockStore, cid: Vec) -> Promise; + + #[wasm_bindgen(method, js_name = "hasBlock")] + pub(crate) fn has_block(store: &BlockStore, cid: Vec) -> Promise; } //-------------------------------------------------------------------------------------------------- @@ -41,18 +41,14 @@ pub struct ForeignBlockStore(pub(crate) BlockStore); //-------------------------------------------------------------------------------------------------- impl WnfsBlockStore for ForeignBlockStore { - async fn put_block(&self, bytes: impl Into, codec: u64) -> Result { + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into) -> Result<()> { let bytes: Bytes = bytes.into(); - let value = JsFuture::from(self.0.put_block(bytes.into(), codec.try_into()?)) + JsFuture::from(self.0.put_block_keyed(cid.to_bytes(), bytes.into())) .await .map_err(anyhow_error("Cannot put block: {:?}"))?; - // Convert the value to a vector of bytes. - let bytes = Uint8Array::new(&value).to_vec(); - - // Construct CID from the bytes. - Ok(Cid::try_from(&bytes[..])?) + Ok(()) } async fn get_block(&self, cid: &Cid) -> Result { diff --git a/wnfs-wasm/src/fs/types.rs b/wnfs-wasm/src/fs/types.rs index 3b52cbd1..002d4344 100644 --- a/wnfs-wasm/src/fs/types.rs +++ b/wnfs-wasm/src/fs/types.rs @@ -3,7 +3,8 @@ use wasm_bindgen::prelude::wasm_bindgen; #[wasm_bindgen(typescript_custom_section)] const TS_BLOCKSTORE: &'static str = r#" export interface BlockStore { - putBlock(bytes: Uint8Array, code: number): Promise; + putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise; getBlock(cid: Uint8Array): Promise; + hasBlock(cid: Uint8Array): Promise; } "#; diff --git a/wnfs-wasm/tests/mock.ts b/wnfs-wasm/tests/mock.ts index 4a7423c5..695a79fd 100644 --- a/wnfs-wasm/tests/mock.ts +++ b/wnfs-wasm/tests/mock.ts @@ -35,11 +35,9 @@ class MemoryBlockStore { } /** Retrieves an array of bytes from the block store with given CID. */ - async putBlock(bytes: Uint8Array, codec: number): Promise { - const hash = await sha256.digest(bytes); - const cid = CID.create(1, codec, hash); - this.store.set(cid.toString(), bytes); - return cid.bytes; + async putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise { + const decodedCid = CID.decode(cid); + this.store.set(decodedCid.toString(), bytes); } /** Finds out whether a block is retrievable from this blockstore */ diff --git a/wnfs-wasm/tests/public.spec.ts b/wnfs-wasm/tests/public.spec.ts index 3cc34a18..02e4df67 100644 --- a/wnfs-wasm/tests/public.spec.ts +++ b/wnfs-wasm/tests/public.spec.ts @@ -368,6 +368,6 @@ test.describe("PublicDirectory", () => { }); expect(result).not.toBeUndefined(); - expect(result).toEqual("bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq"); + expect(result).toEqual("bafkr4ihkr4ld3m4gqkjf4reryxsy2s5tkbxprqkow6fin2iiyvreuzzab4"); }); }); diff --git a/wnfs/examples/tiered_blockstores.rs b/wnfs/examples/tiered_blockstores.rs index 75d067db..078f42a3 100644 --- a/wnfs/examples/tiered_blockstores.rs +++ b/wnfs/examples/tiered_blockstores.rs @@ -97,16 +97,19 @@ struct TieredBlockStore { impl BlockStore for TieredBlockStore { async fn get_block(&self, cid: &Cid) -> Result { - match self.hot.get_block(cid).await { - Ok(block) => Ok(block), - // We could technically get better about this - // and only match "NotFound" errors. - Err(_) => self.cold.get_block(cid).await, + if self.hot.has_block(cid).await? { + self.hot.get_block(cid).await + } else { + self.cold.get_block(cid).await } } async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { - self.hot.put_block(bytes.into(), codec).await + self.hot.put_block(bytes, codec).await + } + + async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + self.hot.put_block_keyed(cid, bytes).await } async fn has_block(&self, cid: &Cid) -> Result { From adc78239d7c9351f21ea33d8a6a14a77e5d8213d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 12:10:04 +0100 Subject: [PATCH 16/20] refactor: Move blockstore interface close to extern --- wnfs-wasm/src/fs/blockstore.rs | 9 +++++++++ wnfs-wasm/src/fs/mod.rs | 1 - wnfs-wasm/src/fs/types.rs | 10 ---------- 3 files changed, 9 insertions(+), 11 deletions(-) delete mode 100644 wnfs-wasm/src/fs/types.rs diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index 681dc7e5..bde0fe96 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -13,6 +13,15 @@ use wnfs::common::BlockStore as WnfsBlockStore; // Externs //-------------------------------------------------------------------------------------------------- +#[wasm_bindgen(typescript_custom_section)] +const TS_BLOCKSTORE: &'static str = r#" +export interface BlockStore { + putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise; + getBlock(cid: Uint8Array): Promise; + hasBlock(cid: Uint8Array): Promise; +} +"#; + #[wasm_bindgen] extern "C" { #[wasm_bindgen(typescript_type = "BlockStore")] diff --git a/wnfs-wasm/src/fs/mod.rs b/wnfs-wasm/src/fs/mod.rs index 6199a17e..74501ab2 100644 --- a/wnfs-wasm/src/fs/mod.rs +++ b/wnfs-wasm/src/fs/mod.rs @@ -2,7 +2,6 @@ mod blockstore; mod metadata; mod private; mod public; -mod types; mod utils; pub use blockstore::*; diff --git a/wnfs-wasm/src/fs/types.rs b/wnfs-wasm/src/fs/types.rs deleted file mode 100644 index 002d4344..00000000 --- a/wnfs-wasm/src/fs/types.rs +++ /dev/null @@ -1,10 +0,0 @@ -use wasm_bindgen::prelude::wasm_bindgen; - -#[wasm_bindgen(typescript_custom_section)] -const TS_BLOCKSTORE: &'static str = r#" -export interface BlockStore { - putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise; - getBlock(cid: Uint8Array): Promise; - hasBlock(cid: Uint8Array): Promise; -} -"#; From df5b83a30893b91c8670dea5b213dea2658459fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 18:00:06 +0100 Subject: [PATCH 17/20] refactor: Use precise error type in `trait BlockStore` --- wnfs-common/Cargo.toml | 1 + wnfs-common/src/blockstore.rs | 71 +++++++++++++++++++---------- wnfs-common/src/error.rs | 8 ++-- wnfs-common/src/storable.rs | 2 +- wnfs-common/src/utils/test.rs | 18 ++++++-- wnfs-unixfs-file/src/types.rs | 4 +- wnfs-wasm/src/fs/blockstore.rs | 12 +++-- wnfs/examples/tiered_blockstores.rs | 18 ++++++-- wnfs/src/private/directory.rs | 2 +- wnfs/src/private/file.rs | 2 +- wnfs/src/private/node/header.rs | 2 +- wnfs/src/root_tree.rs | 6 ++- 12 files changed, 98 insertions(+), 48 deletions(-) diff --git a/wnfs-common/Cargo.toml b/wnfs-common/Cargo.toml index 88755ea7..d4e8904d 100644 --- a/wnfs-common/Cargo.toml +++ b/wnfs-common/Cargo.toml @@ -24,6 +24,7 @@ base64 = { version = "0.21", optional = true } base64-serde = { version = "0.7", optional = true } bytes = { version = "1.4", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +cid = "0.10" dashmap = "5.5.3" futures = "0.3" libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] } diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 64adfbf9..f5dd08e1 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -3,7 +3,6 @@ use crate::{ utils::{Arc, CondSend, CondSync}, BlockStoreError, MAX_BLOCK_SIZE, }; -use anyhow::{bail, Result}; use bytes::Bytes; use futures::Future; use libipld::{ @@ -53,7 +52,10 @@ pub trait BlockStore: CondSync { /// Retrieve a block from this store via its hash (`Cid`). /// /// If this store can't find the block, it may raise an error like `BlockNotFound`. - fn get_block(&self, cid: &Cid) -> impl Future> + CondSend; + fn get_block( + &self, + cid: &Cid, + ) -> impl Future> + CondSend; /// Put some bytes into the blockstore. These bytes should be encoded with the given codec. /// @@ -72,7 +74,7 @@ pub trait BlockStore: CondSync { &self, bytes: impl Into + CondSend, codec: u64, - ) -> impl Future> + CondSend { + ) -> impl Future> + CondSend { let bytes = bytes.into(); async move { let cid = self.create_cid(&bytes, codec)?; @@ -94,19 +96,22 @@ pub trait BlockStore: CondSync { &self, cid: Cid, bytes: impl Into + CondSend, - ) -> impl Future> + CondSend; + ) -> impl Future> + CondSend; /// Find out whether a call to `get_block` would return with a result or not. /// /// This is useful for data exchange protocols to find out what needs to be fetched /// externally and what doesn't. - fn has_block(&self, cid: &Cid) -> impl Future> + CondSend; + fn has_block( + &self, + cid: &Cid, + ) -> impl Future> + CondSend; // This should be the same in all implementations of BlockStore - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { // If there are too many bytes, abandon this task if bytes.len() > MAX_BLOCK_SIZE { - bail!(BlockStoreError::MaximumBlockSizeExceeded(bytes.len())) + return Err(BlockStoreError::MaximumBlockSizeExceeded(bytes.len())); } // Compute the Blake3 hash of the bytes @@ -124,45 +129,61 @@ pub trait BlockStore: CondSync { //-------------------------------------------------------------------------------------------------- impl BlockStore for &B { - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { (**self).get_block(cid).await } - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { (**self).put_block(bytes, codec).await } - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { (**self).put_block_keyed(cid, bytes).await } - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { (**self).has_block(cid).await } - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } } impl BlockStore for Box { - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { (**self).get_block(cid).await } - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { (**self).put_block(bytes, codec).await } - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { (**self).put_block_keyed(cid, bytes).await } - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { (**self).has_block(cid).await } - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { (**self).create_cid(bytes, codec) } } @@ -186,7 +207,7 @@ impl MemoryBlockStore { } impl BlockStore for MemoryBlockStore { - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { let bytes = self .0 .lock() @@ -197,13 +218,17 @@ impl BlockStore for MemoryBlockStore { Ok(bytes) } - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { self.0.lock().insert(cid, bytes.into()); Ok(()) } - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { Ok(self.0.lock().contains_key(cid)) } } @@ -213,7 +238,7 @@ impl BlockStore for MemoryBlockStore { //-------------------------------------------------------------------------------------------------- /// Tests the retrieval property of a BlockStore-conforming type. -pub async fn bs_retrieval_test(store: impl BlockStore) -> Result<()> { +pub async fn bs_retrieval_test(store: impl BlockStore) -> Result<(), BlockStoreError> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = b"hello world".to_vec(); @@ -234,7 +259,7 @@ pub async fn bs_retrieval_test(store: impl BlockStore) -> Result<()> { } /// Tests the duplication of a BlockStore-conforming type. -pub async fn bs_duplication_test(store: impl BlockStore) -> Result<()> { +pub async fn bs_duplication_test(store: impl BlockStore) -> Result<(), BlockStoreError> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = first_bytes.clone(); @@ -261,7 +286,7 @@ pub async fn bs_duplication_test(store: impl BlockStore) -> Result<()> { } /// Tests the serialization of a BlockStore-conforming type. -pub async fn bs_serialization_test(store: &T) -> Result<()> +pub async fn bs_serialization_test(store: &T) -> Result<(), BlockStoreError> where T: BlockStore + Serialize + for<'de> Deserialize<'de>, { diff --git a/wnfs-common/src/error.rs b/wnfs-common/src/error.rs index c75cf329..6db9b2e3 100644 --- a/wnfs-common/src/error.rs +++ b/wnfs-common/src/error.rs @@ -16,9 +16,9 @@ pub enum BlockStoreError { #[error("Cannot find specified CID in block store: {0}")] CIDNotFound(Cid), - #[error("Cannot find handler for block with CID: {0}")] - BlockHandlerNotFound(Cid), + #[error("CID error during blockstore operation: {0}")] + CIDError(#[from] cid::Error), - #[error("Lock poisoned")] - LockPoisoned, + #[error(transparent)] + Custom(#[from] anyhow::Error), } diff --git a/wnfs-common/src/storable.rs b/wnfs-common/src/storable.rs index 7ecb4141..53673936 100644 --- a/wnfs-common/src/storable.rs +++ b/wnfs-common/src/storable.rs @@ -92,7 +92,7 @@ pub trait Storable: Sized { { let store_future = async { let (bytes, codec) = self.to_serializable(store).await?.encode_ipld()?; - store.put_block(bytes, codec).await + Ok(store.put_block(bytes, codec).await?) }; async { diff --git a/wnfs-common/src/utils/test.rs b/wnfs-common/src/utils/test.rs index c173da73..3bbc44e3 100644 --- a/wnfs-common/src/utils/test.rs +++ b/wnfs-common/src/utils/test.rs @@ -1,5 +1,5 @@ use super::{Arc, CondSend, CondSync}; -use crate::{BlockStore, MemoryBlockStore, CODEC_DAG_CBOR, CODEC_RAW}; +use crate::{BlockStore, BlockStoreError, MemoryBlockStore, CODEC_DAG_CBOR, CODEC_RAW}; use anyhow::Result; use base64_serde::base64_serde_type; use bytes::Bytes; @@ -113,22 +113,30 @@ impl SnapshotBlockStore { impl BlockStore for SnapshotBlockStore { #[inline] - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { self.inner.get_block(cid).await } #[inline] - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { self.inner.put_block(bytes, codec).await } #[inline] - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { self.inner.put_block_keyed(cid, bytes).await } #[inline] - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { self.inner.has_block(cid).await } } diff --git a/wnfs-unixfs-file/src/types.rs b/wnfs-unixfs-file/src/types.rs index 79c89700..a669a0c5 100644 --- a/wnfs-unixfs-file/src/types.rs +++ b/wnfs-unixfs-file/src/types.rs @@ -38,7 +38,9 @@ impl Block { } pub async fn store(&self, store: &impl BlockStore) -> Result { - store.put_block(self.data.clone(), self.codec.into()).await + Ok(store + .put_block(self.data.clone(), self.codec.into()) + .await?) } /// Validate the block. Will return an error if the links are wrong. diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index bde0fe96..2e930a23 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -7,7 +7,7 @@ use js_sys::{Promise, Uint8Array}; use libipld_core::cid::Cid; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen_futures::JsFuture; -use wnfs::common::BlockStore as WnfsBlockStore; +use wnfs::common::{BlockStore as WnfsBlockStore, BlockStoreError}; //-------------------------------------------------------------------------------------------------- // Externs @@ -50,7 +50,11 @@ pub struct ForeignBlockStore(pub(crate) BlockStore); //-------------------------------------------------------------------------------------------------- impl WnfsBlockStore for ForeignBlockStore { - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into, + ) -> Result<(), BlockStoreError> { let bytes: Bytes = bytes.into(); JsFuture::from(self.0.put_block_keyed(cid.to_bytes(), bytes.into())) @@ -60,7 +64,7 @@ impl WnfsBlockStore for ForeignBlockStore { Ok(()) } - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { let value = JsFuture::from(self.0.get_block(cid.to_bytes())) .await .map_err(anyhow_error("Cannot get block: {:?}"))?; @@ -70,7 +74,7 @@ impl WnfsBlockStore for ForeignBlockStore { Ok(Bytes::from(bytes)) } - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { let value = JsFuture::from(self.0.has_block(cid.to_bytes())) .await .map_err(anyhow_error("Cannot run has_block: {:?}"))?; diff --git a/wnfs/examples/tiered_blockstores.rs b/wnfs/examples/tiered_blockstores.rs index 078f42a3..4cfab8d4 100644 --- a/wnfs/examples/tiered_blockstores.rs +++ b/wnfs/examples/tiered_blockstores.rs @@ -16,7 +16,7 @@ use wnfs::{ PrivateDirectory, PrivateNode, }, }; -use wnfs_common::{utils::CondSend, Storable}; +use wnfs_common::{utils::CondSend, BlockStoreError, Storable}; #[async_std::main] async fn main() -> Result<()> { @@ -96,7 +96,7 @@ struct TieredBlockStore { } impl BlockStore for TieredBlockStore { - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { if self.hot.has_block(cid).await? { self.hot.get_block(cid).await } else { @@ -104,15 +104,23 @@ impl BlockStore for TieredBlockStore { } } - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { self.hot.put_block(bytes, codec).await } - async fn put_block_keyed(&self, cid: Cid, bytes: impl Into + CondSend) -> Result<()> { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { self.hot.put_block_keyed(cid, bytes).await } - async fn has_block(&self, cid: &Cid) -> Result { + async fn has_block(&self, cid: &Cid) -> Result { if self.hot.has_block(cid).await? { return Ok(true); } diff --git a/wnfs/src/private/directory.rs b/wnfs/src/private/directory.rs index 88c62d0c..f15a6b5a 100644 --- a/wnfs/src/private/directory.rs +++ b/wnfs/src/private/directory.rs @@ -1374,7 +1374,7 @@ impl PrivateDirectoryContent { let block = snapshot_key.encrypt(&bytes, rng)?; // Store content section in blockstore and get Cid. - store.put_block(block, CODEC_RAW).await + Ok(store.put_block(block, CODEC_RAW).await?) }) .await?) } diff --git a/wnfs/src/private/file.rs b/wnfs/src/private/file.rs index dfae4cb0..6920e5c8 100644 --- a/wnfs/src/private/file.rs +++ b/wnfs/src/private/file.rs @@ -826,7 +826,7 @@ impl PrivateFileContent { let block = snapshot_key.encrypt(&bytes, rng)?; // Store content section in blockstore and get Cid. - store.put_block(block, CODEC_RAW).await + Ok(store.put_block(block, CODEC_RAW).await?) }) .await?) } diff --git a/wnfs/src/private/node/header.rs b/wnfs/src/private/node/header.rs index f9dafe7a..fa850b02 100644 --- a/wnfs/src/private/node/header.rs +++ b/wnfs/src/private/node/header.rs @@ -165,7 +165,7 @@ impl PrivateNodeHeader { let temporal_key = self.derive_temporal_key(); let cbor_bytes = serde_ipld_dagcbor::to_vec(&self.to_serializable(forest))?; let ciphertext = temporal_key.key_wrap_encrypt(&cbor_bytes)?; - store.put_block(ciphertext, CODEC_RAW).await + Ok(store.put_block(ciphertext, CODEC_RAW).await?) } pub(crate) fn to_serializable( diff --git a/wnfs/src/root_tree.rs b/wnfs/src/root_tree.rs index 71972c88..4d959a6f 100644 --- a/wnfs/src/root_tree.rs +++ b/wnfs/src/root_tree.rs @@ -311,9 +311,11 @@ where version: WNFS_VERSION, }; - store + let cid = store .put_block(encode(&serializable, DagCborCodec)?, DagCborCodec.into()) - .await + .await?; + + Ok(cid) } pub async fn load( From 3e2ae56bb1f54cba75968ff189ef81c3442640ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 18:07:59 +0100 Subject: [PATCH 18/20] feat: Return correct error in `ForeignBlockStore::get_block` --- wnfs-wasm/src/fs/blockstore.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index 2e930a23..e68d2de1 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -69,6 +69,10 @@ impl WnfsBlockStore for ForeignBlockStore { .await .map_err(anyhow_error("Cannot get block: {:?}"))?; + if value.is_undefined() { + return Err(BlockStoreError::CIDNotFound(*cid)); + } + // Convert the value to a vector of bytes. let bytes = Uint8Array::new(&value).to_vec(); Ok(Bytes::from(bytes)) From 500a6c279663a70fd802c6deb2bfef96f15be435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 18:32:28 +0100 Subject: [PATCH 19/20] refactor: Use `libipld_core::serde::to_ipld` instead of dag-cbor --- wnfs/src/private/keys/access.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/wnfs/src/private/keys/access.rs b/wnfs/src/private/keys/access.rs index 947b03b4..120490c6 100644 --- a/wnfs/src/private/keys/access.rs +++ b/wnfs/src/private/keys/access.rs @@ -124,15 +124,11 @@ impl From<&AccessKey> for Vec { #[cfg(test)] mod snapshot_tests { use super::*; - use libipld_core::ipld::Ipld; use rand::Rng; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; use testresult::TestResult; - use wnfs_common::{ - decode, encode, - libipld::{cbor::DagCborCodec, json::DagJsonCodec}, - }; + use wnfs_common::{encode, libipld::json::DagJsonCodec}; #[async_std::test] async fn test_access_key() -> TestResult { @@ -154,9 +150,7 @@ mod snapshot_tests { } fn as_dag_json_value(s: impl Serialize) -> Result { - let dag_cbor = encode(&s, DagCborCodec)?; - let ipld: Ipld = decode(&dag_cbor, DagCborCodec)?; - let dag_json = encode(&ipld, DagJsonCodec)?; + let dag_json = encode(&libipld_core::serde::to_ipld(s)?, DagJsonCodec)?; let value = serde_json::from_slice(&dag_json)?; Ok(value) } From 3d5793e2278686b2765b03ba60d1b504e7306e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Kr=C3=BCger?= Date: Thu, 15 Feb 2024 18:36:44 +0100 Subject: [PATCH 20/20] docs: Add comments explaining use of `boxed_fut` --- wnfs-hamt/src/node.rs | 1 + wnfs/src/public/directory.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/wnfs-hamt/src/node.rs b/wnfs-hamt/src/node.rs index 43baa161..453e828e 100644 --- a/wnfs-hamt/src/node.rs +++ b/wnfs-hamt/src/node.rs @@ -852,6 +852,7 @@ where let mut pointers = Vec::with_capacity(self.pointers.len()); for pointer in self.pointers.iter() { + // Boxing the future due to recursion pointers.push(boxed_fut(pointer.to_serializable(store)).await?); } diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 02cd6d80..927d6a02 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -833,6 +833,7 @@ impl Storable for PublicDirectory { let userland = { let mut map = BTreeMap::new(); for (name, link) in self.userland.iter() { + // Boxing the future due to recursion map.insert(name.clone(), boxed_fut(link.resolve_cid(store)).await?); } map