From d28a1431030b5980c73cdf1b96838fb672b81800 Mon Sep 17 00:00:00 2001 From: Hannes Karppila <2204863+Dentosal@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:20:06 +0300 Subject: [PATCH] DA compression for fuel-tx types (#670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP * Migrate compact-derive from fuel-core * Add doc comments and cleanup * Add changelog * Cargo.toml fmt * Wrap Compaction return types to anyhow::Result * Update readme * Fix wording Co-authored-by: Brandon Vrooman * Combine some bounds Co-authored-by: Brandon Vrooman * Fix issues after merge * Fix compression for latest types * Keep used gas price * Cargo sort * Introduce ContractId table to da compression registry * Reference table types directly in #[da_compress(registry = _)] * WIP: migrate domain logic to fuel-core * Re-add dummy registry for testing * Add some missing docs * Move block section data for fuel-core * Minor change for fuel-core integration * cargo sort * Fix incorrect feature cfg on default_test_tx * Implement compacting for blobs as well * Fix no_std deps * WIP: working towards adapting Green's trait-based approach * Complete migration to Green's trait-heavy approach * Polish: docs, naming, argument order * Clean up proc macro a bit * Line count reduction :\ * More polish * Move readme from fuel-compression to fuel-core-compression * Enable more lints for fuel-derive, remove unnecessary regex dependency * Remove debug writing to /tmp, as we're hopefully done here * Get rid of anyhow * Add roundtrip tests * Use type-based registry keyspaces * Omit Message amount from DA compressed data * Remove extra "Compressible" bound from "RegistrySubstitutableBy" * Implement da compression for TxId -> TxPointer * Use the type instead of field annotations for determining compressed fields * Make da compression async * Also make compression context async * unused_crate_dependencies fix * More unused_crate_dependencies * cargo sort * Cosmetic changes: - Renamed derive macro `Compressed` -> `CompressibleBy` - Renamed `da_compress` -> `compressible_by` - Removed default constrains from `Compressible::Compressed` - Fixed compilation after removing constrains - Removed `da_compress(bound)` - Added new `TxId` type * Cleanup small nits after review * Revert commits af39dcfcf19d6e41cca6fb39a6707b8614854338 and bf75dbc1354840fb7787d1282821b241d4bb8c89 * Split derive macro "Compressed" into "Compress" and "Decompress" * Re-apply Green's da_compress(bound) removal and bound simplification * Re-introduce Green's TxId type, do some cleanup related to these types * Re-apply bf75dbc1354840fb7787d1282821b241d4bb8c89 * Compress the whole UtxoId instead of just TxId * Compress UtxoId: test fixes * Introduce PredicateCode to allow compressing it * Make fields for Compressed* public * Remove Decompress derive from Message and Coin Also, rename {C,De}omprssibleBy methods to {de}compress_with * Rename da_compress attribute to just compress * Add tests for TxId * Rename remaining instances of compaction into compression * Add explicit CompressedUtxoId type * Rename De/CompressionContext methods Fix tests * Clippy * Ignore incorrect unused crate warning * Fix no-default-features * Remove De/CompressionContext traits, they seem unnecessary * Simplifying tests for DA compression (#816) * Address PR review comments * Fix , => ; * Fix a memory leak in unsafe array de/compress code * Fix typo Co-authored-by: Rafał Chabowski <88321181+rafal-ch@users.noreply.github.com> * Fix typo Co-authored-by: Rafał Chabowski <88321181+rafal-ch@users.noreply.github.com> * Add a test cases to cover skipped fields and nested structs * Remove unneeded change of field ref for struct that impls Deref * Remove useless .scollect()s * Clarify variable namings around serialization in a test case * Add a test case showing that skipped fields are not part of the compressed output --------- Co-authored-by: Brandon Vrooman Co-authored-by: green Co-authored-by: Rafał Chabowski <88321181+rafal-ch@users.noreply.github.com> --- CHANGELOG.md | 10 + Cargo.toml | 2 + README.md | 1 + fuel-asm/Cargo.toml | 2 +- fuel-compression/Cargo.toml | 16 + fuel-compression/src/impls.rs | 194 +++++++ fuel-compression/src/key.rs | 69 +++ fuel-compression/src/lib.rs | 19 + fuel-compression/src/traits.rs | 60 ++ fuel-crypto/src/secp256/signature.rs | 2 +- fuel-derive/Cargo.toml | 2 +- fuel-derive/README.md | 2 +- fuel-derive/src/{ => canonical}/attribute.rs | 0 .../src/{ => canonical}/deserialize.rs | 4 +- fuel-derive/src/{ => canonical}/serialize.rs | 4 +- fuel-derive/src/compression/attribute.rs | 100 ++++ fuel-derive/src/compression/compress.rs | 214 ++++++++ fuel-derive/src/compression/decompress.rs | 170 ++++++ fuel-derive/src/helpers.rs | 14 + fuel-derive/src/lib.rs | 43 +- fuel-tx/Cargo.toml | 6 + fuel-tx/src/builder.rs | 2 +- fuel-tx/src/lib.rs | 8 + fuel-tx/src/tests/da_compression.rs | 515 ++++++++++++++++++ fuel-tx/src/tests/mod.rs | 9 + fuel-tx/src/transaction.rs | 10 +- fuel-tx/src/transaction/policies.rs | 18 +- fuel-tx/src/transaction/types.rs | 7 + fuel-tx/src/transaction/types/blob.rs | 4 + .../types/chargeable_transaction.rs | 62 ++- fuel-tx/src/transaction/types/create.rs | 4 + fuel-tx/src/transaction/types/input.rs | 65 ++- fuel-tx/src/transaction/types/input/coin.rs | 64 ++- .../src/transaction/types/input/contract.rs | 9 + .../src/transaction/types/input/message.rs | 83 ++- .../src/transaction/types/input/predicate.rs | 40 ++ .../transaction/types/input/snapshot_tests.rs | 6 +- fuel-tx/src/transaction/types/mint.rs | 14 + fuel-tx/src/transaction/types/output.rs | 24 +- .../src/transaction/types/output/contract.rs | 6 + fuel-tx/src/transaction/types/script.rs | 47 +- fuel-tx/src/transaction/types/storage.rs | 4 + fuel-tx/src/transaction/types/upgrade.rs | 8 + fuel-tx/src/transaction/types/upload.rs | 4 + fuel-tx/src/transaction/types/utxo_id.rs | 24 + fuel-tx/src/transaction/types/witness.rs | 4 + fuel-tx/src/transaction/validity.rs | 2 +- fuel-tx/src/tx_pointer.rs | 4 + fuel-types/src/array_types.rs | 3 +- fuel-types/src/tests/types.rs | 13 + fuel-vm/Cargo.toml | 2 + fuel-vm/src/interpreter/executors/main.rs | 2 +- fuel-vm/src/lib.rs | 3 + 53 files changed, 1923 insertions(+), 81 deletions(-) create mode 100644 fuel-compression/Cargo.toml create mode 100644 fuel-compression/src/impls.rs create mode 100644 fuel-compression/src/key.rs create mode 100644 fuel-compression/src/lib.rs create mode 100644 fuel-compression/src/traits.rs rename fuel-derive/src/{ => canonical}/attribute.rs (100%) rename fuel-derive/src/{ => canonical}/deserialize.rs (98%) rename fuel-derive/src/{ => canonical}/serialize.rs (98%) create mode 100644 fuel-derive/src/compression/attribute.rs create mode 100644 fuel-derive/src/compression/compress.rs create mode 100644 fuel-derive/src/compression/decompress.rs create mode 100644 fuel-derive/src/helpers.rs create mode 100644 fuel-tx/src/tests/da_compression.rs create mode 100644 fuel-tx/src/transaction/types/input/predicate.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c9d6881dbf..a5fa9db3c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added + +- [#670](https://github.com/FuelLabs/fuel-vm/pull/670): Add DA compression functionality to `Transaction` and any types within + +### Changed + +#### Breaking + +- [#670](https://github.com/FuelLabs/fuel-vm/pull/670): The `predicate` field of `fuel_tx::input::Coin` is now a wrapper struct `PredicateCode`. + ## [Version 0.56.0] ### Added diff --git a/Cargo.toml b/Cargo.toml index 0d41d91da7..aa6a4fb690 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "fuel-asm", + "fuel-compression", "fuel-crypto", "fuel-merkle", "fuel-storage", @@ -22,6 +23,7 @@ version = "0.56.0" [workspace.dependencies] fuel-asm = { version = "0.56.0", path = "fuel-asm", default-features = false } fuel-crypto = { version = "0.56.0", path = "fuel-crypto", default-features = false } +fuel-compression = { version = "0.56.0", path = "fuel-compression", default-features = false } fuel-derive = { version = "0.56.0", path = "fuel-derive", default-features = false } fuel-merkle = { version = "0.56.0", path = "fuel-merkle", default-features = false } fuel-storage = { version = "0.56.0", path = "fuel-storage", default-features = false } diff --git a/README.md b/README.md index 56869fd259..9c77b1bf19 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ and [the Sway compiler](https://github.com/FuelLabs/sway/). | Crate | Version | Description | |--------------|-----------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | fuel-asm | [![crates.io](https://img.shields.io/crates/v/fuel-asm)](https://crates.io/crates/fuel-asm) | Contains the FuelVM instruction set - opcodes used by the Sway and VM. | +| fuel-compression | [![crates.io](https://img.shields.io/crates/v/fuel-compression)](https://crates.io/crates/fuel-compression) | DA-layer compression of Fuel transaction types | | fuel-crypto | [![crates.io](https://img.shields.io/crates/v/fuel-crypto)](https://crates.io/crates/fuel-crypto) | Cryptographic primitives used across Fuel Rust based projects. | | fuel-merkle | [![crates.io](https://img.shields.io/crates/v/fuel-merkle)](https://crates.io/crates/fuel-merkle) | Implementations of the Merkle Tree used by the `fuel-core` to fulfill fraud proofs requirements, and `fuel-tx` to validate transaction validity. | | fuel-storage | [![crates.io](https://img.shields.io/crates/v/fuel-storage)](https://crates.io/crates/fuel-storage) | Storage abstraction is used to connect FuelVM, `fuel-merkle`, and `fuel-core` together without direct access. | diff --git a/fuel-asm/Cargo.toml b/fuel-asm/Cargo.toml index 5d5b7bd201..d1fcc9fb72 100644 --- a/fuel-asm/Cargo.toml +++ b/fuel-asm/Cargo.toml @@ -13,7 +13,7 @@ description = "Atomic types of the FuelVM." [dependencies] arbitrary = { version = "1.1", features = ["derive"], optional = true } bitflags = { workspace = true } -fuel-types = { workspace = true } +fuel-types = { workspace = true, default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } strum = { version = "0.24", default-features = false, features = ["derive"] } wasm-bindgen = { version = "0.2.88", optional = true } diff --git a/fuel-compression/Cargo.toml b/fuel-compression/Cargo.toml new file mode 100644 index 0000000000..e6d230ad9e --- /dev/null +++ b/fuel-compression/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fuel-compression" +version = { workspace = true } +authors = { workspace = true } +categories = ["cryptography::cryptocurrencies"] +edition = { workspace = true } +homepage = { workspace = true } +keywords = ["blockchain", "cryptocurrencies", "fuel-compression"] +license = { workspace = true } +repository = { workspace = true } +description = "Compression and decompression of Fuel blocks for DA storage." + +[dependencies] +fuel-derive = { workspace = true } +fuel-types = { workspace = true } +serde = { version = "1.0", features = ["derive"] } diff --git a/fuel-compression/src/impls.rs b/fuel-compression/src/impls.rs new file mode 100644 index 0000000000..53a70cd595 --- /dev/null +++ b/fuel-compression/src/impls.rs @@ -0,0 +1,194 @@ +//! Trait impls for Rust types + +use super::traits::*; +use crate::RegistryKey; +use core::mem::MaybeUninit; +use fuel_types::{ + Address, + AssetId, + BlobId, + BlockHeight, + Bytes32, + ContractId, + Nonce, + Salt, +}; + +macro_rules! identity_compression { + ($t:ty) => { + impl Compressible for $t { + type Compressed = Self; + } + + impl CompressibleBy for $t + where + Ctx: ContextError, + { + async fn compress_with(&self, _: &mut Ctx) -> Result { + Ok(*self) + } + } + + impl DecompressibleBy for $t + where + Ctx: ContextError, + { + async fn decompress_with( + c: &Self::Compressed, + _: &Ctx, + ) -> Result { + Ok(*c) + } + } + }; +} + +identity_compression!(u8); +identity_compression!(u16); +identity_compression!(u32); +identity_compression!(u64); +identity_compression!(u128); + +identity_compression!(BlockHeight); +identity_compression!(BlobId); +identity_compression!(Bytes32); +identity_compression!(Salt); +identity_compression!(Nonce); + +impl Compressible for Address { + type Compressed = RegistryKey; +} + +impl Compressible for ContractId { + type Compressed = RegistryKey; +} + +impl Compressible for AssetId { + type Compressed = RegistryKey; +} + +impl Compressible for [T; S] +where + T: Compressible, +{ + type Compressed = [T::Compressed; S]; +} + +impl CompressibleBy for [T; S] +where + T: CompressibleBy, + Ctx: ContextError, +{ + #[allow(unsafe_code)] + async fn compress_with(&self, ctx: &mut Ctx) -> Result { + // SAFETY: we are claiming to have initialized an array of `MaybeUninit`s, + // which do not require initialization. + let mut tmp: [MaybeUninit; S] = + unsafe { MaybeUninit::uninit().assume_init() }; + + let mut i = 0; + while i < self.len() { + match self[i].compress_with(ctx).await { + Ok(value) => { + // SAFETY: MaybeUninit can be safely overwritten. + tmp[i].write(value); + } + Err(e) => { + // Drop the already initialized elements, so we don't leak the memory + for initialized_item in tmp.iter_mut().take(i) { + // Safety: First i elements have been initialized successfully. + unsafe { + initialized_item.assume_init_drop(); + } + } + return Err(e); + } + } + i += 1; + } + + // SAFETY: Every element is initialized. In case of error, we have returned + // instead. + let result = tmp.map(|v| unsafe { v.assume_init() }); + Ok(result) + } +} + +impl DecompressibleBy for [T; S] +where + T: DecompressibleBy, + Ctx: ContextError, +{ + #[allow(unsafe_code)] + async fn decompress_with( + c: &Self::Compressed, + ctx: &Ctx, + ) -> Result { + // SAFETY: we are claiming to have initialized an array of `MaybeUninit`s, + // which do not require initialization. + let mut tmp: [MaybeUninit; S] = unsafe { MaybeUninit::uninit().assume_init() }; + + let mut i = 0; + while i < c.len() { + match T::decompress_with(&c[i], ctx).await { + Ok(value) => { + // SAFETY: MaybeUninit can be safely overwritten. + tmp[i].write(value); + } + Err(e) => { + // Drop the already initialized elements, so we don't leak the memory + for initialized_item in tmp.iter_mut().take(i) { + // Safety: First i elements have been initialized successfully. + unsafe { + initialized_item.assume_init_drop(); + } + } + return Err(e); + } + } + i += 1; + } + + // SAFETY: Every element is initialized. + let result = tmp.map(|v| unsafe { v.assume_init() }); + Ok(result) + } +} + +impl Compressible for Vec +where + T: Compressible, +{ + type Compressed = Vec; +} + +impl CompressibleBy for Vec +where + T: CompressibleBy, + Ctx: ContextError, +{ + async fn compress_with(&self, ctx: &mut Ctx) -> Result { + let mut result = Vec::with_capacity(self.len()); + for item in self { + result.push(item.compress_with(ctx).await?); + } + Ok(result) + } +} + +impl DecompressibleBy for Vec +where + T: DecompressibleBy, + Ctx: ContextError, +{ + async fn decompress_with( + c: &Self::Compressed, + ctx: &Ctx, + ) -> Result { + let mut result = Vec::with_capacity(c.len()); + for item in c { + result.push(T::decompress_with(item, ctx).await?); + } + Ok(result) + } +} diff --git a/fuel-compression/src/key.rs b/fuel-compression/src/key.rs new file mode 100644 index 0000000000..b2f614cbbb --- /dev/null +++ b/fuel-compression/src/key.rs @@ -0,0 +1,69 @@ +use serde::{ + Deserialize, + Serialize, +}; + +/// Untyped key pointing to a registry table entry. +/// The last key (all bits set) is reserved for the default value and cannot be written +/// to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct RegistryKey([u8; Self::SIZE]); +impl RegistryKey { + /// Key mapping to default value for the table type. + pub const DEFAULT_VALUE: Self = Self([u8::MAX; Self::SIZE]); + /// Maximum writable key. + pub const MAX_WRITABLE: Self = Self([u8::MAX, u8::MAX, u8::MAX - 1]); + /// Size of the key, in bytes. + pub const SIZE: usize = 3; + /// Zero key. + pub const ZERO: Self = Self([0; Self::SIZE]); + + /// Convert to u32, big-endian. + pub fn as_u32(self) -> u32 { + u32::from_be_bytes([0, self.0[0], self.0[1], self.0[2]]) + } + + /// Wraps around just below max/default value. + /// Panics for max/default value. + pub fn next(self) -> Self { + if self == Self::DEFAULT_VALUE { + panic!("Max/default value has no next key"); + } + let next_raw = self.as_u32() + 1u32; + if next_raw == Self::DEFAULT_VALUE.as_u32() { + Self::ZERO + } else { + Self::try_from(next_raw) + .expect("The procedure above always produces a valid key") + } + } +} +impl TryFrom for RegistryKey { + type Error = &'static str; + + fn try_from(value: u32) -> Result { + let v = value.to_be_bytes(); + if v[0] != 0 { + return Err("RegistryKey must be less than 2^24"); + } + + let mut bytes = [0u8; 3]; + bytes.copy_from_slice(&v[1..]); + Ok(Self(bytes)) + } +} + +#[cfg(test)] +mod tests { + use super::RegistryKey; + + #[test] + fn key_next() { + assert_eq!(RegistryKey::ZERO.next(), RegistryKey([0, 0, 1])); + assert_eq!(RegistryKey::ZERO.next().next(), RegistryKey([0, 0, 2])); + assert_eq!(RegistryKey([0, 0, 255]).next(), RegistryKey([0, 1, 0])); + assert_eq!(RegistryKey([0, 1, 255]).next(), RegistryKey([0, 2, 0])); + assert_eq!(RegistryKey([0, 255, 255]).next(), RegistryKey([1, 0, 0])); + assert_eq!(RegistryKey::MAX_WRITABLE.next(), RegistryKey::ZERO); + } +} diff --git a/fuel-compression/src/lib.rs b/fuel-compression/src/lib.rs new file mode 100644 index 0000000000..ded3849a4b --- /dev/null +++ b/fuel-compression/src/lib.rs @@ -0,0 +1,19 @@ +//! Compression and decompression of fuel-types for the DA layer + +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![warn(missing_docs)] +#![deny(unsafe_code)] +#![deny(unused_crate_dependencies)] +#![deny(clippy::cast_possible_truncation)] + +mod impls; +mod key; +mod traits; + +pub use key::RegistryKey; +pub use traits::*; + +pub use fuel_derive::{ + Compress, + Decompress, +}; diff --git a/fuel-compression/src/traits.rs b/fuel-compression/src/traits.rs new file mode 100644 index 0000000000..ca2635b9cd --- /dev/null +++ b/fuel-compression/src/traits.rs @@ -0,0 +1,60 @@ +#![allow(async_fn_in_trait)] // We control the implementation so this is fine + +/// Defines the error type for the context used in compression and decompression. +pub trait ContextError { + /// The error type returned by the context. + type Error; +} + +/// This type can be compressed to a more compact form and back using +/// `CompressibleBy` and `DecompressibleBy` traits. +pub trait Compressible { + /// The compressed type. + type Compressed: Sized; +} + +/// This type can be compressed to a more compact form and back using +/// `CompressionContext`. +pub trait CompressibleBy: Compressible +where + Ctx: ContextError, +{ + /// Perform compression, returning the compressed data and possibly modifying the + /// context. The context is mutable to allow for stateful compression. + /// For instance, it can be used to extract original data when replacing it with + /// references. + async fn compress_with(&self, ctx: &mut Ctx) -> Result; +} + +/// This type can be decompressed using `CompressionContext`. +pub trait DecompressibleBy: Compressible +where + Ctx: ContextError, + Self: Sized, +{ + /// Perform decompression, returning the original data. + /// The context can be used to resolve references. + async fn decompress_with(c: &Self::Compressed, ctx: &Ctx) + -> Result; +} + +/// The trait allows for decompression of a compressed type. +/// This trait is syntax sugar for `DecompressibleBy` with the compressed type as the +/// receiver. +pub trait Decompress +where + Ctx: ContextError, +{ + /// Perform decompression, returning the original data. + async fn decompress(&self, ctx: &Ctx) -> Result; +} + +impl Decompress for T +where + Ctx: ContextError, + Decompressed: DecompressibleBy, +{ + async fn decompress(&self, ctx: &Ctx) -> Result { + Decompressed::decompress_with(self, ctx).await + } +} diff --git a/fuel-crypto/src/secp256/signature.rs b/fuel-crypto/src/secp256/signature.rs index 132ad55a12..ff51e65052 100644 --- a/fuel-crypto/src/secp256/signature.rs +++ b/fuel-crypto/src/secp256/signature.rs @@ -14,7 +14,7 @@ use core::{ str, }; -/// Compact-form Secp256k1 signature. +/// Compressed-form Secp256k1 signature. #[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] diff --git a/fuel-derive/Cargo.toml b/fuel-derive/Cargo.toml index 6bddd0d857..abf2a00ea7 100644 --- a/fuel-derive/Cargo.toml +++ b/fuel-derive/Cargo.toml @@ -8,7 +8,7 @@ homepage = { workspace = true } keywords = ["blockchain", "cryptocurrencies", "fuel-vm", "vm"] license = { workspace = true } repository = { workspace = true } -description = "FuelVM (de)serialization derive macros for `fuel-vm` data structures." +description = "FuelVM (de)serialization and compression derive macros for `fuel-vm` data structures." [lib] proc-macro = true diff --git a/fuel-derive/README.md b/fuel-derive/README.md index d28edae921..7d98eba4cb 100644 --- a/fuel-derive/README.md +++ b/fuel-derive/README.md @@ -5,4 +5,4 @@ [![docs](https://docs.rs/fuel-derive/badge.svg)](https://docs.rs/fuel-derive/) [![discord](https://img.shields.io/badge/chat%20on-discord-orange?&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/xfpK4Pe) -This crate contains derive macros for canonical serialization and deserialization. This is used with [`fuel-types/src/canonical.rs`](fuel-types/src/canonical.rs) module which contains the associated traits and their implementations for native Rust types. +This crate contains derive macros for canonical serialization and deserialization. This is used with [`fuel-types/src/canonical.rs`](fuel-types/src/canonical.rs) module which contains the associated traits and their implementations for native Rust types. It also contains compression macros exported by `fuel-compression`. diff --git a/fuel-derive/src/attribute.rs b/fuel-derive/src/canonical/attribute.rs similarity index 100% rename from fuel-derive/src/attribute.rs rename to fuel-derive/src/canonical/attribute.rs diff --git a/fuel-derive/src/deserialize.rs b/fuel-derive/src/canonical/deserialize.rs similarity index 98% rename from fuel-derive/src/deserialize.rs rename to fuel-derive/src/canonical/deserialize.rs index 21ad09ba00..8fd1671290 100644 --- a/fuel-derive/src/deserialize.rs +++ b/fuel-derive/src/canonical/deserialize.rs @@ -4,7 +4,7 @@ use quote::{ quote, }; -use crate::attribute::{ +use super::attribute::{ should_skip_field, should_skip_field_binding, StructAttrs, @@ -174,7 +174,7 @@ fn deserialize_enum(s: &synstructure::Structure) -> TokenStream2 { } /// Derives `Deserialize` trait for the given `struct` or `enum`. -pub fn deserialize_derive(mut s: synstructure::Structure) -> TokenStream2 { +pub fn derive(mut s: synstructure::Structure) -> TokenStream2 { s.bind_with(|_| synstructure::BindStyle::RefMut) .add_bounds(synstructure::AddBounds::Fields) .underscore_const(true); diff --git a/fuel-derive/src/serialize.rs b/fuel-derive/src/canonical/serialize.rs similarity index 98% rename from fuel-derive/src/serialize.rs rename to fuel-derive/src/canonical/serialize.rs index 42b2eefdc1..c34234ee42 100644 --- a/fuel-derive/src/serialize.rs +++ b/fuel-derive/src/canonical/serialize.rs @@ -1,7 +1,7 @@ use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use crate::attribute::{ +use super::attribute::{ should_skip_field_binding, StructAttrs, }; @@ -211,7 +211,7 @@ fn serialize_enum(s: &synstructure::Structure) -> TokenStream2 { } /// Derives `Serialize` trait for the given `struct` or `enum`. -pub fn serialize_derive(mut s: synstructure::Structure) -> TokenStream2 { +pub fn derive(mut s: synstructure::Structure) -> TokenStream2 { s.add_bounds(synstructure::AddBounds::Fields) .underscore_const(true); diff --git a/fuel-derive/src/compression/attribute.rs b/fuel-derive/src/compression/attribute.rs new file mode 100644 index 0000000000..617f0c2105 --- /dev/null +++ b/fuel-derive/src/compression/attribute.rs @@ -0,0 +1,100 @@ +use proc_macro2::TokenTree as TokenTree2; + +use syn::parse::{ + Parse, + ParseStream, +}; + +const ATTR: &str = "compress"; + +/// Structure (struct or enum) attributes +#[derive(Debug)] +pub enum StructureAttrs { + /// Discard generic parameter + /// `#[compress(discard(Type))]` + Discard(Vec), +} +impl Parse for StructureAttrs { + fn parse(input: ParseStream) -> syn::Result { + if let Ok(ml) = input.parse::() { + if ml.path.segments.len() == 1 + && ml.path.segments[0].ident.to_string().as_str() == "discard" + { + let mut discard = Vec::new(); + for item in ml.tokens { + match item { + TokenTree2::Ident(ident) => { + discard.push(ident.to_string()); + } + other => { + return Err(syn::Error::new_spanned( + other, + "Expected generic (type) name", + )) + } + } + } + return Ok(Self::Discard(discard)); + } + } + Err(syn::Error::new_spanned( + input.parse::()?, + "Expected `discard`", + )) + } +} +impl StructureAttrs { + pub fn parse(attrs: &[syn::Attribute]) -> syn::Result> { + let mut result = Vec::new(); + for attr in attrs { + if attr.style != syn::AttrStyle::Outer { + continue; + } + + if let syn::Meta::List(ml) = &attr.meta { + if ml.path.segments.len() == 1 && ml.path.segments[0].ident == ATTR { + result.push(syn::parse2::(ml.tokens.clone())?); + } + } + } + + Ok(result) + } +} + +/// Field attributes +pub enum FieldAttrs { + /// Skipped when compressing, and must be reconstructed when decompressing. + /// `#[compress(skip)]` + Skip, + /// Compressed recursively. + Normal, +} +impl FieldAttrs { + pub fn parse(attrs: &[syn::Attribute]) -> Self { + let mut result = Self::Normal; + for attr in attrs { + if attr.style != syn::AttrStyle::Outer { + continue; + } + + if let syn::Meta::List(ml) = &attr.meta { + if ml.path.segments.len() == 1 && ml.path.segments[0].ident == ATTR { + if !matches!(result, Self::Normal) { + panic!("Duplicate attribute: {}", ml.tokens); + } + + if let Ok(ident) = syn::parse2::(ml.tokens.clone()) { + if ident == "skip" { + result = Self::Skip; + continue; + } + } + panic!("Invalid attribute: {}", ml.tokens); + } + } + } + + result + } +} diff --git a/fuel-derive/src/compression/compress.rs b/fuel-derive/src/compression/compress.rs new file mode 100644 index 0000000000..7bcc6cb6e0 --- /dev/null +++ b/fuel-derive/src/compression/compress.rs @@ -0,0 +1,214 @@ +use crate::helpers::where_clause_push; +use proc_macro2::TokenStream as TokenStream2; +use quote::{ + format_ident, + quote, +}; + +use super::attribute::{ + FieldAttrs, + StructureAttrs, +}; + +/// Iterator of field items for the compressed structure. +/// Gives either named or unnamed fields. +fn field_items(fields: &syn::Fields) -> impl Iterator + '_ { + fields.iter().filter_map(|field| { + let attrs = FieldAttrs::parse(&field.attrs); + match &attrs { + FieldAttrs::Skip => None, + FieldAttrs::Normal => { + let ty = &field.ty; + let ftype = quote! { + <#ty as ::fuel_compression::Compressible>::Compressed + }; + Some(if let Some(fname) = field.ident.as_ref() { + quote! { #fname: #ftype } + } else { + quote! { #ftype } + }) + } + } + }) +} + +/// Map field definitions to compressed field definitions. +fn field_defs(fields: &syn::Fields, include_vis: bool) -> TokenStream2 { + let mut defs = TokenStream2::new(); + for item in field_items(fields) { + if include_vis { + defs.extend(quote! { #item, }); + } else { + defs.extend(quote! { pub #item, }); + } + } + match fields { + syn::Fields::Named(_) => quote! {{ #defs }}, + syn::Fields::Unnamed(_) => quote! {(#defs)}, + syn::Fields::Unit => quote! {}, + } +} + +/// Construct compressed version of the struct from the original one +fn construct_compressed( + // The structure to construct, i.e. struct name or enum variant path + compressed: &TokenStream2, + variant: &synstructure::VariantInfo<'_>, +) -> TokenStream2 { + let bound_fields: TokenStream2 = variant + .bindings() + .iter() + .filter(|binding| !matches!(FieldAttrs::parse(&binding.ast().attrs), FieldAttrs::Skip)) + .map(|binding| { + let ty = &binding.ast().ty; + let cname = format_ident!("{}_c", binding.binding); + quote! { + let #cname = <#ty as ::fuel_compression::CompressibleBy<_>>::compress_with(&#binding, ctx).await?; + } + }) + .collect(); + + let construct_fields: TokenStream2 = variant + .bindings() + .iter() + .filter(|binding| { + !matches!(FieldAttrs::parse(&binding.ast().attrs), FieldAttrs::Skip) + }) + .map(|binding| { + let cname = format_ident!("{}_c", binding.binding); + if let Some(fname) = &binding.ast().ident { + quote! { #fname: #cname, } + } else { + quote! { #cname, } + } + }) + .collect(); + + let construct_fields = match variant.ast().fields { + syn::Fields::Named(_) => quote! {{ #construct_fields }}, + syn::Fields::Unnamed(_) => quote! {(#construct_fields)}, + syn::Fields::Unit => quote! {}, + }; + + quote! { + #bound_fields + #compressed #construct_fields + } +} + +/// Derives `Compressible` and `CompressibleBy` traits for the given `struct` or `enum`. +pub fn derive(mut s: synstructure::Structure) -> TokenStream2 { + s.add_bounds(synstructure::AddBounds::None) + .underscore_const(true); + + let s_attrs = match StructureAttrs::parse(&s.ast().attrs) { + Ok(v) => v, + Err(e) => return e.to_compile_error(), + }; + + let name = &s.ast().ident; + let compressed_name = format_ident!("Compressed{}", name); + + let mut g = s.ast().generics.clone(); + let w_structure = g.where_clause.take(); + let w_impl = w_structure.clone(); + for item in &s_attrs { + match item { + StructureAttrs::Discard(discard) => { + g.params = g + .params + .into_pairs() + .filter(|pair| match pair.value() { + syn::GenericParam::Type(t) => { + !discard.contains(&t.ident.to_string()) + } + _ => true, + }) + .collect(); + } + } + } + + let mut w_impl_field_bounds_compress = w_impl.clone(); + for variant in s.variants() { + for field in variant.ast().fields.iter() { + let ty = &field.ty; + match FieldAttrs::parse(&field.attrs) { + FieldAttrs::Skip => {} + FieldAttrs::Normal => { + where_clause_push( + &mut w_impl_field_bounds_compress, + syn::parse_quote! { #ty: ::fuel_compression::CompressibleBy }, + ); + } + } + } + } + where_clause_push( + &mut w_impl_field_bounds_compress, + syn::parse_quote! { Ctx: ::fuel_compression::ContextError }, + ); + + let def = match &s.ast().data { + syn::Data::Struct(v) => { + let variant: &synstructure::VariantInfo = &s.variants()[0]; + let defs = field_defs(variant.ast().fields, false); + let semi = match v.fields { + syn::Fields::Named(_) => quote! {}, + syn::Fields::Unnamed(_) => quote! {;}, + syn::Fields::Unit => quote! {;}, + }; + quote! { + #[derive(Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize)] + #[doc = concat!("Compressed version of `", stringify!(#name), "`.")] + pub struct #compressed_name #g #w_structure #defs #semi + } + } + syn::Data::Enum(_) => { + let variant_defs: TokenStream2 = s + .variants() + .iter() + .map(|variant| { + let vname = variant.ast().ident.clone(); + let defs = field_defs(variant.ast().fields, true); + quote! { + #vname #defs, + } + }) + .collect(); + + quote! { + #[derive(Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize)] + #[doc = concat!("Compressed version of `", stringify!(#name), "`.")] + pub enum #compressed_name #g #w_structure { #variant_defs } + } + } + syn::Data::Union(_) => panic!("unions are not supported"), + }; + + let compress_per_variant = s.each_variant(|variant| { + let vname = variant.ast().ident.clone(); + let construct = match &s.ast().data { + syn::Data::Struct(_) => quote! { #compressed_name }, + syn::Data::Enum(_) => quote! {#compressed_name :: #vname }, + syn::Data::Union(_) => unreachable!(), + }; + construct_compressed(&construct, variant) + }); + + let impls = s.gen_impl(quote! { + gen impl ::fuel_compression::Compressible for @Self #w_impl { + type Compressed = #compressed_name #g; + } + + gen impl ::fuel_compression::CompressibleBy for @Self #w_impl_field_bounds_compress { + async fn compress_with(&self, ctx: &mut Ctx) -> Result { + Ok(match self { #compress_per_variant }) + } + } + }); + quote! { + #def + #impls + } +} diff --git a/fuel-derive/src/compression/decompress.rs b/fuel-derive/src/compression/decompress.rs new file mode 100644 index 0000000000..97a721d6c4 --- /dev/null +++ b/fuel-derive/src/compression/decompress.rs @@ -0,0 +1,170 @@ +use proc_macro2::TokenStream as TokenStream2; +use quote::{ + format_ident, + quote, +}; + +use crate::helpers::where_clause_push; + +use super::attribute::{ + FieldAttrs, + StructureAttrs, +}; + +/// Generate a match arm for each variant of the compressed structure +/// using the given function to generate the pattern body. +fn each_variant_compressed) -> TokenStream2>( + s: &synstructure::Structure, + compressed_name: &TokenStream2, + mut f: F, +) -> TokenStream2 { + s.variants() + .iter() + .map(|variant| { + // Modify the binding pattern to match the compressed variant + let mut v2 = variant.clone(); + v2.filter(|field| { + let attrs = FieldAttrs::parse(&field.ast().attrs); + !matches!(attrs, FieldAttrs::Skip) + }); + v2.bindings_mut().iter_mut().for_each(|binding| { + binding.style = synstructure::BindStyle::Move; + }); + let mut p = v2.pat().into_iter(); + let _ = p.next().expect("pattern always begins with an identifier"); + let p = quote! { #compressed_name #(#p)* }; + + let decompressed = f(variant); + quote! { + #p => { #decompressed } + } + }) + .collect() +} + +/// Construct original version of the struct from the compressed one +fn construct_decompress( + // The original structure to construct, i.e. struct name or enum variant path + original: &TokenStream2, + variant: &synstructure::VariantInfo<'_>, +) -> TokenStream2 { + let bound_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let ty = &binding.ast().ty; + let cname = format_ident!("{}_c", binding.binding); + + match FieldAttrs::parse(&binding.ast().attrs) { + FieldAttrs::Skip => quote! { + let #cname = ::core::default::Default::default(); + }, + FieldAttrs::Normal => { + quote! { + let #cname = <#ty as ::fuel_compression::DecompressibleBy<_>>::decompress_with(#binding, ctx).await?; + } + } + } + }) + .collect(); + + let construct_fields: TokenStream2 = variant + .bindings() + .iter() + .map(|binding| { + let cname = format_ident!("{}_c", binding.binding); + if let Some(fname) = &binding.ast().ident { + quote! { #fname: #cname, } + } else { + quote! { #cname, } + } + }) + .collect(); + + let construct_fields = match variant.ast().fields { + syn::Fields::Named(_) => quote! {{ #construct_fields }}, + syn::Fields::Unnamed(_) => quote! {(#construct_fields)}, + syn::Fields::Unit => quote! {}, + }; + + quote! { + #bound_fields + #original #construct_fields + } +} + +/// Derives `DecompressibleBy` trait for the given `struct` or `enum`. +pub fn derive(mut s: synstructure::Structure) -> TokenStream2 { + s.add_bounds(synstructure::AddBounds::None) + .underscore_const(true); + + let s_attrs = match StructureAttrs::parse(&s.ast().attrs) { + Ok(v) => v, + Err(e) => return e.to_compile_error(), + }; + + let name = &s.ast().ident; + let compressed_name = format_ident!("Compressed{}", name); + + let mut g = s.ast().generics.clone(); + let w_structure = g.where_clause.take(); + let w_impl = w_structure.clone(); + for item in &s_attrs { + match item { + StructureAttrs::Discard(discard) => { + g.params = g + .params + .into_pairs() + .filter(|pair| match pair.value() { + syn::GenericParam::Type(t) => { + !discard.contains(&t.ident.to_string()) + } + _ => true, + }) + .collect(); + } + } + } + + let mut w_impl_field_bounds_decompress = w_impl.clone(); + for variant in s.variants() { + for field in variant.ast().fields.iter() { + let ty = &field.ty; + match FieldAttrs::parse(&field.attrs) { + FieldAttrs::Skip => {} + FieldAttrs::Normal => { + where_clause_push( + &mut w_impl_field_bounds_decompress, + syn::parse_quote! { #ty: ::fuel_compression::DecompressibleBy }, + ); + } + } + } + } + where_clause_push( + &mut w_impl_field_bounds_decompress, + syn::parse_quote! { Ctx: ::fuel_compression::ContextError }, + ); + + let decompress_per_variant = + each_variant_compressed(&s, "e! {#compressed_name}, |variant| { + let vname = variant.ast().ident.clone(); + let construct = match &s.ast().data { + syn::Data::Struct(_) => quote! { #name }, + syn::Data::Enum(_) => quote! {#name :: #vname }, + syn::Data::Union(_) => unreachable!(), + }; + construct_decompress(&construct, variant) + }); + + let impls = s.gen_impl(quote! { + gen impl ::fuel_compression::DecompressibleBy for @Self #w_impl_field_bounds_decompress { + async fn decompress_with(compressed: &Self::Compressed, ctx: &Ctx) -> Result { + Ok(match compressed { #decompress_per_variant }) + } + } + }); + quote! { + #impls + } +} diff --git a/fuel-derive/src/helpers.rs b/fuel-derive/src/helpers.rs new file mode 100644 index 0000000000..1a12806357 --- /dev/null +++ b/fuel-derive/src/helpers.rs @@ -0,0 +1,14 @@ +use proc_macro2::TokenStream as TokenStream2; + +pub fn where_clause_push(w: &mut Option, p: TokenStream2) { + if w.is_none() { + *w = Some(syn::WhereClause { + where_token: syn::Token![where](proc_macro2::Span::call_site()), + predicates: ::core::default::Default::default(), + }); + } + w.as_mut() + .unwrap() + .predicates + .push(syn::parse_quote! { #p }); +} diff --git a/fuel-derive/src/lib.rs b/fuel-derive/src/lib.rs index 758a7d4a07..37644db472 100644 --- a/fuel-derive/src/lib.rs +++ b/fuel-derive/src/lib.rs @@ -1,6 +1,7 @@ //! Derive macros for canonical type serialization and deserialization. -#![deny(unused_must_use, missing_docs)] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![deny(unused_must_use, unsafe_code, unused_crate_dependencies, missing_docs)] #![deny( clippy::arithmetic_side_effects, clippy::cast_sign_loss, @@ -9,22 +10,40 @@ clippy::string_slice )] +mod helpers; + extern crate proc_macro; -mod attribute; -mod deserialize; -mod serialize; -use self::{ - deserialize::deserialize_derive, - serialize::serialize_derive, -}; +mod canonical { + mod attribute; + pub mod deserialize; + pub mod serialize; +} + +synstructure::decl_derive!( + [Serialize, attributes(canonical)] => + /// Derives `Serialize` trait for the given `struct` or `enum`. + canonical::serialize::derive +); synstructure::decl_derive!( [Deserialize, attributes(canonical)] => /// Derives `Deserialize` trait for the given `struct` or `enum`. - deserialize_derive + canonical::deserialize::derive ); + +mod compression { + mod attribute; + pub mod compress; + pub mod decompress; +} + synstructure::decl_derive!( - [Serialize, attributes(canonical)] => - /// Derives `Serialize` trait for the given `struct` or `enum`. - serialize_derive + [Compress, attributes(compress)] => + /// Derives `Compressible` and `CompressibleBy` traits for the given `struct` or `enum`. + compression::compress::derive +); +synstructure::decl_derive!( + [Decompress, attributes(compress)] => + /// Derives `DecompressibleBy` trait for the given `struct` or `enum`. + compression::decompress::derive ); diff --git a/fuel-tx/Cargo.toml b/fuel-tx/Cargo.toml index b1ccdb1dc8..8459f76e16 100644 --- a/fuel-tx/Cargo.toml +++ b/fuel-tx/Cargo.toml @@ -15,6 +15,7 @@ bitflags = { workspace = true } derivative = { version = "2.2.0", default-features = false, features = ["use_core"], optional = true } derive_more = { version = "0.99", default-features = false, features = ["display"] } fuel-asm = { workspace = true, default-features = false } +fuel-compression = { workspace = true, optional = true } fuel-crypto = { workspace = true, default-features = false } fuel-merkle = { workspace = true, default-features = false, optional = true } fuel-types = { workspace = true, default-features = false, features = ["serde"] } @@ -31,17 +32,21 @@ strum_macros = { version = "0.24", optional = true } wasm-bindgen = { version = "0.2.88", optional = true } [dev-dependencies] +bimap = "0.6" bincode = { workspace = true } fuel-crypto = { workspace = true, default-features = false, features = ["random"] } fuel-tx = { path = ".", features = ["random", "serde", "test-helpers"] } fuel-types = { workspace = true, default-features = false, features = ["random"] } hex = { version = "0.4", default-features = false } insta = "1.0" +postcard = { version = "1.0", features = ["use-std"] } +pretty_assertions = "1.4.0" quickcheck = "1.0" quickcheck_macros = "1.0" rand = { version = "0.8", default-features = false, features = ["std_rng"] } rstest = "0.15" serde_json = { version = "1.0" } +tokio = { version = "1.27", features = ["full"] } [features] default = ["fuel-asm/default", "fuel-crypto/default", "fuel-merkle/default", "fuel-types/default", "std"] @@ -53,3 +58,4 @@ std = ["alloc", "fuel-asm/std", "fuel-crypto/std", "fuel-merkle/std", "fuel-type alloc = ["hashbrown", "fuel-types/alloc", "itertools/use_alloc", "derivative", "fuel-merkle", "strum", "strum_macros"] # serde is requiring alloc because its mandatory for serde_json. to avoid adding a new feature only for serde_json, we just require `alloc` here since as of the moment we don't have a use case of serde without alloc. serde = ["alloc", "fuel-asm/serde", "fuel-crypto/serde", "fuel-merkle/serde", "serde_json", "hashbrown/serde", "bitflags/serde"] +da-compression = ["serde", "fuel-compression"] diff --git a/fuel-tx/src/builder.rs b/fuel-tx/src/builder.rs index 6adb846424..67ebfe5a7a 100644 --- a/fuel-tx/src/builder.rs +++ b/fuel-tx/src/builder.rs @@ -131,7 +131,7 @@ impl TransactionBuilder