update serialization libraries (#85)

* ahash 0.7.8 fixed stdsimd problem: update to latest nightly

* disable wiring

* update codegen

* updates

.github/workflows/bench.yml

@@ -28,7 +28,7 @@ jobs:
 
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2024-02-01
+          toolchain: nightly
           override: true
           components: rust-src
 

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
 
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2024-02-01
+          toolchain: nightly
           override: true
           components: rust-src, rustfmt
 
@@ -42,7 +42,7 @@ jobs:
           mkdir proto
           (
             cd proto
-            wget "https://github.com/protocolbuffers/protobuf/releases/download/v26.0/protoc-26.0-linux-x86_64.zip" -O protoc.zip
+            wget "https://github.com/protocolbuffers/protobuf/releases/download/v27.2/protoc-27.2-linux-x86_64.zip" -O protoc.zip
             unzip protoc.zip
             chmod +x bin/protoc
             mv bin/protoc /usr/local/bin/
@@ -52,7 +52,7 @@ jobs:
           mkdir flatbuffers
           (
             cd flatbuffers
-            wget "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" -O flatbuffers.zip
+            wget "https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.clang++-15.zip" -O flatbuffers.zip
             unzip flatbuffers.zip
             chmod +x ./flatc
             mv ./flatc /usr/local/bin

Cargo.toml

@@ -50,50 +50,49 @@ alkahest = { version = "=0.1.5", optional = true, features = [
   "nightly",
 ] }
 bebop = { version = "=2.8.7", optional = true }
-bilrost = { version = "=0.1007.0", optional = true }
+bilrost = { version = "=0.1010.0", optional = true }
 bincode1 = { package = "bincode", version = "=1.3.3", optional = true }
 # Can't call it bincode2 because of a current issue of bincode2
 bincode = { package = "bincode", version = "=2.0.0-rc.3", optional = true }
 bitcode = { version = "=0.6.0", optional = true }
-borsh = { version = "=1.3.1", features = ["derive"], optional = true }
+borsh = { version = "=1.5.1", features = ["derive"], optional = true }
 # TODO: Unfork after bson adds support for pre-warmed serialization buffers
 # https://github.com/mongodb/bson-rust/pull/328
 bson = { version = "=2.9.0", git = "https://github.com/djkoloski/bson-rust", branch = "add_into_vec", optional = true }
 bytecheck = { version = "=0.6.12", optional = true }
-bytemuck = { version = "=1.14.1", optional = true }
-capnp = { version = "=0.18.13", optional = true }
+bytemuck = { version = "=1.16.1", optional = true }
+capnp = { version = "=0.19.6", optional = true }
 cbor4ii = { version = "=0.3.2", features = [ "use_std", "serde1" ], optional = true }
 ciborium = { version = "=0.2.2", optional = true }
 criterion = "=0.5.1"
-databuf = { version = "=0.5", optional = true }
-dlhn = { version = "=0.1.6", optional = true }
-flatbuffers = { version = "=23.5.26", optional = true }
-libflate = "=1.4.0"
+databuf = { version = "=0.5.0", optional = true }
+dlhn = { version = "=0.1.7", optional = true }
+flatbuffers = { version = "=24.3.25", optional = true }
+libflate = "=2.1.0"
 msgpacker = { version = "=0.4.3", optional = true }
 nachricht-serde = { version = "=0.4.0", optional = true }
 nanoserde = {version = "=0.1.37", optional = true }
-parity-scale-codec = { version = "=3.6.9", features = ["full"], optional = true }
-parity-scale-codec-derive = { version = "=3.6.9", optional = true }
+parity-scale-codec = { version = "=3.6.12", features = ["full"], optional = true }
+parity-scale-codec-derive = { version = "=3.6.12", optional = true }
 postcard = { version = "=1.0.8", features = ["alloc"], optional = true }
 pot = { version = "=3.0.0", optional = true }
 pprof = { version = "=0.13.0", features = ["flamegraph"], optional = true }
-prost = { version = "=0.12.4", optional = true }
+prost = { version = "=0.12.6", optional = true }
 rand = "=0.8.5"
-# TODO: unfork after rkyv updates to 0.8 or `stdsimd` cfg for nightly in aHash 0.7 is fixed
-rkyv = { version = "=0.7.44", git = "https://github.com/rkyv/rkyv", branch = "0.7-hashbrown-0.14", features = ["validation"], optional = true }
-rmp-serde = { version = "=1.1.2", optional = true }
+rkyv = { version = "=0.7.44", features = ["validation"], optional = true }
+rmp-serde = { version = "=1.3.0", optional = true }
 ron = { version = "=0.8.1", optional = true }
-serde = { version = "=1.0.196", features = ["derive"], optional = true }
+serde = { version = "=1.0.204", features = ["derive"], optional = true }
 serde_bare = { version = "=0.5.0", optional = true }
 serde_cbor = { version = "=0.11.2", optional = true }
-serde_json = { version = "=1.0.115", features = ["float_roundtrip"], optional = true }
-simd-json = { version = "=0.13.9", optional = true }
+serde_json = { version = "=1.0.120", features = ["float_roundtrip"], optional = true }
+simd-json = { version = "=0.13.10", optional = true }
 simd-json-derive = { version = "=0.13.0", optional = true }
 speedy = { version = "=0.8.7", optional = true }
-savefile = { version = "=0.16.5", optional = true }
-savefile-derive = { version = "=0.16.5", optional = true }
+savefile = { version = "=0.17.6", optional = true }
+savefile-derive = { version = "=0.17.6", optional = true }
 wiring = { version = "=0.2.1", optional = true }
-zstd = "=0.12.4"
+zstd = "=0.13.2"
 
 [features]
 default = [
@@ -132,7 +131,7 @@ default = [
   "simd-json",
   "speedy",
   "savefile",
-  "wiring"
+  # "wiring" # does not build: slice_flatten is being stabilized and has been renamed
 ]
 capnp = ["dep:capnp"]
 prost = ["dep:capnp", "dep:prost"]
@@ -150,10 +149,10 @@ rand_pcg = "0.3.1"
 
 [build-dependencies]
 bebop-tools = "=2.8.7"
-capnp = "=0.18.13"
-capnpc = { version = "=0.18.1", optional = true }
+capnp = "=0.19.6"
+capnpc = { version = "=0.19.0", optional = true }
 flatc-rust = { version = "=0.2.0", optional = true }
-prost-build = { version = "=0.12.4", optional = true }
+prost-build = { version = "=0.12.6", optional = true }
 
 [[bench]]
 harness = false

src/datasets/log/log_capnp.rs

@@ -116,7 +116,7 @@ pub mod address {
     }
   }
 
-  impl <'a,> ::capnp::traits::SetPointerBuilder for Reader<'a,>  {
+  impl <'a,> ::capnp::traits::SetterInput<Owned<>> for Reader<'a,>  {
     fn set_pointer_builder(mut pointer: ::capnp::private::layout::PointerBuilder<'_>, value: Self, canonicalize: bool) -> ::capnp::Result<()> { pointer.set_struct(&value.reader, canonicalize) }
   }
 
@@ -274,9 +274,11 @@ pub mod address {
       encoded_node: &ENCODED_NODE,
       nonunion_members: NONUNION_MEMBERS,
       members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
+      members_by_name: MEMBERS_BY_NAME,
     };
     pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3];
     pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
+    pub static MEMBERS_BY_NAME : &[u16] = &[0,1,2,3];
     pub const TYPE_ID: u64 = 0x96d2_eb22_6a8b_d47a;
   }
 }
@@ -426,7 +428,7 @@ pub mod log {
     }
   }
 
-  impl <'a,> ::capnp::traits::SetPointerBuilder for Reader<'a,>  {
+  impl <'a,> ::capnp::traits::SetterInput<Owned<>> for Reader<'a,>  {
     fn set_pointer_builder(mut pointer: ::capnp::private::layout::PointerBuilder<'_>, value: Self, canonicalize: bool) -> ::capnp::Result<()> { pointer.set_struct(&value.reader, canonicalize) }
   }
 
@@ -450,7 +452,7 @@ pub mod log {
     }
     #[inline]
     pub fn set_address(&mut self, value: crate::datasets::log::log_capnp::address::Reader<'_>) -> ::capnp::Result<()> {
-      ::capnp::traits::SetPointerBuilder::set_pointer_builder(self.builder.reborrow().get_pointer_field(0), value, false)
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(0), value, false)
     }
     #[inline]
     pub fn init_address(self, ) -> crate::datasets::log::log_capnp::address::Builder<'a> {
@@ -465,8 +467,8 @@ pub mod log {
       ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(1), ::core::option::Option::None)
     }
     #[inline]
-    pub fn set_identity(&mut self, value: ::capnp::text::Reader<'_>)  {
-      self.builder.reborrow().get_pointer_field(1).set_text(value);
+    pub fn set_identity(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text::Owned>)  {
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(1), value, false).unwrap()
     }
     #[inline]
     pub fn init_identity(self, size: u32) -> ::capnp::text::Builder<'a> {
@@ -481,8 +483,8 @@ pub mod log {
       ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(2), ::core::option::Option::None)
     }
     #[inline]
-    pub fn set_userid(&mut self, value: ::capnp::text::Reader<'_>)  {
-      self.builder.reborrow().get_pointer_field(2).set_text(value);
+    pub fn set_userid(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text::Owned>)  {
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(2), value, false).unwrap()
     }
     #[inline]
     pub fn init_userid(self, size: u32) -> ::capnp::text::Builder<'a> {
@@ -497,8 +499,8 @@ pub mod log {
       ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(3), ::core::option::Option::None)
     }
     #[inline]
-    pub fn set_date(&mut self, value: ::capnp::text::Reader<'_>)  {
-      self.builder.reborrow().get_pointer_field(3).set_text(value);
+    pub fn set_date(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text::Owned>)  {
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(3), value, false).unwrap()
     }
     #[inline]
     pub fn init_date(self, size: u32) -> ::capnp::text::Builder<'a> {
@@ -513,8 +515,8 @@ pub mod log {
       ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(4), ::core::option::Option::None)
     }
     #[inline]
-    pub fn set_request(&mut self, value: ::capnp::text::Reader<'_>)  {
-      self.builder.reborrow().get_pointer_field(4).set_text(value);
+    pub fn set_request(&mut self, value: impl ::capnp::traits::SetterInput<::capnp::text::Owned>)  {
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(4), value, false).unwrap()
     }
     #[inline]
     pub fn init_request(self, size: u32) -> ::capnp::text::Builder<'a> {
@@ -699,9 +701,11 @@ pub mod log {
       encoded_node: &ENCODED_NODE,
       nonunion_members: NONUNION_MEMBERS,
       members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
+      members_by_name: MEMBERS_BY_NAME,
     };
     pub static NONUNION_MEMBERS : &[u16] = &[0,1,2,3,4,5,6];
     pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
+    pub static MEMBERS_BY_NAME : &[u16] = &[0,5,3,1,4,6,2];
     pub const TYPE_ID: u64 = 0x87f6_c7ab_2de3_81ff;
   }
 }
@@ -811,7 +815,7 @@ pub mod logs {
     }
   }
 
-  impl <'a,> ::capnp::traits::SetPointerBuilder for Reader<'a,>  {
+  impl <'a,> ::capnp::traits::SetterInput<Owned<>> for Reader<'a,>  {
     fn set_pointer_builder(mut pointer: ::capnp::private::layout::PointerBuilder<'_>, value: Self, canonicalize: bool) -> ::capnp::Result<()> { pointer.set_struct(&value.reader, canonicalize) }
   }
 
@@ -835,7 +839,7 @@ pub mod logs {
     }
     #[inline]
     pub fn set_logs(&mut self, value: ::capnp::struct_list::Reader<'_,crate::datasets::log::log_capnp::log::Owned>) -> ::capnp::Result<()> {
-      ::capnp::traits::SetPointerBuilder::set_pointer_builder(self.builder.reborrow().get_pointer_field(0), value, false)
+      ::capnp::traits::SetterInput::set_pointer_builder(self.builder.reborrow().get_pointer_field(0), value, false)
     }
     #[inline]
     pub fn init_logs(self, size: u32) -> ::capnp::struct_list::Builder<'a,crate::datasets::log::log_capnp::log::Owned> {
@@ -908,9 +912,11 @@ pub mod logs {
       encoded_node: &ENCODED_NODE,
       nonunion_members: NONUNION_MEMBERS,
       members_by_discriminant: MEMBERS_BY_DISCRIMINANT,
+      members_by_name: MEMBERS_BY_NAME,
     };
     pub static NONUNION_MEMBERS : &[u16] = &[0];
     pub static MEMBERS_BY_DISCRIMINANT : &[u16] = &[];
+    pub static MEMBERS_BY_NAME : &[u16] = &[0];
     pub const TYPE_ID: u64 = 0x9658_6a57_8e4a_5409;
   }
 }

src/datasets/log/log_generated.rs

@@ -235,8 +235,8 @@ impl<'a> Log<'a> {
     Log { _tab: table }
   }
   #[allow(unused_mut)]
-  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
-    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
     args: &'args LogArgs<'args>
   ) -> flatbuffers::WIPOffset<Log<'bldr>> {
     let mut builder = LogBuilder::new(_fbb);
@@ -344,11 +344,11 @@ impl<'a> Default for LogArgs<'a> {
   }
 }
 
-pub struct LogBuilder<'a: 'b, 'b> {
-  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+pub struct LogBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
   start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
 }
-impl<'a: 'b, 'b> LogBuilder<'a, 'b> {
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LogBuilder<'a, 'b, A> {
   #[inline]
   pub fn add_address(&mut self, address: &Address) {
     self.fbb_.push_slot_always::<&Address>(Log::VT_ADDRESS, address);
@@ -378,7 +378,7 @@ impl<'a: 'b, 'b> LogBuilder<'a, 'b> {
     self.fbb_.push_slot::<u64>(Log::VT_SIZE_, size_, 0);
   }
   #[inline]
-  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LogBuilder<'a, 'b> {
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> LogBuilder<'a, 'b, A> {
     let start = _fbb.start_table();
     LogBuilder {
       fbb_: _fbb,
@@ -432,8 +432,8 @@ impl<'a> Logs<'a> {
     Logs { _tab: table }
   }
   #[allow(unused_mut)]
-  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>(
-    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
     args: &'args LogsArgs<'args>
   ) -> flatbuffers::WIPOffset<Logs<'bldr>> {
     let mut builder = LogsBuilder::new(_fbb);
@@ -475,17 +475,17 @@ impl<'a> Default for LogsArgs<'a> {
   }
 }
 
-pub struct LogsBuilder<'a: 'b, 'b> {
-  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,
+pub struct LogsBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
   start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
 }
-impl<'a: 'b, 'b> LogsBuilder<'a, 'b> {
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LogsBuilder<'a, 'b, A> {
   #[inline]
   pub fn add_logs(&mut self, logs: flatbuffers::WIPOffset<flatbuffers::Vector<'b , flatbuffers::ForwardsUOffset<Log<'b >>>>) {
     self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Logs::VT_LOGS, logs);
   }
   #[inline]
-  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> LogsBuilder<'a, 'b> {
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> LogsBuilder<'a, 'b, A> {
     let start = _fbb.start_table();
     LogsBuilder {
       fbb_: _fbb,