change build-time codegen to by-feature only (#68)

* change build-time codegen to by-feature only

instead of requiring a feature to *not* perform codegen at build time, we only enable that codegen by specific features. this allows using committed generated code for protocol buffers, flatbuffers, and cap'n proto by default, while still allowing tests or developers to assert that the relevant generated code is up-to-date automatically.

* upgrade prost to 0.12.3

* actually fail build step when out of date

* harden the codegen tools in the CI config, with updates and more pinning

* update generated data and protoc version to try to match CI

.github/workflows/build.yml

@@ -35,20 +35,45 @@ jobs:
 
           cargo fmt --check
 
-      - name: build
+      - name: set up codegen dependencies
         shell: bash
         run: |
 
-          wget "https://github.com/protocolbuffers/protobuf/releases/download/v23.3/protoc-23.3-linux-x86_64.zip" -O protoc.zip
-          unzip protoc.zip
-          chmod +x bin/protoc
-          mv bin/protoc /usr/local/bin
+          mkdir proto
+          (
+            cd proto
+            wget "https://github.com/protocolbuffers/protobuf/releases/download/v26.0/protoc-26.0-linux-x86_64.zip" -O protoc.zip
+            unzip protoc.zip
+            chmod +x bin/protoc
+            mv bin/protoc /usr/local/bin/
+          )
+          protoc --version
 
-          wget "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" -O flatbuffers
-          unzip flatbuffers
-          chmod +x flatc
-          mv flatc /usr/local/bin
+          mkdir flatbuffers
+          (
+            cd flatbuffers
+            wget "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" -O flatbuffers.zip
+            unzip flatbuffers.zip
+            chmod +x ./flatc
+            mv ./flatc /usr/local/bin
+          )
+          flatc --version
 
-          sudo apt-get install -y capnproto libprotobuf-dev
+          wget "https://capnproto.org/capnproto-c++-1.0.2.tar.gz" -O capnproto.tar.gz
+          tar zxf capnproto.tar.gz
+          (
+            cd capnproto-c++-1.0.2
+            ./configure
+            sudo make -j install
+          )
+          capnp --version
 
-          cargo build --benches
+      - name: build
+        shell: bash
+        run: |
+          cargo build --benches --features regenerate-capnp,regenerate-flatbuffers,regenerate-prost
+
+      - name: check generated code
+        shell: bash
+        run: |
+          git diff --exit-code && echo 'ok!' || (echo 'committed generated code is not up to date!'; exit 1)

Cargo.toml

@@ -76,7 +76,7 @@ parity-scale-codec-derive = { version = "3.6.5", optional = true }
 postcard = { version = "1.0.8", features = ["alloc"], optional = true }
 pot = { version = "3.0.0", optional = true }
 pprof = { version = "0.13", features = ["flamegraph"], optional = true }
-prost = { version = "0.12.1", optional = true }
+prost = { version = "0.12.3", optional = true }
 rand = "0.8.5"
 # TODO: unfork after rkyv updates to 0.8 or `stdsimd` cfg for nightly in aHash 0.7 is fixed
 rkyv = { version = "0.7.44", git = "https://github.com/rkyv/rkyv", branch = "0.7-hashbrown-0.14", features = ["validation"], optional = true }
@@ -119,7 +119,6 @@ default = [
   "postcard",
   "pot",
   "prost",
-  "prost-build",
   "rkyv",
   "rmp-serde",
   "ron",
@@ -131,26 +130,26 @@ default = [
   "speedy",
   "savefile"
 ]
+capnp = ["dep:capnp"]
+prost = ["dep:capnp", "dep:prost"]
 simd-json = ["dep:simd-json", "simd-json-derive"]
-
 savefile = ["dep:savefile", "savefile-derive"]
-
 scale = ["parity-scale-codec", "parity-scale-codec-derive"]
 
-# Enable these to use the already-committed generated files for capnp and
-# flatbuffers rather than regenerating them.
-use-committed-capnp = []
-use-committed-flatbuffers = []
+# Enable these features to regenerate generated files rather than using the committed versions.
+regenerate-capnp = ["dep:capnpc"]
+regenerate-flatbuffers = ["dep:flatc-rust"]
+regenerate-prost = ["dep:prost-build"]
 
 [dev-dependencies]
 rand_pcg = "0.3.1"
 
 [build-dependencies]
 bebop-tools = "2.8.7"
 capnp = "0.18.3"
-capnpc = "0.18.0"
-flatc-rust = "0.2.0"
-prost-build = { version = "0.12.1", optional = true }
+capnpc = { version = "0.18.0", optional = true }
+flatc-rust = { version = "0.2.0", optional = true }
+prost-build = { version = "0.12.3", optional = true }
 
 [[bench]]
 harness = false

build.rs

@@ -1,3 +1,4 @@
+#[allow(unused_imports)]
 use std::{
     env,
     path::{Path, PathBuf},
@@ -15,6 +16,7 @@ fn bebop_compile_dataset(name: &'static str) {
     );
 }
 
+#[cfg(feature = "regenerate-capnp")]
 fn capnpc_compile_dataset(name: &'static str) -> capnp::Result<()> {
     let mut command = capnpc::CompilerCommand::new();
     #[cfg(windows)]
@@ -25,6 +27,7 @@ fn capnpc_compile_dataset(name: &'static str) -> capnp::Result<()> {
     command.run()
 }
 
+#[cfg(feature = "regenerate-flatbuffers")]
 fn flatc_compile_dataset(name: &'static str) -> flatc_rust::Result<()> {
     #[cfg(windows)]
     let flatc = flatc_rust::Flatc::from_path("./prebuilt/flatc.exe");
@@ -40,7 +43,7 @@ fn flatc_compile_dataset(name: &'static str) -> flatc_rust::Result<()> {
     })
 }
 
-#[cfg(feature = "prost-build")]
+#[cfg(feature = "regenerate-prost")]
 fn prost_compile_dataset(name: &'static str) -> std::io::Result<()> {
     if cfg!(windows) {
         match env::var("PROTOC") {
@@ -52,21 +55,29 @@ fn prost_compile_dataset(name: &'static str) -> std::io::Result<()> {
     }
     let mut prost_config = prost_build::Config::new();
     prost_config.protoc_arg("--experimental_allow_proto3_optional");
+    prost_config.out_dir(format!("./src/datasets/{name}"));
     prost_config.compile_protos(
-        &[format!("./src/datasets/{0}/{0}.proto", name).as_str()],
+        &[format!("./src/datasets/{name}/{name}.proto").as_str()],
         &["src"],
     )
 }
 
 fn main() {
-    const DATASETS: &[&str] = &["log", "mesh", "minecraft_savedata", "mk48"];
-    for &name in DATASETS.iter() {
-        // bebop_compile_dataset(name);
-        #[cfg(all(feature = "capnp", not(feature = "use-committed-capnp")))]
-        capnpc_compile_dataset(name).unwrap();
-        #[cfg(all(feature = "flatbuffers", not(feature = "use-committed-flatbuffers")))]
-        flatc_compile_dataset(name).unwrap();
-        #[cfg(feature = "prost-build")]
-        prost_compile_dataset(name).unwrap();
+    #[cfg(any(
+        feature = "regenerate-capnp",
+        feature = "regenerate-flatbuffers",
+        feature = "regenerate-prost"
+    ))]
+    {
+        const DATASETS: &[&str] = &["log", "mesh", "minecraft_savedata", "mk48"];
+        for &name in DATASETS.iter() {
+            // bebop_compile_dataset(name);
+            #[cfg(feature = "regenerate-capnp")]
+            capnpc_compile_dataset(name).unwrap();
+            #[cfg(feature = "regenerate-flatbuffers")]
+            flatc_compile_dataset(name).unwrap();
+            #[cfg(feature = "regenerate-prost")]
+            prost_compile_dataset(name).unwrap();
+        }
     }
 }

src/datasets/log/log_capnp.rs

@@ -834,7 +834,7 @@ pub mod logs {
       ::capnp::traits::FromPointerBuilder::get_from_pointer(self.builder.get_pointer_field(0), ::core::option::Option::None)
     }
     #[inline]
-    pub fn set_logs(&mut self, value: ::capnp::struct_list::Reader<'a,crate::datasets::log::log_capnp::log::Owned>) -> ::capnp::Result<()> {
+    pub fn set_logs(&mut self, value: ::capnp::struct_list::Reader<'_,crate::datasets::log::log_capnp::log::Owned>) -> ::capnp::Result<()> {
       ::capnp::traits::SetPointerBuilder::set_pointer_builder(self.builder.reborrow().get_pointer_field(0), value, false)
     }
     #[inline]