Rewrite automation in Rust

.github/workflows/bench.yml

@@ -3,6 +3,9 @@ name: bench
 on:
   workflow_dispatch:
   push:
+    paths-ignore:
+      - 'README.md'
+      - 'benchmark_results/**'
 
 env:
   CI: true
@@ -47,7 +50,7 @@ jobs:
           git config --global user.name github-actions
           git config --global user.email github-actions@github.com
 
-          node update_benchmark.mjs
+          cargo run -p bencher
 
           git add -A benchmark_results
           git add README.md

.gitignore

@@ -1,5 +1,3 @@
-*.log
-.date
 Cargo.lock
 /target
 /.vscode

Cargo.toml

@@ -1,9 +1,36 @@
-[package]
+[workspace]
+members = [
+  "tools/bencher",
+  "tools/formatter",
+  "tools/parser",
+  "tools/schema",
+]
+
+[workspace.package]
+version = "0.1.0"
 authors = ["David Koloski <djkoloski@gmail.com>"]
-build = "build.rs"
 edition = "2021"
+license = "MIT"
+publish = false
+
+[workspace.dependencies]
+cargo_metadata = "0.15"
+clap = { version = "4", features = ["derive"] }
+regex = "1.9"
+schema = { path = "tools/schema" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tempfile = "3.6"
+time = "0.3"
+
+[package]
 name = "rust_serialization_benchmark"
-version = "0.1.1"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 # Some features may require multiple dependencies to compile properly
@@ -26,7 +53,7 @@ bytecheck = { version = "0.6.10", optional = true }
 bytemuck = { version = "1.13.1", optional = true }
 capnp = { version = "0.16.1", optional = true }
 ciborium = { version = "0.2.0", optional = true }
-criterion = "0.4.0"
+criterion = "0.5"
 dlhn = { version = "0.1.4", optional = true }
 flatbuffers = { version = "23.1.21", optional = true }
 libflate = "1.3.0"

build.rs

@@ -18,7 +18,7 @@ fn bebop_compile_dataset(name: &'static str) {
 fn capnpc_compile_dataset(name: &'static str) -> capnp::Result<()> {
     let mut command = capnpc::CompilerCommand::new();
     #[cfg(windows)]
-    command.capnp_executable("tools/capnp.exe");
+    command.capnp_executable("prebuilt/capnp.exe");
     command.file(&format!("src/datasets/{0}/{0}.capnp", name));
     command.output_path(".");
     command.default_parent_module(vec!["datasets".into(), name.into()]);
@@ -27,7 +27,7 @@ fn capnpc_compile_dataset(name: &'static str) -> capnp::Result<()> {
 
 fn flatc_compile_dataset(name: &'static str) -> flatc_rust::Result<()> {
     #[cfg(windows)]
-    let flatc = flatc_rust::Flatc::from_path("./tools/flatc.exe");
+    let flatc = flatc_rust::Flatc::from_path("./prebuilt/flatc.exe");
     #[cfg(not(windows))]
     let flatc = flatc_rust::Flatc::from_env_path();
 
@@ -45,7 +45,7 @@ fn prost_compile_dataset(name: &'static str) -> std::io::Result<()> {
     if cfg!(windows) {
         match env::var("PROTOC") {
             Err(_) => {
-                env::set_var("PROTOC", "./tools/protoc.exe");
+                env::set_var("PROTOC", "./prebuilt/protoc.exe");
             }
             _ => {}
         }

README.md.template → tools/README.md.template

@@ -1,3 +1,5 @@
+{dne}
+
 # Rust serialization benchmark
 
 The goal of these benchmarks is to provide thorough and complete benchmarks for various rust
@@ -24,9 +26,9 @@ Zero-copy deserialization libraries have an additional set of benchmarks:
 
 Some benchmark results may be italicized and followed by an asterisk. Mouse over these for more details on what situation was benchmarked. Other footnotes are located at the bottom.
 
-## Last updated: ${date}
+## Last updated: {date}
 
-${results}
+{tables}{links}
 
 ## Footnotes:
 

tools/bencher/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "bencher"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[dependencies]
+tempfile.workspace = true
+time.workspace = true

tools/bencher/src/main.rs

@@ -0,0 +1,80 @@
+use std::{fs, path::PathBuf, process::Command};
+
+use tempfile::NamedTempFile;
+use time::OffsetDateTime;
+
+fn main() {
+    let now = OffsetDateTime::now_utc();
+
+    let metadata_path = NamedTempFile::new().unwrap().into_temp_path();
+    let metadata = Command::new("cargo")
+        .args(["metadata"])
+        .output()
+        .unwrap()
+        .stdout;
+    fs::write(&metadata_path, metadata).unwrap();
+
+    let mut bench_path = PathBuf::from("benchmark_results");
+    bench_path.push(&format!(
+        "{}-{}-{}_{}-{}-{}",
+        now.year(),
+        now.month() as usize,
+        now.day(),
+        now.hour(),
+        now.minute(),
+        now.second(),
+    ));
+
+    let mut log_path = bench_path.clone();
+    log_path.set_extension("log");
+    let log = Command::new("cargo")
+        .args(["bench"])
+        .output()
+        .unwrap()
+        .stdout;
+    fs::write(&log_path, log).unwrap();
+
+    let mut json_path = bench_path.clone();
+    json_path.set_extension("json");
+    Command::new("cargo")
+        .args(["run", "-p", "parser", "--", "--log"])
+        .arg(&log_path)
+        .arg("--meta")
+        .arg(&metadata_path)
+        .arg("--output")
+        .arg(&json_path)
+        .status()
+        .unwrap();
+
+    let mut config_path = PathBuf::from("tools");
+    config_path.push("config.json");
+
+    let mut template_path = PathBuf::from("tools");
+    template_path.push("README.md.template");
+
+    Command::new("cargo")
+        .args(["run", "-p", "formatter", "--"])
+        .arg(&json_path)
+        .arg("--config")
+        .arg(&config_path)
+        .arg("--template")
+        .arg(&template_path)
+        .args([
+            "--date",
+            &format!(
+                "{}-{}-{} {}:{}:{}",
+                now.year(),
+                now.month() as usize,
+                now.day(),
+                now.hour(),
+                now.minute(),
+                now.second(),
+            ),
+            "--output",
+            "README.md",
+        ])
+        .status()
+        .unwrap();
+
+    metadata_path.close().unwrap();
+}

tools/config.json

@@ -0,0 +1,9 @@
+{
+    "descriptions": {
+        "log": "This data set is composed of HTTP request logs that are small and contain many strings.",
+        "mesh": "This data set is a single mesh. The mesh contains an array of triangles, each of which has three vertices and a normal vector.",
+        "mk48": "This data set is composed of mk48.io game updates that contain data with many exploitable patterns and invariants.",
+        "minecraft_savedata": "This data set is composed of Minecraft player saves that contain highly structured data."
+    },
+    "do_not_edit": "<!-- AUTOMATICALLY GENERATED, DO NOT EDIT -->\n<!-- edit README.md.template instead -->"
+}

tools/formatter/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "formatter"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[dependencies]
+clap.workspace = true
+schema.workspace = true
+serde.workspace = true
+serde_json.workspace = true