fix!: ensure encoding is compatible with Go's klauspost/reedsolomon

Cargo.toml

@@ -9,4 +9,19 @@ lazy_static = "1.4"
 thiserror = "1"
 
 [dev-dependencies]
-serde_json = "1"
+go-leopard = { path = "go-leopard" }
+proptest = "1.4.0"
+rand = "0.8"
+test-strategy = "0.3"
+
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+getrandom = { version = "0.2.10", features = ["js"] }
+
+[workspace]
+members = ["go-leopard"]
+
+[profile.test.package.proptest]
+opt-level = 3
+
+[profile.test.package.rand_chacha]
+opt-level = 3

go-leopard/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "go-leopard"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+thiserror = "1"
+
+[build-dependencies]
+bindgen = "0.69"

go-leopard/build.rs

@@ -0,0 +1,36 @@
+//! build go leopard library with bindings
+
+use std::{env, path::PathBuf, process::Command};
+
+use bindgen::Builder;
+
+fn main() {
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+    // build go as a static lib and a header with cgo
+    Command::new("go")
+        .arg("build")
+        .arg("-buildmode=c-archive")
+        .arg("-o")
+        .arg(out_path.join("libgoleopard.a"))
+        .arg("leopard.go")
+        .current_dir("go")
+        .status()
+        .expect("Building libgoleopard.a failed");
+
+    // generate bindings
+    Builder::default()
+        .header(out_path.join("libgoleopard.h").to_str().unwrap())
+        .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
+        .generate()
+        .expect("Unable to generate bindings")
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("Couldn't write bindings!");
+
+    println!("cargo:rerun-if-changed=go");
+    println!(
+        "cargo:rustc-link-search=native={}",
+        out_path.to_str().unwrap()
+    );
+    println!("cargo:rustc-link-lib=static=goleopard");
+}

go-leopard/go/go.mod

@@ -0,0 +1,9 @@
+module github.com/eigerco/leopard-codec
+
+go 1.23
+
+require (
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/klauspost/reedsolomon v1.12.1
+	golang.org/x/sys v0.5.0 // indirect
+)

go-leopard/go/go.sum

@@ -0,0 +1,6 @@
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q=
+github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

go-leopard/go/leopard.go

@@ -0,0 +1,41 @@
+package main
+
+import "C"
+
+import (
+	"unsafe"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+//export encode
+func encode(ptr **byte, shard_size uint, data_shards uint, total_shards uint) int {
+	input := unsafe.Slice(ptr, total_shards)
+	shards := make([][]byte, total_shards)
+
+	// copy shards to go
+	for i := uint(0); i < total_shards; i++ {
+		shard := unsafe.Slice(input[i], shard_size)
+		shards[i] = make([]byte, shard_size)
+		copy(shards[i], shard)
+	}
+
+	// create the codec and encode
+	codec, err := reedsolomon.New(int(data_shards), int(total_shards-data_shards), reedsolomon.WithLeopardGF(true))
+	if err != nil {
+		return -1
+	}
+	if codec.Encode(shards) != nil {
+		return -2
+	}
+
+	// copy shards back to the input ptr
+	for i := uint(0); i < total_shards; i++ {
+		shard := unsafe.Slice(input[i], shard_size)
+		copy(shard, shards[i])
+	}
+
+	return 0
+}
+
+func main() {}

go-leopard/src/lib.rs

@@ -0,0 +1,53 @@
+//! Rust wrapper over Go's reedsolomon module
+
+use std::ffi::c_uchar;
+
+use thiserror::Error;
+
+#[allow(non_upper_case_globals)]
+#[allow(non_camel_case_types)]
+#[allow(non_snake_case)]
+#[allow(unused)]
+mod go {
+    include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+}
+
+/// Possible errors that can happen when interacting with `go-leopard`
+#[derive(Debug, Error)]
+pub enum Error {
+    /// Creating codec failed
+    #[error("Creating the codec failed")]
+    CreatingCodecFailed,
+
+    /// Encoding failed
+    #[error("Encoding failed")]
+    EncodingFailed,
+}
+
+/// Encode the shards using the Go's reedsolomon module
+pub fn encode(
+    shards: &mut [&mut [u8]],
+    data_shards: usize,
+    shard_size: usize,
+) -> Result<(), Error> {
+    let mut shards_out: Vec<_> = shards
+        .iter_mut()
+        .map(|shard| shard.as_mut_ptr() as *mut c_uchar)
+        .collect();
+
+    let res = unsafe {
+        go::encode(
+            shards_out.as_mut_ptr(),
+            shard_size as u64,
+            data_shards as u64,
+            shards.len() as u64,
+        )
+    };
+
+    match res {
+        0 => Ok(()),
+        -1 => Err(Error::CreatingCodecFailed),
+        -2 => Err(Error::EncodingFailed),
+        _ => unreachable!(),
+    }
+}