Improve benchmark infrastructure and add comparison with unoptimised …

…implementation
serokell · Oct 9, 2024 · 2df6d2a · 2df6d2a
1 parent 643266c
commit 2df6d2a
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 10 deletions.
diff --git a/collections/motoko/dfx.json b/collections/motoko/dfx.json
@@ -7,6 +7,10 @@
     "persistentmap": {
       "type": "motoko",
       "main": "src/persistentmap.mo"
+    },
+    "persistentmap_baseline": {
+      "type": "motoko",
+      "main": "src/persistentmap_baseline.mo"
     }
   },
   "defaults": {

diff --git a/collections/motoko/mops.template.toml b/collections/motoko/mops.template.toml
@@ -1,5 +1,6 @@
 [dependencies]
-base = "https://github.com/serokell/motoko-base#milestone-1"
+base = "https://github.com/serokell/motoko-base#$TEST_BRANCH"
+base-unopt = "https://github.com/serokell/motoko-base#$BASELINE_BRANCH"
 stableheapbtreemap = "1.3.0"
 map = "9.0.0"
 splay = "0.1.0"

diff --git a/collections/motoko/src/persistentmap_baseline.mo b/collections/motoko/src/persistentmap_baseline.mo
@@ -0,0 +1,58 @@
+import Map "mo:base-unopt/PersistentOrderedMap";
+import Nat64 "mo:base/Nat64";
+import Nat32 "mo:base/Nat32";
+import Iter "mo:base/Iter";
+import Option "mo:base/Option";
+import Random "random";
+import Profiling "../../../utils/motoko/Profiling";
+
+actor {
+    stable let profiling = Profiling.init();
+
+    let mapOps = Map.MapOps<Nat64>(Nat64.compare);
+    stable var rbMap = Map.empty<Nat64, Nat64>();
+    let rand = Random.new(null, 42);
+
+    public func generate(size: Nat32) : async () {
+        let rand = Random.new(?size, 1);
+        let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
+        rbMap := mapOps.fromIter(iter);
+    };
+    public query func get_mem() : async (Nat,Nat,Nat) {
+        Random.get_memory()
+    };
+    public func batch_get(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func batch_put(n : Nat) : async () {
+        for (_ in Iter.range(1, n)) {
+            let k = Option.get<Nat64>(rand.next(), 0);
+            rbMap := mapOps.put(rbMap, k, k);
+        }
+    };
+    public func batch_remove(n : Nat) : async () {
+        let rand = Random.new(null, 1);
+        for (_ in Iter.range(1, n)) {
+            rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
+        }
+    };
+    public func foldLeft() : async () {
+        ignore Map.foldLeft<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func foldRight() : async () {
+        ignore Map.foldRight<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
+    };
+    public func mapfilter() : async () {
+        ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
+          switch (key % 2) {
+            case 1 { null };
+            case _ { ?value }
+          };
+        })
+    };
+    public func map() : async () {
+      ignore Map.map<Nat64, Nat64, Nat64>(rbMap, func (key, value) {key + value})
+    };
+}
diff --git a/collections/perf.sh b/collections/perf.sh
@@ -7,6 +7,7 @@ let mo_config = record { start_page = 16; page_limit = 128 };
 // let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
 let rbtree = wasm_profiling("motoko/.dfx/local/canisters/rbtree/rbtree.wasm", mo_config);
 let persistentmap = wasm_profiling("motoko/.dfx/local/canisters/persistentmap/persistentmap.wasm", mo_config);
+let persistentmap_baseline = wasm_profiling("motoko/.dfx/local/canisters/persistentmap_baseline/persistentmap_baseline.wasm", mo_config);
 // let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
 // let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
 // let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
@@ -77,40 +78,45 @@ function compare_rb_maps(init_size){
 
 sizes.map(compare_rb_maps);
 
-function perf_persistent_map(init) {
-  let cid = install(persistentmap, encode (), null);
+function perf_persistent_map(wasm, title, init) {
+  let cid = install(wasm, encode (), null);
 
-  output(file, stringify("|", init, "|"));
+  output(file, stringify("|", title, "|", init, "|"));
   call cid.__toggle_tracing();
   call cid.generate(init);
 
   call cid.__toggle_tracing();
   call cid.foldLeft();
-  let svg = stringify("persistentmap_foldLeft_", init, ".svg");
+  let svg = stringify(title, "_foldLeft_", init, ".svg");
   output(file, stringify("[", __cost__, "](", svg, ")|"));
   flamegraph(cid, "persistentmap.foldLeft", svg);
 
   call cid.foldRight();
-  let svg = stringify("persistentmap_foldRight_", init, ".svg");
+  let svg = stringify(title, "_foldRight_", init, ".svg");
   output(file, stringify("[", __cost__, "](", svg, ")|"));
   flamegraph(cid, "persistentmap.foldRight", svg);
 
   call cid.mapfilter();
-  let svg = stringify("persistentmap_mapfilter_", init, ".svg");
+  let svg = stringify(title, "_mapfilter_", init, ".svg");
   output(file, stringify("[", __cost__, "](", svg, ")|"));
   flamegraph(cid, "persistentmap.mapfilter", svg);
 
   call cid.map();
-  let svg = stringify("persistentmap_map_", init, ".svg");
+  let svg = stringify(title, "_map_", init, ".svg");
   output(file, stringify("[", __cost__, "](", svg, ")|\n"));
   flamegraph(cid, "persistentmap.map", svg);
 
   uninstall(cid);
 };
 
-output(file, stringify("\n## Persistent map API\n\n|size|foldLeft|foldRight|mapfilter|map|\n|--:|--:|--:|--:|--:|\n"));
+function compare_persistent_maps(init){
+  perf_persistent_map(persistentmap, "persistentmap", init);
+  perf_persistent_map(persistentmap_baseline, "persistentmap_baseline", init);
+};
+
+output(file, stringify("\n## Persistent map API\n\n| |size|foldLeft|foldRight|mapfilter|map|\n|--:|--:|--:|--:|--:|--:|\n"));
 
-sizes.map(perf_persistent_map);
+sizes.map(compare_persistent_maps);
 
 /*
 perf(hashmap, "hashmap", init_size, batch_size);