Skip to content

Commit

Permalink
[DMS-72] Implement benchmarks for PersistentOrderedMap
Browse files Browse the repository at this point in the history
This is just an experiment and isn't suitable for
merging to the upstream in the current state.

Modifications in collections/ folder compare
PersistentOrderedMap with RBTree. Second test show
performance of functions from PersistentOrderedMap,
that have no analogue in RBTree module.
  • Loading branch information
s-and-witch committed Oct 7, 2024
1 parent e87dd0d commit 643266c
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 55 deletions.
2 changes: 1 addition & 1 deletion collections/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ motoko:
rust:
$(call build,rust)

build: motoko rust
build: motoko # rust

perf:
$(call perf,collections,perf.sh)
32 changes: 2 additions & 30 deletions collections/motoko/dfx.json
Original file line number Diff line number Diff line change
@@ -1,40 +1,12 @@
{
"canisters": {
"hashmap": {
"type": "motoko",
"main": "src/hashmap.mo"
},
"triemap": {
"type": "motoko",
"main": "src/triemap.mo"
},
"rbtree": {
"type": "motoko",
"main": "src/rbtree.mo"
},
"splay": {
"type": "motoko",
"main": "src/splay.mo"
},
"heap": {
"type": "motoko",
"main": "src/heap.mo"
},
"btreemap": {
"type": "motoko",
"main": "src/btreemap.mo"
},
"zhenya_hashmap": {
"type": "motoko",
"main": "src/ZhenyaHashmap.mo"
},
"buffer": {
"type": "motoko",
"main": "src/buffer.mo"
},
"vector": {
"persistentmap": {
"type": "motoko",
"main": "src/vector.mo"
"main": "src/persistentmap.mo"
}
},
"defaults": {
Expand Down
2 changes: 1 addition & 1 deletion collections/motoko/mops.template.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[dependencies]
base = "https://github.com/dfinity/motoko-base#moc-$MOC_VERSION"
base = "https://github.com/serokell/motoko-base#milestone-1"
stableheapbtreemap = "1.3.0"
map = "9.0.0"
splay = "0.1.0"
Expand Down
58 changes: 58 additions & 0 deletions collections/motoko/src/persistentmap.mo
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import Map "mo:base/PersistentOrderedMap";
import Nat64 "mo:base/Nat64";
import Nat32 "mo:base/Nat32";
import Iter "mo:base/Iter";
import Option "mo:base/Option";
import Random "random";
import Profiling "../../../utils/motoko/Profiling";

actor {
stable let profiling = Profiling.init();

let mapOps = Map.MapOps<Nat64>(Nat64.compare);
stable var rbMap = Map.empty<Nat64, Nat64>();
let rand = Random.new(null, 42);

public func generate(size: Nat32) : async () {
let rand = Random.new(?size, 1);
let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
rbMap := mapOps.fromIter(iter);
};
public query func get_mem() : async (Nat,Nat,Nat) {
Random.get_memory()
};
public func batch_get(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func batch_put(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
let k = Option.get<Nat64>(rand.next(), 0);
rbMap := mapOps.put(rbMap, k, k);
}
};
public func batch_remove(n : Nat) : async () {
let rand = Random.new(null, 1);
for (_ in Iter.range(1, n)) {
rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func foldLeft() : async () {
ignore Map.foldLeft<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func foldRight() : async () {
ignore Map.foldRight<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func mapfilter() : async () {
ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
switch (key % 2) {
case 1 { null };
case _ { ?value }
};
})
};
public func map() : async () {
ignore Map.map<Nat64, Nat64, Nat64>(rbMap, func (key, value) {key + value})
};
}
10 changes: 9 additions & 1 deletion collections/motoko/tests/perf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ let hashmap = install(wasm_profiling("../.dfx/local/canisters/hashmap/hashmap.wa
let triemap = install(wasm_profiling("../.dfx/local/canisters/triemap/triemap.wasm"), null, null);
let rbtree = install(wasm_profiling("../.dfx/local/canisters/rbtree/rbtree.wasm"), null, null);
let splay = install(wasm_profiling("../.dfx/local/canisters/splay/splay.wasm"), null, null);
let persistentmap = install(wasm_profiling("../.dfx/local/canisters/persistentmap/persistentmap.wasm"), null, null);

call hashmap.__toggle_tracing();
call triemap.__toggle_tracing();
call rbtree.__toggle_tracing();
call splay.__toggle_tracing();
call persistentmap.__toggle_tracing();
call hashmap.generate(50000);
let _ = get_memory(hashmap);
call triemap.generate(50000);
Expand All @@ -18,15 +20,19 @@ call rbtree.generate(50000);
let _ = get_memory(rbtree);
call splay.generate(50000);
let _ = get_memory(splay);
call persistentmap.generate(50000);
let _ = get_memory(persistentmap);

call hashmap.__toggle_tracing();
call triemap.__toggle_tracing();
call rbtree.__toggle_tracing();
call splay.__toggle_tracing();
call persistentmap.__toggle_tracing();
call hashmap.batch_get(50);
call triemap.batch_get(50);
call rbtree.batch_get(50);
call splay.batch_get(50);
call persistentmap.batch_get(50);

call hashmap.batch_put(50);
let _ = get_memory(hashmap);
Expand All @@ -36,9 +42,11 @@ call rbtree.batch_put(50);
let _ = get_memory(rbtree);
call splay.batch_put(50);
let _ = get_memory(splay);
call persistentmap.batch_put(50);
let _ = get_memory(persistentmap);

call hashmap.batch_remove(50);
call triemap.batch_remove(50);
call rbtree.batch_remove(50);
call splay.batch_remove(50);

call persistentmap.batch_remove(50);
89 changes: 67 additions & 22 deletions collections/perf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,26 @@ load "../prelude.sh";

// use smaller page_limit to speed things up, since the whole trace is too large even with 256M.
let mo_config = record { start_page = 16; page_limit = 128 };
let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
// let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
// let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
let rbtree = wasm_profiling("motoko/.dfx/local/canisters/rbtree/rbtree.wasm", mo_config);
let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);

let rs_config = record { start_page = 1; page_limit = 128 };
let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);
let persistentmap = wasm_profiling("motoko/.dfx/local/canisters/persistentmap/persistentmap.wasm", mo_config);
// let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
// let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
// let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
// let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
// let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
// let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);

// let rs_config = record { start_page = 1; page_limit = 128 };
// let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
// let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
// let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
// let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
// let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
// let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
// let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
// let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);

//let movm_rs = wasm_profiling("rust/.dfx/local/canisters/movm/movm.wasm");
//let movm_dynamic_rs = wasm_profiling("rust/.dfx/local/canisters/movm_dynamic/movm_dynamic.wasm");
Expand Down Expand Up @@ -63,12 +64,57 @@ function perf(wasm, title, init, batch) {
uninstall(cid);
};

let init_size = 1_000_000;

let batch_size = 50;
output(file, "\n## Map\n\n| |binary_size|generate 1m|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
let sizes = vec {100; 1000; 10_000; 100_000; 1_000_000};

output(file, stringify("\n## Map comparison\n\n| |binary_size|generate|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n"));

function compare_rb_maps(init_size){
perf(rbtree, stringify("rbtree+", init_size), init_size, batch_size);
perf(persistentmap, stringify("persistentmap+", init_size), init_size, batch_size);
};

sizes.map(compare_rb_maps);

function perf_persistent_map(init) {
let cid = install(persistentmap, encode (), null);

output(file, stringify("|", init, "|"));
call cid.__toggle_tracing();
call cid.generate(init);

call cid.__toggle_tracing();
call cid.foldLeft();
let svg = stringify("persistentmap_foldLeft_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.foldLeft", svg);

call cid.foldRight();
let svg = stringify("persistentmap_foldRight_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.foldRight", svg);

call cid.mapfilter();
let svg = stringify("persistentmap_mapfilter_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.mapfilter", svg);

call cid.map();
let svg = stringify("persistentmap_map_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|\n"));
flamegraph(cid, "persistentmap.map", svg);

uninstall(cid);
};

output(file, stringify("\n## Persistent map API\n\n|size|foldLeft|foldRight|mapfilter|map|\n|--:|--:|--:|--:|--:|\n"));

sizes.map(perf_persistent_map);

/*
perf(hashmap, "hashmap", init_size, batch_size);
perf(triemap, "triemap", init_size, batch_size);
perf(rbtree, "rbtree", init_size, batch_size);
perf(splay, "splay", init_size, batch_size);
perf(btree, "btree", init_size, batch_size);
perf(zhenya, "zhenya_hashmap", init_size, batch_size);
Expand Down Expand Up @@ -97,7 +143,6 @@ perf(heap_stable_rs, "heap_stable_rs", init_size, batch_size);
perf(vector_rs, "vec_rs", init_size, batch_size);
perf(vector_stable_rs, "vec_stable_rs", init_size, batch_size);

/*
let movm_size = 10000;
output(file, "\n## MoVM\n\n| |binary_size|generate 10k|max mem|batch_get 50|batch_put 50|batch_remove 50|\n|--:|--:|--:|--:|--:|--:|--:|\n");
perf(hashmap, "hashmap", movm_size);
Expand Down

0 comments on commit 643266c

Please sign in to comment.