Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DMS-72] Implement benchmarks for PersistentOrderedMap #1

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[DMS-72] Implement benchmarks for PersistentOrderedMap #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
643266c
[DMS-72] Implement benchmarks for PersistentOrderedMap
s-and-witch Oct 7, 2024
2df6d2a
Improve benchmark infrastructure and add comparison with unoptimised …
s-and-witch Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion collections/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ motoko:
rust:
$(call build,rust)

build: motoko rust
build: motoko # rust

perf:
$(call perf,collections,perf.sh)
32 changes: 4 additions & 28 deletions collections/motoko/dfx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,40 +1,16 @@
{
"canisters": {
"hashmap": {
"type": "motoko",
"main": "src/hashmap.mo"
},
"triemap": {
"type": "motoko",
"main": "src/triemap.mo"
},
"rbtree": {
"type": "motoko",
"main": "src/rbtree.mo"
},
"splay": {
"type": "motoko",
"main": "src/splay.mo"
},
"heap": {
"type": "motoko",
"main": "src/heap.mo"
},
"btreemap": {
"type": "motoko",
"main": "src/btreemap.mo"
},
"zhenya_hashmap": {
"type": "motoko",
"main": "src/ZhenyaHashmap.mo"
},
"buffer": {
"persistentmap": {
"type": "motoko",
"main": "src/buffer.mo"
"main": "src/persistentmap.mo"
},
"vector": {
"persistentmap_baseline": {
"type": "motoko",
"main": "src/vector.mo"
"main": "src/persistentmap_baseline.mo"
}
},
"defaults": {
Expand Down
3 changes: 2 additions & 1 deletion collections/motoko/mops.template.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[dependencies]
base = "https://github.com/dfinity/motoko-base#moc-$MOC_VERSION"
base = "https://github.com/serokell/motoko-base#$TEST_BRANCH"
base-unopt = "https://github.com/serokell/motoko-base#$BASELINE_BRANCH"
stableheapbtreemap = "1.3.0"
map = "9.0.0"
splay = "0.1.0"
Expand Down
58 changes: 58 additions & 0 deletions collections/motoko/src/persistentmap.mo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import Map "mo:base/PersistentOrderedMap";
import Nat64 "mo:base/Nat64";
import Nat32 "mo:base/Nat32";
import Iter "mo:base/Iter";
import Option "mo:base/Option";
import Random "random";
import Profiling "../../../utils/motoko/Profiling";

actor {
stable let profiling = Profiling.init();

let mapOps = Map.MapOps<Nat64>(Nat64.compare);
stable var rbMap = Map.empty<Nat64, Nat64>();
let rand = Random.new(null, 42);

public func generate(size: Nat32) : async () {
let rand = Random.new(?size, 1);
let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
rbMap := mapOps.fromIter(iter);
};
public query func get_mem() : async (Nat,Nat,Nat) {
Random.get_memory()
};
public func batch_get(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func batch_put(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
let k = Option.get<Nat64>(rand.next(), 0);
rbMap := mapOps.put(rbMap, k, k);
}
};
public func batch_remove(n : Nat) : async () {
let rand = Random.new(null, 1);
for (_ in Iter.range(1, n)) {
rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func foldLeft() : async () {
ignore Map.foldLeft<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func foldRight() : async () {
ignore Map.foldRight<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func mapfilter() : async () {
ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
switch (key % 2) {
case 1 { null };
case _ { ?value }
};
})
};
public func map() : async () {
ignore Map.map<Nat64, Nat64, Nat64>(rbMap, func (key, value) {key + value})
};
}
58 changes: 58 additions & 0 deletions collections/motoko/src/persistentmap_baseline.mo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import Map "mo:base-unopt/PersistentOrderedMap";
import Nat64 "mo:base/Nat64";
import Nat32 "mo:base/Nat32";
import Iter "mo:base/Iter";
import Option "mo:base/Option";
import Random "random";
import Profiling "../../../utils/motoko/Profiling";

actor {
stable let profiling = Profiling.init();

let mapOps = Map.MapOps<Nat64>(Nat64.compare);
stable var rbMap = Map.empty<Nat64, Nat64>();
let rand = Random.new(null, 42);

public func generate(size: Nat32) : async () {
let rand = Random.new(?size, 1);
let iter = Iter.map<Nat64, (Nat64, Nat64)>(rand, func x = (x, x));
rbMap := mapOps.fromIter(iter);
};
public query func get_mem() : async (Nat,Nat,Nat) {
Random.get_memory()
};
public func batch_get(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
ignore mapOps.get(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func batch_put(n : Nat) : async () {
for (_ in Iter.range(1, n)) {
let k = Option.get<Nat64>(rand.next(), 0);
rbMap := mapOps.put(rbMap, k, k);
}
};
public func batch_remove(n : Nat) : async () {
let rand = Random.new(null, 1);
for (_ in Iter.range(1, n)) {
rbMap := mapOps.delete(rbMap, Option.get<Nat64>(rand.next(), 0));
}
};
public func foldLeft() : async () {
ignore Map.foldLeft<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func foldRight() : async () {
ignore Map.foldRight<Nat64, Nat64, Nat64>(rbMap, 0, func (x, y, acc) {x + y + acc});
};
public func mapfilter() : async () {
ignore mapOps.mapFilter<Nat64, Nat64>(rbMap, func (key, value) {
switch (key % 2) {
case 1 { null };
case _ { ?value }
};
})
};
public func map() : async () {
ignore Map.map<Nat64, Nat64, Nat64>(rbMap, func (key, value) {key + value})
};
}
10 changes: 9 additions & 1 deletion collections/motoko/tests/perf.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ let hashmap = install(wasm_profiling("../.dfx/local/canisters/hashmap/hashmap.wa
let triemap = install(wasm_profiling("../.dfx/local/canisters/triemap/triemap.wasm"), null, null);
let rbtree = install(wasm_profiling("../.dfx/local/canisters/rbtree/rbtree.wasm"), null, null);
let splay = install(wasm_profiling("../.dfx/local/canisters/splay/splay.wasm"), null, null);
let persistentmap = install(wasm_profiling("../.dfx/local/canisters/persistentmap/persistentmap.wasm"), null, null);

call hashmap.__toggle_tracing();
call triemap.__toggle_tracing();
call rbtree.__toggle_tracing();
call splay.__toggle_tracing();
call persistentmap.__toggle_tracing();
call hashmap.generate(50000);
let _ = get_memory(hashmap);
call triemap.generate(50000);
Expand All @@ -18,15 +20,19 @@ call rbtree.generate(50000);
let _ = get_memory(rbtree);
call splay.generate(50000);
let _ = get_memory(splay);
call persistentmap.generate(50000);
let _ = get_memory(persistentmap);

call hashmap.__toggle_tracing();
call triemap.__toggle_tracing();
call rbtree.__toggle_tracing();
call splay.__toggle_tracing();
call persistentmap.__toggle_tracing();
call hashmap.batch_get(50);
call triemap.batch_get(50);
call rbtree.batch_get(50);
call splay.batch_get(50);
call persistentmap.batch_get(50);

call hashmap.batch_put(50);
let _ = get_memory(hashmap);
Expand All @@ -36,9 +42,11 @@ call rbtree.batch_put(50);
let _ = get_memory(rbtree);
call splay.batch_put(50);
let _ = get_memory(splay);
call persistentmap.batch_put(50);
let _ = get_memory(persistentmap);

call hashmap.batch_remove(50);
call triemap.batch_remove(50);
call rbtree.batch_remove(50);
call splay.batch_remove(50);

call persistentmap.batch_remove(50);
95 changes: 73 additions & 22 deletions collections/perf.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,27 @@ load "../prelude.sh";

// use smaller page_limit to speed things up, since the whole trace is too large even with 256M.
let mo_config = record { start_page = 16; page_limit = 128 };
let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
// let hashmap = wasm_profiling("motoko/.dfx/local/canisters/hashmap/hashmap.wasm", mo_config);
// let triemap = wasm_profiling("motoko/.dfx/local/canisters/triemap/triemap.wasm", mo_config);
let rbtree = wasm_profiling("motoko/.dfx/local/canisters/rbtree/rbtree.wasm", mo_config);
let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);

let rs_config = record { start_page = 1; page_limit = 128 };
let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);
let persistentmap = wasm_profiling("motoko/.dfx/local/canisters/persistentmap/persistentmap.wasm", mo_config);
let persistentmap_baseline = wasm_profiling("motoko/.dfx/local/canisters/persistentmap_baseline/persistentmap_baseline.wasm", mo_config);
// let splay = wasm_profiling("motoko/.dfx/local/canisters/splay/splay.wasm", mo_config);
// let btree = wasm_profiling("motoko/.dfx/local/canisters/btreemap/btreemap.wasm", mo_config);
// let zhenya = wasm_profiling("motoko/.dfx/local/canisters/zhenya_hashmap/zhenya_hashmap.wasm", mo_config);
// let heap = wasm_profiling("motoko/.dfx/local/canisters/heap/heap.wasm", mo_config);
// let buffer = wasm_profiling("motoko/.dfx/local/canisters/buffer/buffer.wasm", mo_config);
// let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", mo_config);

// let rs_config = record { start_page = 1; page_limit = 128 };
// let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", rs_config);
// let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", rs_config);
// let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", rs_config);
// let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", rs_config);
// let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", rs_config);
// let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", rs_config);
// let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", rs_config);
// let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", rs_config);

//let movm_rs = wasm_profiling("rust/.dfx/local/canisters/movm/movm.wasm");
//let movm_dynamic_rs = wasm_profiling("rust/.dfx/local/canisters/movm_dynamic/movm_dynamic.wasm");
Expand Down Expand Up @@ -63,12 +65,62 @@ function perf(wasm, title, init, batch) {
uninstall(cid);
};

let init_size = 1_000_000;

let batch_size = 50;
output(file, "\n## Map\n\n| |binary_size|generate 1m|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
let sizes = vec {100; 1000; 10_000; 100_000; 1_000_000};

output(file, stringify("\n## Map comparison\n\n| |binary_size|generate|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n"));

function compare_rb_maps(init_size){
perf(rbtree, stringify("rbtree+", init_size), init_size, batch_size);
perf(persistentmap, stringify("persistentmap+", init_size), init_size, batch_size);
};

sizes.map(compare_rb_maps);

function perf_persistent_map(wasm, title, init) {
let cid = install(wasm, encode (), null);

output(file, stringify("|", title, "|", init, "|"));
call cid.__toggle_tracing();
call cid.generate(init);

call cid.__toggle_tracing();
call cid.foldLeft();
let svg = stringify(title, "_foldLeft_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.foldLeft", svg);

call cid.foldRight();
let svg = stringify(title, "_foldRight_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.foldRight", svg);

call cid.mapfilter();
let svg = stringify(title, "_mapfilter_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|"));
flamegraph(cid, "persistentmap.mapfilter", svg);

call cid.map();
let svg = stringify(title, "_map_", init, ".svg");
output(file, stringify("[", __cost__, "](", svg, ")|\n"));
flamegraph(cid, "persistentmap.map", svg);

uninstall(cid);
};

function compare_persistent_maps(init){
perf_persistent_map(persistentmap, "persistentmap", init);
perf_persistent_map(persistentmap_baseline, "persistentmap_baseline", init);
};

output(file, stringify("\n## Persistent map API\n\n| |size|foldLeft|foldRight|mapfilter|map|\n|--:|--:|--:|--:|--:|--:|\n"));

sizes.map(compare_persistent_maps);

/*
perf(hashmap, "hashmap", init_size, batch_size);
perf(triemap, "triemap", init_size, batch_size);
perf(rbtree, "rbtree", init_size, batch_size);
perf(splay, "splay", init_size, batch_size);
perf(btree, "btree", init_size, batch_size);
perf(zhenya, "zhenya_hashmap", init_size, batch_size);
Expand Down Expand Up @@ -97,7 +149,6 @@ perf(heap_stable_rs, "heap_stable_rs", init_size, batch_size);
perf(vector_rs, "vec_rs", init_size, batch_size);
perf(vector_stable_rs, "vec_stable_rs", init_size, batch_size);

/*
let movm_size = 10000;
output(file, "\n## MoVM\n\n| |binary_size|generate 10k|max mem|batch_get 50|batch_put 50|batch_remove 50|\n|--:|--:|--:|--:|--:|--:|--:|\n");
perf(hashmap, "hashmap", movm_size);
Expand Down
Loading