diff --git a/.github/workflows/diff.py b/.github/workflows/diff.py index c77d055a..f021d577 100644 --- a/.github/workflows/diff.py +++ b/.github/workflows/diff.py @@ -1,8 +1,15 @@ +import stats import sys import pandas as pd import markdown import re +if len(sys.argv) == 2 and sys.argv[1] == "final": + stats.load_stats() + print("# Overall Statistics\n") + stats.output_stats() + sys.exit(0) + if len(sys.argv) < 3: print("Usage: python diff.py [current.md] [main.md]") sys.exit(1) @@ -46,6 +53,8 @@ def read_tables(file): print(f"> **Warning**\n> Skip {file}. File not found.\n") sys.exit(0) + + current = read_tables(sys.argv[1]) main = read_tables(sys.argv[2]) @@ -53,6 +62,8 @@ def read_tables(file): print(f"> **Warning**\n> Skip {sys.argv[1]}, due to the number of tables mismatches from main branch.\n") sys.exit(0) +flaky_benchmarks = ["## Heartbeat"] + for i, ((header, current), (header2, main)) in enumerate(zip(current, main)): if header == header2 and current.shape == main.shape and all(current.columns == main.columns) and all(current.index == main.index): result = pd.DataFrame(index=current.index, columns=current.columns) @@ -72,8 +83,20 @@ def read_tables(file): result.loc[idx, col] = f"{x:_} ($\\textcolor{{red}}{{{d:.2f}\\\\%}}$)" else: result.loc[idx, col] = f"{x:_}" + if header in flaky_benchmarks: + continue + if col.endswith("binary_size"): + stats.data["binary_size"].append(d) + elif col.endswith("max mem"): + stats.data["max_mem"].append(d) + else: + stats.data["cycles"].append(d) print(result.to_markdown()) print(f"\n") else: print(f"> **Warning**\n> Skip table {i} {header} from {sys.argv[1]}, due to table shape mismatches from main branch.\n") +print(f"## Statistics\n\n") +stats.output_stats() + +stats.save_stats() diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index f2ee2a57..b2ddf3c4 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -14,14 +14,21 @@ jobs: DFX_VERSION: 0.14.3 IC_REPL_VERSION: 0.4.1 MOC_VERSION: 0.9.7 + IC_WASM_VERSION: 0.4.0 steps: - uses: actions/checkout@v3 - name: Checkout out gh-pages report - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'build_base') uses: actions/checkout@v3 with: ref: gh-pages path: main/_out + - name: Checkout out base branch + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build_base') + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + path: main/ - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -44,7 +51,7 @@ jobs: if: github.event_name == 'pull_request' run: | python -m pip install --upgrade pip - pip install pandas markdown lxml html5lib bs4 tabulate + pip install pandas markdown lxml html5lib bs4 tabulate scipy - uses: actions/setup-node@v3 with: node-version: 18 @@ -59,10 +66,19 @@ jobs: cd $(dfx cache show) wget https://github.com/dfinity/motoko/releases/download/$MOC_VERSION/motoko-linux64-$MOC_VERSION.tar.gz tar zxvf motoko-linux64-$MOC_VERSION.tar.gz - cargo install --git https://github.com/dfinity/ic-wasm.git + wget https://github.com/dfinity/ic-wasm/releases/download/$IC_WASM_VERSION/ic-wasm-linux64 + cp ./ic-wasm-linux64 /usr/local/bin/ic-wasm + chmod a+x /usr/local/bin/ic-wasm - name: Start dfx run: | dfx start --background + - name: Run perf for base branch + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build_base') + run: | + cd main + make + dfx stop + dfx start --clean --background - name: Run perf run: make - name: Generate table @@ -103,6 +119,7 @@ jobs: fi fi done + python .github/workflows/diff.py final >> DIFF.md - name: Read table if: github.event_name == 'pull_request' id: perf diff --git a/.github/workflows/stats.py b/.github/workflows/stats.py new file mode 100644 index 00000000..3674da46 --- /dev/null +++ b/.github/workflows/stats.py @@ -0,0 +1,41 @@ +import statistics +import pickle +from scipy.stats import t + +data = {'binary_size': [], 'max_mem': [], 'cycles': []} +file = "stats.pk" + +def stats(array, ignoreZeros=True): + if ignoreZeros: + array = [x for x in array if x != 0.] + if len(array) == 0: + return f"no change" + elif len(array) == 1: + return f"{array[0]:.2f}%" + mean = statistics.mean(array) + std = statistics.stdev(array) + conf_level = 0.9 + t_value = t.ppf(1 - (1 - conf_level) / 2, len(array) - 1) + interval = t_value * std / len(array)**0.5 + l, r = mean - interval, mean + interval + res = f"{mean:.2f}% [{l:.2f}%, {r:.2f}%]" + return res + +def output_stats(): + for name, array in data.items(): + print(f"* {name}: {stats(array)}") + +def save_stats(): + with open(file, 'ab') as fp: + pickle.dump(data, fp) + +def load_stats(): + with open(file, 'rb') as fr: + try: + while True: + d = pickle.load(fr) + for name, array in d.items(): + data[name].extend(array) + except EOFError: + pass + diff --git a/Cargo.toml b/Cargo.toml index 926d64b6..00169588 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ [profile.release] panic = "abort" lto = true -opt-level = 3 +opt-level = 2 [workspace.dependencies] ic-cdk = "0.10.0" diff --git a/collections/perf.sh b/collections/perf.sh index f6fe69f6..8690d717 100644 --- a/collections/perf.sh +++ b/collections/perf.sh @@ -56,7 +56,7 @@ perf(zhenya, "zhenya_hashmap", init_size); perf(btreemap_rs, "btreemap_rs", init_size); perf(hashmap_rs, "hashmap_rs", init_size); -output(file, "\n## Priority queue\n\n| |binary_size|heapify 50k|mem|pop_min 50|put 50|\n|--:|--:|--:|--:|--:|--:|\n"); +output(file, "\n## Priority queue\n\n| |binary_size|heapify 50k|max mem|pop_min 50|put 50|\n|--:|--:|--:|--:|--:|--:|\n"); perf(heap, "heap", init_size); perf(heap_rs, "heap_rs", init_size);