From 41b22efd8289976ac5b70cbdc118a84ea3619de6 Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 11:57:30 +0200 Subject: [PATCH 1/8] Benchmark with HPy 0.0.3 and PyPy3 7.3.6 --- README.md | 63 +++++++++++++++++------------------ bench/bench.jl | 4 +-- bench/bench_cpy_vs_hpy.py | 57 ++++++++++++++++++++++++------- bench/make_bench_piconumpy.py | 9 ++--- 4 files changed, 83 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index a0bad3d..6b2a908 100644 --- a/README.md +++ b/README.md @@ -104,19 +104,13 @@ pypy -m pip install pip -U pypy -m pip install numpy cython pytest transonic pythran ``` -We need to install the correct version of HPy for the version of PyPy we are using: +One can check which HPy version is vendored with PyPy: ```bash pypy -c "import hpy.universal as u; print(u.get_version())" ``` -gives `('0.0.2rc2.dev12+gc9660c2', 'c9660c2')`. - -```bash -cd ~/Dev/hpy -# update to the correct commit -pypy setup.py develop -``` +gives `('0.0.3', '2196f14')`. Now we can build-install PicoNumpy: @@ -136,36 +130,36 @@ make ## Few results -As of today (6 July 2021), HPy is not yet ready for high performance, but at -least (with HPy 0.0.2) it runs ! +As of today (12 October 2021), HPy is not yet ready for high performance, but at +least (with HPy 0.0.3) it runs ! ### At home (Intel(R) Core(TM) i5-8400 CPU @ 2.80GHz) - With CPython ``` -Julia : 1 * norm = 0.00196 s -PicoNumpy (CPython C-API) : 9.42 * norm -PicoNumpy (HPy CPy ABI) : 9.95 * norm -PicoNumpy (HPy Universal) : 10.4 * norm -Transonic-Pythran : 0.497 * norm -Numpy : 27.5 * norm -PicoNumpy (purepy) : 37.3 * norm -PicoNumpy (purepy_array) : 37.7 * norm -PicoNumpy (Cython) : 28.9 * norm +Julia : 1 * norm = 0.0171 s +PicoNumpy (CPython C-API) : 11.1 * norm +PicoNumpy (HPy CPy ABI) : 11.6 * norm +PicoNumpy (HPy Universal) : 12.1 * norm +Transonic-Pythran : 0.537 * norm +Numpy : 33.8 * norm +PicoNumpy (purepy) : 43.7 * norm +PicoNumpy (purepy_array) : 44.8 * norm +PicoNumpy (Cython) : 33.9 * norm ``` - With PyPy3 ``` -Julia : 1 * norm = 0.00196 s -PicoNumpy (CPython C-API) : 34.1 * norm -PicoNumpy (HPy Universal) : 12.8 * norm -Transonic-Pythran : 0.539 * norm -Numpy : 232 * norm -PicoNumpy (purepy) : 4.39 * norm -PicoNumpy (purepy_array) : 6.33 * norm -PicoNumpy (Cython) : 274 * norm +Julia : 1 * norm = 0.0171 s +PicoNumpy (CPython C-API) : 39.2 * norm +PicoNumpy (HPy Universal) : 13.1 * norm +Transonic-Pythran : 0.562 * norm +Numpy : 286 * norm +PicoNumpy (purepy) : 5.59 * norm +PicoNumpy (purepy_array) : 7.41 * norm +PicoNumpy (Cython) : 282 * norm ``` #### Simpler benchmarks (bench/bench_cpy_vs_hpy.py) @@ -173,14 +167,19 @@ PicoNumpy (Cython) : 274 * norm - With CPython ``` -CPython C-API: 1.92 seconds -HPy [Universal]: 2.08 seconds -HPy [CPy ABI]: 2.02 seconds +{'cache_tag': 'cpython-39', + 'version': sys.version_info(major=3, minor=9, micro=6, releaselevel='final', serial=0)} +CPython C-API: 0.193 seconds (11.2 * Julia) +HPy [Universal]: 0.208 seconds (12.1 * Julia) +HPy [CPy ABI]: 0.201 seconds (11.7 * Julia) ``` - With PyPy3 ``` -CPython C-API: 5.75 seconds -HPy [Universal]: 2.11 seconds +{'cache_tag': 'pypy37', + 'version': sys.pypy_version_info(major=7, minor=3, micro=6, releaselevel='final', serial=0)} +CPython C-API: 0.592 seconds (34.6 * Julia) +HPy [Universal]: 0.207 seconds (12.1 * Julia) +Python list: 0.093 seconds ( 5.4 * Julia) ``` diff --git a/bench/bench.jl b/bench/bench.jl index 00cedff..c9d08ef 100644 --- a/bench/bench.jl +++ b/bench/bench.jl @@ -65,10 +65,10 @@ function bench(n_sleds, n_time) end -n_sleds = 10 +n_sleds = 100 n_time = 200 -nb_runs = 200 +nb_runs = 50 times = zeros(nb_runs) diff --git a/bench/bench_cpy_vs_hpy.py b/bench/bench_cpy_vs_hpy.py index e54ad54..c97f3d8 100644 --- a/bench/bench_cpy_vs_hpy.py +++ b/bench/bench_cpy_vs_hpy.py @@ -1,8 +1,9 @@ import sys -import time +from time import perf_counter import random from math import pi, cos, sin from pathlib import Path +from pprint import pprint here = Path(__file__).absolute().parent @@ -75,14 +76,18 @@ def bench(mod, n_sleds, n_time): u_init = mod.zeros(n_sleds) for i in range(n_sleds): u_init[i] += 3.5 - start = time.time() - solver(mod, board, x_init, y_init, u_init, v_init, 0.01, n_time) - end = time.time() - return end - start + times = [] + for _ in range(20): + start = perf_counter() + solver(mod, board, x_init, y_init, u_init, v_init, 0.01, n_time) + times.append(perf_counter() - start) + + times.sort() + return times[len(times) // 2] N_SLEDS = 100 -N_TIME = 2000 +N_TIME = 200 def import_piconumpy_hpy_universal(): @@ -97,18 +102,46 @@ def main(): import piconumpy._piconumpy_cpython_capi as pnp_capi - t = bench(pnp_capi, N_SLEDS, N_TIME) - print(f"CPython C-API: {t:.2f} seconds") + pprint({key: sys.implementation.__dict__[key] for key in ("cache_tag", "version")}) + + tmp_result_julia = Path("tmp_result_julia.txt") + if tmp_result_julia.exists(): + with open("tmp_result_julia.txt") as file: + norm = float(file.read()) + end = "" + else: + norm = False + end = "\n" + + t_capi = bench(pnp_capi, N_SLEDS, N_TIME) + print(f"CPython C-API: {t_capi:.3f} seconds", end=end) + if norm: + print(f" ({t_capi/norm:.1f} * Julia)") pnp_hpy_universal = import_piconumpy_hpy_universal() - t = bench(pnp_hpy_universal, N_SLEDS, N_TIME) - print(f"HPy [Universal]: {t:.2f} seconds") + t_hpy_univ = bench(pnp_hpy_universal, N_SLEDS, N_TIME) + print(f"HPy [Universal]: {t_hpy_univ:.3f} seconds", end=end) + + if norm: + print(f" ({t_hpy_univ/norm:.1f} * Julia)") if not IS_PYPY: import piconumpy._piconumpy_hpy as pnp_hpy - t = bench(pnp_hpy, N_SLEDS, N_TIME) - print(f"HPy [CPy ABI]: {t:.2f} seconds") + t_hpy_cpy_abi = bench(pnp_hpy, N_SLEDS, N_TIME) + print(f"HPy [CPy ABI]: {t_hpy_cpy_abi:.3f} seconds", end=end) + + if norm: + print(f" ({t_hpy_cpy_abi/norm:.1f} * Julia)") + + if IS_PYPY: + import piconumpy.purepy as pnp_with_list + + t_with_list = bench(pnp_with_list, N_SLEDS, N_TIME) + print(f"Python list: {t_with_list:.3f} seconds", end=end) + + if norm: + print(f" ({t_with_list/norm:4.1f} * Julia)") if __name__ == "__main__": diff --git a/bench/make_bench_piconumpy.py b/bench/make_bench_piconumpy.py index c15b3f6..c1a92d0 100644 --- a/bench/make_bench_piconumpy.py +++ b/bench/make_bench_piconumpy.py @@ -75,12 +75,12 @@ def create_tmp_file(name_module): name = fmt_name.format("Julia") print(f"{name}: 1 * norm = {norm:4.3g} s") -n_sleds = 10 +n_sleds = 100 n_time = 200 g = locals() -def timeit(name_func, name): +def timeit(name_func, name, total_duration=2): return timeit_verbose( name_func + "(n_sleds, n_time)", globals=g, @@ -88,6 +88,7 @@ def timeit(name_func, name): print_time=False, norm=norm, max_length_name=max_length_name, + total_duration=total_duration, ) timeit("bench", name="PicoNumpy (CPython C-API)") @@ -95,14 +96,14 @@ def timeit(name_func, name): timeit("bench_hpy", name="PicoNumpy (HPy CPy ABI)") timeit("bench_hpy_universal", name="PicoNumpy (HPy Universal)") timeit("bench_pythran", name="Transonic-Pythran") -timeit("bench_numpy", name="Numpy") +timeit("bench_numpy", name="Numpy", total_duration=4) timeit( "bench_piconumpy_purepy", name="PicoNumpy (purepy)", ) timeit( "bench_piconumpy_purepy_array", name="PicoNumpy (purepy_array)", ) -timeit("bench_cython", name="PicoNumpy (Cython)") +timeit("bench_cython", name="PicoNumpy (Cython)", total_duration=4) """ ) From ba809f3dc55a249537a275013418f5c74d72b31c Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 12:07:44 +0200 Subject: [PATCH 2/8] CI: use HPy 0.0.3 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8083adf..3385e2c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,9 +24,8 @@ jobs: - name: Install dependencies run: | - git clone -b master --single-branch https://github.com/hpyproject/hpy + git clone -b release/0.0.3 --single-branch https://github.com/hpyproject/hpy cd hpy - git checkout 7b45ce522 pip install . pip install numpy cython pytest transonic pythran @@ -47,5 +46,6 @@ jobs: - name: Run bench run: | cd bench + make tmp_result_julia.txt make bench_hpy make From 9e5aae731f104ab9c53a379649452a6e58cb7ed4 Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 13:56:15 +0200 Subject: [PATCH 3/8] CI: pypy-3.7-nightly --- .github/workflows/tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3385e2c..7e1601d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, pypy-3.7-nightly] steps: @@ -22,11 +22,15 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - if: startsWith(matrix.python-version, 'pypy') != true + name: Install HPy (only for CPython) run: | git clone -b release/0.0.3 --single-branch https://github.com/hpyproject/hpy cd hpy pip install . + + - name: Install dependencies + run: | pip install numpy cython pytest transonic pythran - name: Checkout From d3025c54652a73ce519a0db7921ca2dc15899399 Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 14:49:33 +0200 Subject: [PATCH 4/8] xfail 2 tests for PyPy --- piconumpy/test_cpython_capi.py | 1 + piconumpy/test_hpy_universal.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/piconumpy/test_cpython_capi.py b/piconumpy/test_cpython_capi.py index a1638dc..cedbed5 100644 --- a/piconumpy/test_cpython_capi.py +++ b/piconumpy/test_cpython_capi.py @@ -6,6 +6,7 @@ class Tests: piconumpy = _piconumpy_cpython_capi + def _array(self, *args): return self.piconumpy.array(*args) diff --git a/piconumpy/test_hpy_universal.py b/piconumpy/test_hpy_universal.py index fbf5ce4..2a470ca 100644 --- a/piconumpy/test_hpy_universal.py +++ b/piconumpy/test_hpy_universal.py @@ -1,3 +1,5 @@ +import sys + import pytest from .util_hpy import import_ext @@ -15,3 +17,15 @@ ) class TestsCPyABI(_Tests): piconumpy = piconumpy_universal + + def test_multiply(self): + if sys.implementation.name == "pypy": + pytest.xfail("Expected failure with PyPy (but should work)") + + super().test_multiply() + + def test_add(self): + if sys.implementation.name == "pypy": + pytest.xfail("Expected failure with PyPy (but should work)") + + super().test_add() From 0b809bc2f51b3063782260851892aba46e1d97f3 Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 15:04:20 +0200 Subject: [PATCH 5/8] rm piconumpy/_piconumpy_hpy.py --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e1601d..d989a0c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,10 +42,11 @@ jobs: run: | python setup.py develop python setup.py --hpy-abi=universal develop + rm -f piconumpy/_piconumpy_hpy.py - name: Run tests run: | - pytest -s + pytest -v - name: Run bench run: | From f788f1a04d01993c982681f58953ef9fad80219f Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 15:22:11 +0200 Subject: [PATCH 6/8] Skip too long benchmarks --- bench/make_bench_piconumpy.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bench/make_bench_piconumpy.py b/bench/make_bench_piconumpy.py index c1a92d0..b02433f 100644 --- a/bench/make_bench_piconumpy.py +++ b/bench/make_bench_piconumpy.py @@ -96,14 +96,20 @@ def timeit(name_func, name, total_duration=2): timeit("bench_hpy", name="PicoNumpy (HPy CPy ABI)") timeit("bench_hpy_universal", name="PicoNumpy (HPy Universal)") timeit("bench_pythran", name="Transonic-Pythran") -timeit("bench_numpy", name="Numpy", total_duration=4) +try: + timeit("bench_numpy", name="Numpy", total_duration=8) +except RuntimeError: + print("Skip bench_numpy because it's too slow") timeit( "bench_piconumpy_purepy", name="PicoNumpy (purepy)", ) timeit( "bench_piconumpy_purepy_array", name="PicoNumpy (purepy_array)", ) -timeit("bench_cython", name="PicoNumpy (Cython)", total_duration=4) +try: + timeit("bench_cython", name="PicoNumpy (Cython)", total_duration=8) +except RuntimeError: + print("Skip bench_cython because it's too slow") """ ) From 6f1d5119f79675f5b81d261721a4a9c2b18ee9e2 Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 15:25:49 +0200 Subject: [PATCH 7/8] rerun bench_hpy --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d989a0c..eff1152 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,3 +54,5 @@ jobs: make tmp_result_julia.txt make bench_hpy make + # let's rerun bench_hpy to get these results also at the end + make bench_hpy From aa6f146860e02aa8a4a4ddb89ac4d5e0c7a78c1c Mon Sep 17 00:00:00 2001 From: paugier Date: Tue, 12 Oct 2021 16:03:28 +0200 Subject: [PATCH 8/8] Small improvements --- bench/bench.jl | 2 +- bench/bench_cpy_vs_hpy.py | 7 ++++--- bench/make_bench_piconumpy.py | 3 +++ bench/profile_piconumpy.py | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bench/bench.jl b/bench/bench.jl index c9d08ef..bd98571 100644 --- a/bench/bench.jl +++ b/bench/bench.jl @@ -68,7 +68,7 @@ end n_sleds = 100 n_time = 200 -nb_runs = 50 +nb_runs = 200 times = zeros(nb_runs) diff --git a/bench/bench_cpy_vs_hpy.py b/bench/bench_cpy_vs_hpy.py index c97f3d8..38f1ba1 100644 --- a/bench/bench_cpy_vs_hpy.py +++ b/bench/bench_cpy_vs_hpy.py @@ -109,6 +109,7 @@ def main(): with open("tmp_result_julia.txt") as file: norm = float(file.read()) end = "" + print(f"Julia: {norm:.3f} seconds") else: norm = False end = "\n" @@ -116,14 +117,14 @@ def main(): t_capi = bench(pnp_capi, N_SLEDS, N_TIME) print(f"CPython C-API: {t_capi:.3f} seconds", end=end) if norm: - print(f" ({t_capi/norm:.1f} * Julia)") + print(f" ({t_capi/norm:4.1f} * Julia)") pnp_hpy_universal = import_piconumpy_hpy_universal() t_hpy_univ = bench(pnp_hpy_universal, N_SLEDS, N_TIME) print(f"HPy [Universal]: {t_hpy_univ:.3f} seconds", end=end) if norm: - print(f" ({t_hpy_univ/norm:.1f} * Julia)") + print(f" ({t_hpy_univ/norm:4.1f} * Julia)") if not IS_PYPY: import piconumpy._piconumpy_hpy as pnp_hpy @@ -132,7 +133,7 @@ def main(): print(f"HPy [CPy ABI]: {t_hpy_cpy_abi:.3f} seconds", end=end) if norm: - print(f" ({t_hpy_cpy_abi/norm:.1f} * Julia)") + print(f" ({t_hpy_cpy_abi/norm:4.1f} * Julia)") if IS_PYPY: import piconumpy.purepy as pnp_with_list diff --git a/bench/make_bench_piconumpy.py b/bench/make_bench_piconumpy.py index b02433f..4a76e9a 100644 --- a/bench/make_bench_piconumpy.py +++ b/bench/make_bench_piconumpy.py @@ -47,6 +47,7 @@ def create_tmp_file(name_module): import numpy as np from piconumpy import array from math import pi, cos, sin +from pprint import pprint IS_PYPY = hasattr(sys, 'pypy_version_info') """ @@ -65,6 +66,8 @@ def create_tmp_file(name_module): if not IS_PYPY: from tmp_hpy import bench as bench_hpy +pprint({key: sys.implementation.__dict__[key] for key in ("cache_tag", "version")}) + # get norm from Julia benchmark with open("tmp_result_julia.txt") as file: norm = float(file.read()) diff --git a/bench/profile_piconumpy.py b/bench/profile_piconumpy.py index b7de388..3bde5ae 100644 --- a/bench/profile_piconumpy.py +++ b/bench/profile_piconumpy.py @@ -7,12 +7,14 @@ import tmp_purepy import tmp_purepy_array import tmp_cython +import tmp_hpy_universal methods = { "cpython-c-api": bench_array1d, "purepy": tmp_purepy, "purepy_array": tmp_purepy_array, "cython": tmp_cython, + "universal": tmp_hpy_universal, } module = methods.get(sys.argv[-1], bench_array1d)