From f10fc267cae53e569547993866f84278b4ca5134 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Jun 2022 06:30:17 -0400 Subject: [PATCH 01/20] Until PyO3 0.17 is released, 3.11 support needs to run off of git. --- Cargo.lock | 9 +++------ filpreload/Cargo.toml | 2 +- memapi/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65c90fce..83fcae31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,8 +717,7 @@ dependencies = [ [[package]] name = "pyo3" version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6302e85060011447471887705bb7838f14aba43fcb06957d823739a496b3dc" +source = "git+https://github.com/pyo3/pyo3#9dfeaa38d4a3b7e0b7cf77b61ab3f58e489a0277" dependencies = [ "cfg-if", "libc", @@ -730,8 +729,7 @@ dependencies = [ [[package]] name = "pyo3-build-config" version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b65b546c35d8a3b1b2f0ddbac7c6a569d759f357f2b9df884f5d6b719152c8" +source = "git+https://github.com/pyo3/pyo3#9dfeaa38d4a3b7e0b7cf77b61ab3f58e489a0277" dependencies = [ "once_cell", "target-lexicon", @@ -740,8 +738,7 @@ dependencies = [ [[package]] name = "pyo3-ffi" version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c275a07127c1aca33031a563e384ffdd485aee34ef131116fcd58e3430d1742b" +source = "git+https://github.com/pyo3/pyo3#9dfeaa38d4a3b7e0b7cf77b61ab3f58e489a0277" dependencies = [ "libc", "pyo3-build-config", diff --git a/filpreload/Cargo.toml b/filpreload/Cargo.toml index fa8b6b5d..5180a84b 100644 --- a/filpreload/Cargo.toml +++ b/filpreload/Cargo.toml @@ -18,7 +18,7 @@ path = "../memapi" features = [] [dependencies.pyo3] -version = "0.16" +git = "https://github.com/pyo3/pyo3" default-features = false [build-dependencies] diff --git a/memapi/Cargo.toml b/memapi/Cargo.toml index cadd298d..6f9c1565 100644 --- a/memapi/Cargo.toml +++ b/memapi/Cargo.toml @@ -28,7 +28,7 @@ default-features = false features = ["memory", "process"] [dependencies.pyo3] -version = "0.16" +git = "https://github.com/pyo3/pyo3" default-features = false features = [] From 524fd333c97310c032a0f0fea103aedee988b581 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 09:57:19 -0400 Subject: [PATCH 02/20] 3.11 build infrastructure. --- .github/workflows/main.yml | 4 ++-- wheels/build-wheels.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b3fc08ed..91308a6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: name: "${{ matrix.os }}: Python ${{ matrix.python-version }}" strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] os: ["ubuntu-20.04", "macos-10.15"] runs-on: "${{ matrix.os }}" @@ -80,7 +80,7 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "10.15" CIBW_ARCHS_MACOS: "x86_64 arm64" - CIBW_SKIP: "cp37-macosx_arm64 cp36* pp* cp311*" + CIBW_SKIP: "cp37-macosx_arm64 cp36* pp*" CIBW_BEFORE_BUILD: "touch filpreload/src/_filpreload.c" # force rebuild of Python code with new interpreter CIBW_TEST_COMMAND: python -m filprofiler run {project}/benchmarks/pystone.py with: diff --git a/wheels/build-wheels.sh b/wheels/build-wheels.sh index 667c3f43..11f4e859 100755 --- a/wheels/build-wheels.sh +++ b/wheels/build-wheels.sh @@ -17,7 +17,7 @@ rm -f filprofiler/_filpreload*.so rm -f filprofiler/_filpreload*.dylib rm -rf build -for PYBIN in /opt/python/cp{37,38,39,310}*/bin; do +for PYBIN in /opt/python/cp{37,38,39,310,311}*/bin; do touch filpreload/src/_filpreload.c # force rebuild of Python code with new interpreter export PYO3_PYTHON="$PYBIN/python" "${PYBIN}/pip" install -U setuptools wheel setuptools-rust pip @@ -28,4 +28,5 @@ auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp38*whl auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp39*whl auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp310*whl +auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp311*whl From e4bfb17f3358d66c39d2e352cbe1f9a2b599d75d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 10:25:15 -0400 Subject: [PATCH 03/20] Compile on 3.11. --- filpreload/src/_filpreload.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index 000b3751..d5b4bef0 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -1,10 +1,16 @@ #include "Python.h" +#if PY_MINOR_VERSION < 11 #include "code.h" +#else +#include "internal/pycore_code.h" +#include "internal/pycore_frame.h" +#endif +#include "frameobject.h" #include "object.h" + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif -#include "frameobject.h" #include #include #include @@ -237,23 +243,24 @@ fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { */ uint64_t function_id = 0; assert(extra_code_index != -1); - _PyCode_GetExtra((PyObject *)frame->f_code, extra_code_index, - (void **)&function_id); + PyCodeObject *code = PyFrame_GetCode(frame); + _PyCode_GetExtra((PyObject *)code, extra_code_index, (void **)&function_id); if (function_id == 0) { Py_ssize_t filename_length, function_length; - const char* filename = PyUnicode_AsUTF8AndSize(frame->f_code->co_filename, + const char* filename = PyUnicode_AsUTF8AndSize(code->co_filename, &filename_length); - const char* function_name = PyUnicode_AsUTF8AndSize(frame->f_code->co_name, + const char* function_name = PyUnicode_AsUTF8AndSize(code->co_name, &function_length); increment_reentrancy(); function_id = pymemprofile_add_function_location(filename, (uint64_t)filename_length, function_name, (uint64_t)function_length); decrement_reentrancy(); - _PyCode_SetExtra((PyObject *)frame->f_code, extra_code_index, + _PyCode_SetExtra((PyObject *)code, extra_code_index, (void *)function_id + 1); } else { function_id -= 1; } start_call(function_id, frame->f_lineno); + Py_DECREF(code); break; case PyTrace_RETURN: finish_call(); From f61e300ceb2dcadbf29633d0fc47f240fbff68cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 10:25:44 -0400 Subject: [PATCH 04/20] News file. --- .changelog/381.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 .changelog/381.feature diff --git a/.changelog/381.feature b/.changelog/381.feature new file mode 100644 index 00000000..2cbbb3e4 --- /dev/null +++ b/.changelog/381.feature @@ -0,0 +1 @@ +Added Python 3.11 support. \ No newline at end of file From 7cffb7c5907529c885c10c44c62ba71e9faf8526 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 10:34:13 -0400 Subject: [PATCH 05/20] Filter runpy correctly on Python 3.11. --- memapi/src/memorytracking.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/memapi/src/memorytracking.rs b/memapi/src/memorytracking.rs index 879c898e..e06b3c24 100644 --- a/memapi/src/memorytracking.rs +++ b/memapi/src/memorytracking.rs @@ -234,7 +234,8 @@ fn runpy_prefix_length(calls: std::slice::Iter<(CallSiteId, (&str, &str))>) -> u let mut length = 0; let runpy_path = get_runpy_path(); for (_, (_, filename)) in calls { - if *filename == runpy_path { + // On Python 3.11 it uses for some reason. + if *filename == runpy_path || *filename == "" { length += 1; } else { return length; From 817a062f9bfa3cc1282ca7b78cc066ad8ffe62c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 12:32:26 -0400 Subject: [PATCH 06/20] Get rid of reliance on current frame. --- filpreload/src/_filpreload.c | 55 +++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index d5b4bef0..dbedfde2 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -1,4 +1,6 @@ #include "Python.h" +#include "ceval.h" +#include "pyframe.h" #if PY_MINOR_VERSION < 11 #include "code.h" #else @@ -124,8 +126,22 @@ static inline int should_track_memory() { return (likely(initialized) && atomic_load_explicit(&tracking_allocations, memory_order_acquire) && !am_i_reentrant()); } -// Current thread's Python state: -static _Thread_local PyFrameObject *current_frame = NULL; +// Current thread's Python state; typically only set in C functions where GIL +// might be released. +static _Thread_local int current_line_number = -1; + +static inline int get_current_line_number() { + if (PyGILState_Check()) { + PyFrameObject *frame = PyEval_GetFrame(); + if (frame != NULL) { + return PyFrame_GetLineNumber(frame); + } + } + if (current_line_number != -1) { + return current_line_number; + } + return 0; +} // The file and function name responsible for an allocation. struct FunctionLocation { @@ -206,7 +222,7 @@ static void __attribute__((constructor)) constructor() { initialized = 1; } -static void start_call(uint64_t function_id, uint16_t line_number) { +static void start_call(uint64_t function_id, uint16_t line_number, PyFrameObject* current_frame) { if (should_track_memory()) { increment_reentrancy(); uint16_t parent_line_number = 0; @@ -232,15 +248,13 @@ __attribute__((visibility("hidden"))) int fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { switch (what) { case PyTrace_CALL: - // Store the current frame, so malloc() can look up line number: - current_frame = frame; - /* We want an efficient identifier for filename+fuction name. So we register the function + filename with some Rust code that gives back its ID, and then store the ID. Due to bad API design, value 0 indicates "no result", so we actually store the result + 1. */ + current_line_number = frame->f_lineno; uint64_t function_id = 0; assert(extra_code_index != -1); PyCodeObject *code = PyFrame_GetCode(frame); @@ -259,13 +273,24 @@ fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { } else { function_id -= 1; } - start_call(function_id, frame->f_lineno); + start_call(function_id, current_line_number, frame); Py_DECREF(code); break; case PyTrace_RETURN: finish_call(); - // We're done with this frame, so set the parent frame: - current_frame = frame->f_back; + if (frame->f_back == NULL) { + current_line_number = -1; + } else { + current_line_number = frame->f_back->f_lineno; + } + break; + case PyTrace_C_CALL: + // C calls might release GIL, in which case they won't change the line + // number, so record it. + current_line_number = frame->f_lineno; + break; + case PyTrace_C_RETURN: + current_line_number = -1; break; default: break; @@ -322,20 +347,12 @@ fil_dump_peak_to_flamegraph(const char *path) { // *** End APIs called by Python *** static void add_allocation(size_t address, size_t size) { - uint16_t line_number = 0; - PyFrameObject *f = current_frame; - if (f != NULL) { - line_number = PyFrame_GetLineNumber(f); - } + uint16_t line_number = get_current_line_number(); pymemprofile_add_allocation(address, size, line_number); } static void add_anon_mmap(size_t address, size_t size) { - uint16_t line_number = 0; - PyFrameObject *f = current_frame; - if (f != NULL) { - line_number = PyFrame_GetLineNumber(f); - } + uint16_t line_number = get_current_line_number(); pymemprofile_add_anon_mmap(address, size, line_number); } From f7b3eb39d88903362435cd6dd0c25c2e25a05b17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 12:38:39 -0400 Subject: [PATCH 07/20] Tests pass on 3.11. --- filpreload/src/_filpreload.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index dbedfde2..4b9dc4cf 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -226,9 +226,11 @@ static void start_call(uint64_t function_id, uint16_t line_number, PyFrameObject if (should_track_memory()) { increment_reentrancy(); uint16_t parent_line_number = 0; - if (current_frame != NULL && current_frame->f_back != NULL) { - PyFrameObject *f = current_frame->f_back; - parent_line_number = PyFrame_GetLineNumber(f); + if (current_frame != NULL) { + PyFrameObject *parent = PyFrame_GetBack(current_frame); + if (parent != NULL ){ + parent_line_number = PyFrame_GetLineNumber(parent); + } } pymemprofile_start_call(parent_line_number, function_id, line_number); decrement_reentrancy(); From 617f9e1e85ada18fabbb606fec87105b5e4e5ce6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Oct 2022 12:41:15 -0400 Subject: [PATCH 08/20] Get rid of direct poking at frame objects except in one place where it still seems to work. --- filpreload/src/_filpreload.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index 4b9dc4cf..503cb1fe 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -280,16 +280,17 @@ fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { break; case PyTrace_RETURN: finish_call(); - if (frame->f_back == NULL) { + PyFrameObject* parent = PyFrame_GetBack(frame); + if (parent == NULL) { current_line_number = -1; } else { - current_line_number = frame->f_back->f_lineno; + current_line_number = PyFrame_GetLineNumber(parent); } break; case PyTrace_C_CALL: // C calls might release GIL, in which case they won't change the line // number, so record it. - current_line_number = frame->f_lineno; + current_line_number = PyFrame_GetLineNumber(frame); break; case PyTrace_C_RETURN: current_line_number = -1; From f9e84ed7a6590cb23f866406d493d8f0fa3c1450 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 30 Oct 2022 17:51:21 -0400 Subject: [PATCH 09/20] Fix compilation on older versions of Python. --- filpreload/src/_filpreload.c | 1 - 1 file changed, 1 deletion(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index 503cb1fe..144d49fb 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -1,6 +1,5 @@ #include "Python.h" #include "ceval.h" -#include "pyframe.h" #if PY_MINOR_VERSION < 11 #include "code.h" #else From ac0688017874496b86387c6091c5b2fded32d2a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 30 Oct 2022 19:02:00 -0400 Subject: [PATCH 10/20] Add missing functions for Python < 3.9, fix decrefing so it only happens when needed. --- filpreload/src/_filpreload.c | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index 144d49fb..06c61adb 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -23,6 +23,20 @@ #include #include +#if PY_MINOR_VERSION < 9 +PyFrameObject *PyFrame_GetBack(PyFrameObject *frame) { + if (frame->f_back != NULL) { + Py_INCREF(frame->f_back); + } + return frame->f_back; +} + +PyCodeObject *PyFrame_GetCode(PyFrameObject *frame) { + Py_INCREF(frame->f_code); + return frame->f_code; +} +#endif + // Macro to create the publicly exposed symbol: #ifdef __APPLE__ #define SYMBOL_PREFIX(func) reimplemented_##func @@ -40,9 +54,9 @@ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) -// Underlying APIs we're wrapping: -static void *(*underlying_real_mmap)(void *addr, size_t length, int prot, - int flags, int fd, off_t offset) = 0; + // Underlying APIs we're wrapping: + static void *(*underlying_real_mmap)(void *addr, size_t length, int prot, + int flags, int fd, off_t offset) = 0; static int (*underlying_real_pthread_create)(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), @@ -229,6 +243,7 @@ static void start_call(uint64_t function_id, uint16_t line_number, PyFrameObject PyFrameObject *parent = PyFrame_GetBack(current_frame); if (parent != NULL ){ parent_line_number = PyFrame_GetLineNumber(parent); + Py_DECREF(parent); } } pymemprofile_start_call(parent_line_number, function_id, line_number); @@ -271,19 +286,21 @@ fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { decrement_reentrancy(); _PyCode_SetExtra((PyObject *)code, extra_code_index, (void *)function_id + 1); + Py_DECREF(code); } else { function_id -= 1; } start_call(function_id, current_line_number, frame); - Py_DECREF(code); break; case PyTrace_RETURN: finish_call(); - PyFrameObject* parent = PyFrame_GetBack(frame); - if (parent == NULL) { - current_line_number = -1; - } else { - current_line_number = PyFrame_GetLineNumber(parent); + if (frame != NULL) { + PyFrameObject* parent = PyFrame_GetBack(frame); + if (parent == NULL) { + current_line_number = -1; + } else { + current_line_number = PyFrame_GetLineNumber(parent); + } } break; case PyTrace_C_CALL: From dfcecc360179ba7d2531754cd366f99df1ff94da Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 30 Oct 2022 19:05:01 -0400 Subject: [PATCH 11/20] Tweak output. --- filprofiler/_report.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/filprofiler/_report.py b/filprofiler/_report.py index 2a8e2d3c..308151d3 100644 --- a/filprofiler/_report.py +++ b/filprofiler/_report.py @@ -76,16 +76,18 @@ def render_report(output_path: str, now: datetime) -> str:

Profiling result

+

Check out my other project:

Find memory and performance bottlenecks in production!

When your data pipeline is too slow in production, reproducing the problem on your laptop is hard or impossible—which means identifying and fixing the problem can be tricky.

What if you knew the cause of the problem as soon as you realized it was happening?

-

That's why you need - the Sciagraph profiler, designed to find performance +

That's how + the Sciagraph profiler can help you: + it's designed to find performance and memory bottlenecks by continuously profiling in production.



@@ -93,6 +95,7 @@ def render_report(output_path: str, now: datetime) -> str:


+
Need help, or does something look wrong? Read the documentation, From fa061eddb24b5a8f1588f83faf490eec962f8c14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 17:11:36 -0500 Subject: [PATCH 12/20] Newer macOS. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91308a6e..c68a354a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - os: ["ubuntu-20.04", "macos-10.15"] + os: ["ubuntu-20.04", "macos-latest"] runs-on: "${{ matrix.os }}" From d79c7a667e5224f339daef42dacd5dc3a3854a99 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 19:39:32 -0500 Subject: [PATCH 13/20] Try switching to cibuildwheel on Linux too. --- .github/workflows/main.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c68a354a..8877fe42 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,7 @@ jobs: make test - name: "Build macOS wheels" if: startsWith(matrix.os, 'mac') && (matrix.python-version == '3.9') - uses: pypa/cibuildwheel@v2.9.0 + uses: pypa/cibuildwheel@v2.11.2 env: MACOSX_DEPLOYMENT_TARGET: "10.15" CIBW_ARCHS_MACOS: "x86_64 arm64" @@ -86,13 +86,18 @@ jobs: with: output-dir: dist - name: "Build manylinux wheels" + if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == '3.8') + uses: pypa/cibuildwheel@v2.11.2 + env: + CIBW_SKIP: "cp36* pp* *musl*" + CIBW_BEFORE_BUILD: "touch filpreload/src/_filpreload.c" # force rebuild of Python code with new interpreter + CIBW_TEST_COMMAND: python -m filprofiler run {project}/benchmarks/pystone.py + with: + output-dir: dist + - name: "Test manylinux, just a bit" if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == '3.8') run: | set -euo pipefail - . venv/bin/activate - make manylinux-wheel - # Test wheel - deactivate python3.8 -m venv venv2 . venv2/bin/activate pip install -r requirements-dev.txt From f3d25299a4e5b36e9dcba483a56087838828c46b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 20:04:57 -0500 Subject: [PATCH 14/20] Switch to newer base image that has 3.11. --- wheels/Dockerfile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wheels/Dockerfile.build b/wheels/Dockerfile.build index 05929fee..245b5440 100644 --- a/wheels/Dockerfile.build +++ b/wheels/Dockerfile.build @@ -1,4 +1,4 @@ -FROM quay.io/pypa/manylinux2010_x86_64:latest +FROM quay.io/pypa/manylinux2014_x86_64:latest COPY --from=rust:1.41.0-slim /usr/local/cargo /usr/local/cargo COPY --from=rust:1.41.0-slim /usr/local/rustup /usr/local/rustup From 35b13107e24dbe27c48110f9216e150777a184e8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 20:05:16 -0500 Subject: [PATCH 15/20] Revert back to custom scheme for now. --- .github/workflows/main.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8877fe42..8c3d4d28 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,18 +86,13 @@ jobs: with: output-dir: dist - name: "Build manylinux wheels" - if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == '3.8') - uses: pypa/cibuildwheel@v2.11.2 - env: - CIBW_SKIP: "cp36* pp* *musl*" - CIBW_BEFORE_BUILD: "touch filpreload/src/_filpreload.c" # force rebuild of Python code with new interpreter - CIBW_TEST_COMMAND: python -m filprofiler run {project}/benchmarks/pystone.py - with: - output-dir: dist - - name: "Test manylinux, just a bit" if: startsWith(matrix.os, 'ubuntu') && (matrix.python-version == '3.8') run: | set -euo pipefail + . venv/bin/activate + make manylinux-wheel + # Test wheel + deactivate python3.8 -m venv venv2 . venv2/bin/activate pip install -r requirements-dev.txt From d338505ddb785392239f881667c829c065f76d1a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 20:05:47 -0500 Subject: [PATCH 16/20] It's 2014 now. --- wheels/build-wheels.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wheels/build-wheels.sh b/wheels/build-wheels.sh index 11f4e859..d1b51fe3 100755 --- a/wheels/build-wheels.sh +++ b/wheels/build-wheels.sh @@ -24,9 +24,9 @@ for PYBIN in /opt/python/cp{37,38,39,310,311}*/bin; do "${PYBIN}/python" -m pip wheel -w /tmp/wheel . done -auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp37*whl -auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp38*whl -auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp39*whl -auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp310*whl -auditwheel repair --plat manylinux2010_x86_64 -w dist/ /tmp/wheel/filprofiler*cp311*whl +auditwheel repair --plat manylinux2014_x86_64 -w dist/ /tmp/wheel/filprofiler*cp37*whl +auditwheel repair --plat manylinux2014_x86_64 -w dist/ /tmp/wheel/filprofiler*cp38*whl +auditwheel repair --plat manylinux2014_x86_64 -w dist/ /tmp/wheel/filprofiler*cp39*whl +auditwheel repair --plat manylinux2014_x86_64 -w dist/ /tmp/wheel/filprofiler*cp310*whl +auditwheel repair --plat manylinux2014_x86_64 -w dist/ /tmp/wheel/filprofiler*cp311*whl From 1182302239c8e3dd8ff857e33eac0049aff245a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 20:54:05 -0500 Subject: [PATCH 17/20] Fix Docker image. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cf288346..066489ca 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ wheel: .PHONY: manylinux-wheel manylinux-wheel: - docker run -u $(shell id -u):$(shell id -g) -v $(PWD):/src quay.io/pypa/manylinux2010_x86_64:latest /src/wheels/build-wheels.sh + docker run -u $(shell id -u):$(shell id -g) -v $(PWD):/src quay.io/pypa/manylinux2014_x86_64:latest /src/wheels/build-wheels.sh .PHONY: clean clean: From aa7554d72eda8c6cfdc00685717a94ca7edcffd5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sun, 6 Nov 2022 20:54:52 -0500 Subject: [PATCH 18/20] Delete unused code. --- Makefile | 4 ---- wheels/Dockerfile.build | 9 --------- 2 files changed, 13 deletions(-) delete mode 100644 wheels/Dockerfile.build diff --git a/Makefile b/Makefile index 066489ca..ae1d2562 100644 --- a/Makefile +++ b/Makefile @@ -40,10 +40,6 @@ test-python-no-deps: cd tests/test-scripts && python -m numpy.f2py -c fortran.f90 -m fortran env RUST_BACKTRACE=1 py.test -v tests/ -.PHONY: docker-image -docker-image: - docker build -t manylinux-rust -f wheels/Dockerfile.build . - .PHONY: wheel wheel: python setup.py bdist_wheel diff --git a/wheels/Dockerfile.build b/wheels/Dockerfile.build deleted file mode 100644 index 245b5440..00000000 --- a/wheels/Dockerfile.build +++ /dev/null @@ -1,9 +0,0 @@ -FROM quay.io/pypa/manylinux2014_x86_64:latest - -COPY --from=rust:1.41.0-slim /usr/local/cargo /usr/local/cargo -COPY --from=rust:1.41.0-slim /usr/local/rustup /usr/local/rustup - -ENV PATH=$PATH:/usr/local/cargo/bin -ENV CARGO_HOME=/usr/local/cargo -ENV RUSTUP_HOME=/usr/local/rustup -RUN chmod a+w /usr/local/cargo From e877c4cc3eda34bcede33b44dfb812ffe7478d2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 12:15:44 -0500 Subject: [PATCH 19/20] Don't leak frames. --- filpreload/src/_filpreload.c | 1 + 1 file changed, 1 insertion(+) diff --git a/filpreload/src/_filpreload.c b/filpreload/src/_filpreload.c index 06c61adb..ada67d81 100644 --- a/filpreload/src/_filpreload.c +++ b/filpreload/src/_filpreload.c @@ -300,6 +300,7 @@ fil_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) { current_line_number = -1; } else { current_line_number = PyFrame_GetLineNumber(parent); + Py_DECREF(parent); } } break; From eda0a0c955018574af68944a8c0f13330586b39a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 12:19:26 -0500 Subject: [PATCH 20/20] Prepare for release. --- .changelog/381.feature | 1 - CHANGELOG.md | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 .changelog/381.feature diff --git a/.changelog/381.feature b/.changelog/381.feature deleted file mode 100644 index 2cbbb3e4..00000000 --- a/.changelog/381.feature +++ /dev/null @@ -1 +0,0 @@ -Added Python 3.11 support. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 7173957c..d9d30c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Release notes +## 2022.11.0 (2022-11-07) + +### Features + +- Added initial Python 3.11 support; unfortunately this increased performance overhead a little. ([#381](https://github.com/pythonspeed/filprofiler/issues/381)) + ## 2022.10.0 (2022-10-19) ### Bugfixes