From 357e9e9da3929cb9d55ea31896e66f488e44e8f2 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Thu, 13 Jul 2023 02:31:17 +0530 Subject: [PATCH 01/75] gh-106602: [Enum] Add __copy__ and __deepcopy__ (GH-106666) --- Lib/enum.py | 6 ++++++ Lib/test/test_enum.py | 8 ++++++++ .../2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst | 1 + 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst diff --git a/Lib/enum.py b/Lib/enum.py index 202f0da028bdfe..0c985b2c778569 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1218,6 +1218,12 @@ def __hash__(self): def __reduce_ex__(self, proto): return self.__class__, (self._value_, ) + def __deepcopy__(self,memo): + return self + + def __copy__(self): + return self + # enum.property is used to provide access to the `name` and # `value` attributes of enum members while keeping some measure of # protection from modification, while still allowing for an enumeration diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index adb1e0e52c5485..a286411f7bcf57 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -804,9 +804,17 @@ def test_copy(self): TE = self.MainEnum copied = copy.copy(TE) self.assertEqual(copied, TE) + self.assertIs(copied, TE) deep = copy.deepcopy(TE) self.assertEqual(deep, TE) + self.assertIs(deep, TE) + def test_copy_member(self): + TE = self.MainEnum + copied = copy.copy(TE.first) + self.assertIs(copied, TE.first) + deep = copy.deepcopy(TE.first) + self.assertIs(deep, TE.first) class _FlagTests: diff --git a/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst b/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst new file mode 100644 index 00000000000000..d9c122f1d3c723 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst @@ -0,0 +1 @@ +Add __copy__ and __deepcopy__ in :mod:`enum` From a180e7a0df342e9f089998fc680be83ad2e49a79 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 12 Jul 2023 22:33:47 +0100 Subject: [PATCH 02/75] gh-104050: Argument clinic: Annotate the `Destination` class (#106655) Co-authored-by: Nikita Sobolev --- Tools/clinic/clinic.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a0cf50b29c978d..ce3039cdd8bff4 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1954,27 +1954,32 @@ def dump(self): return "".join(texts) +@dc.dataclass(slots=True, repr=False) class Destination: - def __init__(self, name, type, clinic, *args): - self.name = name - self.type = type - self.clinic = clinic - self.buffers = BufferSeries() + name: str + type: str + clinic: Clinic + buffers: BufferSeries = dc.field(init=False, default_factory=BufferSeries) + filename: str = dc.field(init=False) # set in __post_init__ + args: dc.InitVar[tuple[str, ...]] = () + + def __post_init__(self, args: tuple[str, ...]) -> None: valid_types = ('buffer', 'file', 'suppress') - if type not in valid_types: + if self.type not in valid_types: fail( - f"Invalid destination type {type!r} for {name}, " + f"Invalid destination type {self.type!r} for {self.name}, " f"must be {', '.join(valid_types)}" ) - extra_arguments = 1 if type == "file" else 0 + extra_arguments = 1 if self.type == "file" else 0 if len(args) < extra_arguments: - fail(f"Not enough arguments for destination {name} new {type}") + fail(f"Not enough arguments for destination {self.name} new {self.type}") if len(args) > extra_arguments: - fail(f"Too many arguments for destination {name} new {type}") - if type =='file': + fail(f"Too many arguments for destination {self.name} new {self.type}") + if self.type =='file': d = {} - filename = clinic.filename + filename = self.clinic.filename + assert filename is not None d['path'] = filename dirname, basename = os.path.split(filename) if not dirname: @@ -1984,19 +1989,19 @@ def __init__(self, name, type, clinic, *args): d['basename_root'], d['basename_extension'] = os.path.splitext(filename) self.filename = args[0].format_map(d) - def __repr__(self): + def __repr__(self) -> str: if self.type == 'file': file_repr = " " + repr(self.filename) else: file_repr = '' return "".join(("")) - def clear(self): + def clear(self) -> None: if self.type != 'buffer': fail("Can't clear destination" + self.name + " , it's not of type buffer") self.buffers.clear() - def dump(self): + def dump(self) -> str: return self.buffers.dump() @@ -2164,11 +2169,11 @@ def add_destination( self, name: str, type: str, - *args + *args: str ) -> None: if name in self.destinations: fail("Destination already exists: " + repr(name)) - self.destinations[name] = Destination(name, type, self, *args) + self.destinations[name] = Destination(name, type, self, args) def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) From 8aa4beaad0d95917b1bb12d146bc15c1aa815e08 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 12 Jul 2023 23:48:36 +0100 Subject: [PATCH 03/75] gh-104683: Argument clinic: modernise `cpp.Monitor` (#106698) --- Tools/clinic/cpp.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index c1a2eeef22deca..fbac81336b545e 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -1,3 +1,4 @@ +import dataclasses as dc import re import sys from collections.abc import Callable @@ -15,6 +16,11 @@ def negate(condition: str) -> str: return condition[1:] return "!" + condition + +is_a_simple_defined = re.compile(r'^defined\s*\(\s*[A-Za-z0-9_]+\s*\)$').match + + +@dc.dataclass(repr=False) class Monitor: """ A simple C preprocessor that scans C source and computes, line by line, @@ -27,25 +33,20 @@ class Monitor: Anyway this implementation seems to work well enough for the CPython sources. """ + filename: str | None = None + _: dc.KW_ONLY + verbose: bool = False - is_a_simple_defined: Callable[[str], re.Match[str] | None] - is_a_simple_defined = re.compile(r'^defined\s*\(\s*[A-Za-z0-9_]+\s*\)$').match - - def __init__(self, filename: str | None = None, *, verbose: bool = False) -> None: + def __post_init__(self) -> None: self.stack: TokenStack = [] self.in_comment = False self.continuation: str | None = None self.line_number = 0 - self.filename = filename - self.verbose = verbose def __repr__(self) -> str: - return ''.join(( - '")) + return ( + f"" + ) def status(self) -> str: return str(self.line_number).rjust(4) + ": " + self.condition() @@ -152,7 +153,7 @@ def pop_stack() -> TokenAndCondition: if not condition: self.fail("Invalid format for #" + token + " line: no argument!") if token in {'if', 'elif'}: - if not self.is_a_simple_defined(condition): + if not is_a_simple_defined(condition): condition = "(" + condition + ")" if token == 'elif': previous_token, previous_condition = pop_stack() From 2d43beec225a0495ffa0344f961b99717e5f1106 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 13 Jul 2023 00:49:30 +0200 Subject: [PATCH 04/75] gh-104050: Argument Clinic: Annotate nested function parser_body() in the CLanguage class (#106699) --- Tools/clinic/clinic.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ce3039cdd8bff4..8a75aba1e4de93 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -898,22 +898,24 @@ def output_templates(self, f): # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields = () - def parser_body(prototype, *fields, declarations=''): + def parser_body( + prototype: str, + *fields: str, + declarations: str = '' + ) -> str: nonlocal parser_body_fields add, output = text_accumulator() add(prototype) parser_body_fields = fields - fields = list(fields) - fields.insert(0, normalize_snippet(""" + preamble = normalize_snippet(""" {{ {return_value_declaration} {parser_declarations} {declarations} {initializers} - """) + "\n") - # just imagine--your code is here in the middle - fields.append(normalize_snippet(""" + """) + "\n" + finale = normalize_snippet(""" {modifications} {return_value} = {c_basename}_impl({impl_arguments}); {return_conversion} @@ -923,8 +925,8 @@ def parser_body(prototype, *fields, declarations=''): {cleanup} return return_value; }} - """)) - for field in fields: + """) + for field in preamble, *fields, finale: add('\n') add(field) return linear_format(output(), parser_declarations=declarations) From ab86426a3472ab68747815299d390b213793c3d1 Mon Sep 17 00:00:00 2001 From: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Date: Wed, 12 Jul 2023 22:50:45 -0400 Subject: [PATCH 05/75] gh-105235: Prevent reading outside buffer during mmap.find() (#105252) * Add a special case for s[-m:] == p in _PyBytes_Find * Add tests for _PyBytes_Find * Make sure that start <= end in mmap.find --- Lib/test/test_mmap.py | 21 ++++ ...-06-02-19-37-29.gh-issue-105235.fgFGTi.rst | 1 + Modules/_testinternalcapi.c | 114 ++++++++++++++++++ Modules/mmapmodule.c | 7 +- Objects/bytesobject.c | 21 +++- 5 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 517cbe0cb115ab..bab868600895c1 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -299,6 +299,27 @@ def test_find_end(self): self.assertEqual(m.find(b'one', 1, -2), -1) self.assertEqual(m.find(bytearray(b'one')), 0) + for i in range(-n-1, n+1): + for j in range(-n-1, n+1): + for p in [b"o", b"on", b"two", b"ones", b"s"]: + expected = data.find(p, i, j) + self.assertEqual(m.find(p, i, j), expected, (p, i, j)) + + def test_find_does_not_access_beyond_buffer(self): + try: + flags = mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS + PAGESIZE = mmap.PAGESIZE + PROT_NONE = 0 + PROT_READ = mmap.PROT_READ + except AttributeError as e: + raise unittest.SkipTest("mmap flags unavailable") from e + for i in range(0, 2049): + with mmap.mmap(-1, PAGESIZE * (i + 1), + flags=flags, prot=PROT_NONE) as guard: + with mmap.mmap(-1, PAGESIZE * (i + 2048), + flags=flags, prot=PROT_READ) as fm: + fm.find(b"fo", -2) + def test_rfind(self): # test the new 'end' parameter works as expected diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst new file mode 100644 index 00000000000000..c28d0101cd4bad --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst @@ -0,0 +1 @@ +Prevent out-of-bounds memory access during ``mmap.find()`` calls. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 52e524a40672ed..7745dd5abc22f0 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,6 +14,7 @@ #include "interpreteridobject.h" // _PyInterpreterID_LookUp() #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble #include "pycore_ceval.h" // _PyEval_AddPendingCall #include "pycore_fileutils.h" // _Py_normpath @@ -443,6 +444,118 @@ test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args)) } +static int +check_bytes_find(const char *haystack0, const char *needle0, + int offset, Py_ssize_t expected) +{ + Py_ssize_t len_haystack = strlen(haystack0); + Py_ssize_t len_needle = strlen(needle0); + Py_ssize_t result_1 = _PyBytes_Find(haystack0, len_haystack, + needle0, len_needle, offset); + if (result_1 != expected) { + PyErr_Format(PyExc_AssertionError, + "Incorrect result_1: '%s' in '%s' (offset=%zd)", + needle0, haystack0, offset); + return -1; + } + // Allocate new buffer with no NULL terminator. + char *haystack = PyMem_Malloc(len_haystack); + if (haystack == NULL) { + PyErr_NoMemory(); + return -1; + } + char *needle = PyMem_Malloc(len_needle); + if (needle == NULL) { + PyMem_Free(haystack); + PyErr_NoMemory(); + return -1; + } + memcpy(haystack, haystack0, len_haystack); + memcpy(needle, needle0, len_needle); + Py_ssize_t result_2 = _PyBytes_Find(haystack, len_haystack, + needle, len_needle, offset); + PyMem_Free(haystack); + PyMem_Free(needle); + if (result_2 != expected) { + PyErr_Format(PyExc_AssertionError, + "Incorrect result_2: '%s' in '%s' (offset=%zd)", + needle0, haystack0, offset); + return -1; + } + return 0; +} + +static int +check_bytes_find_large(Py_ssize_t len_haystack, Py_ssize_t len_needle, + const char *needle) +{ + char *zeros = PyMem_RawCalloc(len_haystack, 1); + if (zeros == NULL) { + PyErr_NoMemory(); + return -1; + } + Py_ssize_t res = _PyBytes_Find(zeros, len_haystack, needle, len_needle, 0); + PyMem_RawFree(zeros); + if (res != -1) { + PyErr_Format(PyExc_AssertionError, + "check_bytes_find_large(%zd, %zd) found %zd", + len_haystack, len_needle, res); + return -1; + } + return 0; +} + +static PyObject * +test_bytes_find(PyObject *self, PyObject *Py_UNUSED(args)) +{ + #define CHECK(H, N, O, E) do { \ + if (check_bytes_find(H, N, O, E) < 0) { \ + return NULL; \ + } \ + } while (0) + + CHECK("", "", 0, 0); + CHECK("Python", "", 0, 0); + CHECK("Python", "", 3, 3); + CHECK("Python", "", 6, 6); + CHECK("Python", "yth", 0, 1); + CHECK("ython", "yth", 1, 1); + CHECK("thon", "yth", 2, -1); + CHECK("Python", "thon", 0, 2); + CHECK("ython", "thon", 1, 2); + CHECK("thon", "thon", 2, 2); + CHECK("hon", "thon", 3, -1); + CHECK("Pytho", "zz", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ab", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ba", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bb", 0, -1); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", "ab", 0, 30); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaba", "ba", 0, 30); + CHECK("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaabb", "bb", 0, 30); + #undef CHECK + + // Hunt for segfaults + // n, m chosen here so that (n - m) % (m + 1) == 0 + // This would make default_find in fastsearch.h access haystack[n]. + if (check_bytes_find_large(2048, 2, "ab") < 0) { + return NULL; + } + if (check_bytes_find_large(4096, 16, "0123456789abcdef") < 0) { + return NULL; + } + if (check_bytes_find_large(8192, 2, "ab") < 0) { + return NULL; + } + if (check_bytes_find_large(16384, 4, "abcd") < 0) { + return NULL; + } + if (check_bytes_find_large(32768, 2, "ab") < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject * normalize_path(PyObject *self, PyObject *filename) { @@ -1328,6 +1441,7 @@ static PyMethodDef module_functions[] = { {"reset_path_config", test_reset_path_config, METH_NOARGS}, {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS}, {"test_edit_cost", test_edit_cost, METH_NOARGS}, + {"test_bytes_find", test_bytes_find, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fef27123a3282b..c1cd5b0efaa3d2 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -342,12 +342,17 @@ mmap_gfind(mmap_object *self, Py_ssize_t res; CHECK_VALID_OR_RELEASE(NULL, view); - if (reverse) { + if (end < start) { + res = -1; + } + else if (reverse) { + assert(0 <= start && start <= end && end <= self->size); res = _PyBytes_ReverseFind( self->data + start, end - start, view.buf, view.len, start); } else { + assert(0 <= start && start <= end && end <= self->size); res = _PyBytes_Find( self->data + start, end - start, view.buf, view.len, start); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 477bc4d31e812b..6b9231a9fa7693 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1272,8 +1272,25 @@ _PyBytes_Find(const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset) { - return stringlib_find(haystack, len_haystack, - needle, len_needle, offset); + assert(len_haystack >= 0); + assert(len_needle >= 0); + // Extra checks because stringlib_find accesses haystack[len_haystack]. + if (len_needle == 0) { + return offset; + } + if (len_needle > len_haystack) { + return -1; + } + assert(len_haystack >= 1); + Py_ssize_t res = stringlib_find(haystack, len_haystack - 1, + needle, len_needle, offset); + if (res == -1) { + Py_ssize_t last_align = len_haystack - len_needle; + if (memcmp(haystack + last_align, needle, len_needle) == 0) { + return offset + last_align; + } + } + return res; } Py_ssize_t From af51bd7cda9c0cba149b882c1e501765595e5fc3 Mon Sep 17 00:00:00 2001 From: Jim Porter <826865+jimporter@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:08:33 -0700 Subject: [PATCH 06/75] =?UTF-8?q?gh-89427:=20Set=20VIRTUAL=5FENV=5FPROMPT?= =?UTF-8?q?=20even=20when=20VIRTUAL=5FENV=5FDISABLE=5FPROMPT=E2=80=A6=20(G?= =?UTF-8?q?H-106643)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/venv/scripts/common/Activate.ps1 | 3 ++- Lib/venv/scripts/common/activate | 5 +++-- Lib/venv/scripts/posix/activate.csh | 2 +- Lib/venv/scripts/posix/activate.fish | 2 +- .../Library/2023-07-11-12-34-04.gh-issue-89427.GOkCp9.rst | 2 ++ 5 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-11-12-34-04.gh-issue-89427.GOkCp9.rst diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index eeea3583fa130d..d75b8fbcfc7778 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -219,6 +219,8 @@ deactivate -nondestructive # that there is an activated venv. $env:VIRTUAL_ENV = $VenvDir +$env:VIRTUAL_ENV_PROMPT = $Prompt + if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Verbose "Setting prompt to '$Prompt'" @@ -233,7 +235,6 @@ if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " _OLD_VIRTUAL_PROMPT } - $env:VIRTUAL_ENV_PROMPT = $Prompt } # Clear PYTHONHOME diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 408df5cb93b9e9..458740a35b0d20 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -52,6 +52,9 @@ _OLD_VIRTUAL_PATH="$PATH" PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" export PATH +VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" +export VIRTUAL_ENV_PROMPT + # unset PYTHONHOME if set # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) # could use `if (set -u; : $PYTHONHOME) ;` in bash @@ -64,8 +67,6 @@ if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" PS1="__VENV_PROMPT__${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" - export VIRTUAL_ENV_PROMPT fi # This should detect bash and zsh, which have a hash command that must diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index 5e8d66fa9e5061..9caf138a919a86 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -13,13 +13,13 @@ setenv VIRTUAL_ENV "__VENV_DIR__" set _OLD_VIRTUAL_PATH="$PATH" setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" endif alias pydoc python -m pydoc diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index 91ad6442e05692..565df23d1e2a13 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -37,6 +37,7 @@ set -gx VIRTUAL_ENV "__VENV_DIR__" set -gx _OLD_VIRTUAL_PATH $PATH set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -65,5 +66,4 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" end diff --git a/Misc/NEWS.d/next/Library/2023-07-11-12-34-04.gh-issue-89427.GOkCp9.rst b/Misc/NEWS.d/next/Library/2023-07-11-12-34-04.gh-issue-89427.GOkCp9.rst new file mode 100644 index 00000000000000..1605920cb8138b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-11-12-34-04.gh-issue-89427.GOkCp9.rst @@ -0,0 +1,2 @@ +Set the environment variable ``VIRTUAL_ENV_PROMPT`` at :mod:`venv` +activation, even when ``VIRTUAL_ENV_DISABLE_PROMPT`` is set. From 7e6ce48872fa3de98c986057764f35e1b2f4b936 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Thu, 13 Jul 2023 08:12:56 +0200 Subject: [PATCH 07/75] gh-106628: email parsing speedup (gh-106629) --- Lib/email/feedparser.py | 15 +++++++++------ ...2023-07-11-16-36-22.gh-issue-106628.Kx8Zvc.rst | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-11-16-36-22.gh-issue-106628.Kx8Zvc.rst diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 885097c7dda067..53d71f50225152 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -37,6 +37,8 @@ headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' NL = '\n' +boundaryendRE = re.compile( + r'(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') NeedMoreData = object() @@ -327,9 +329,10 @@ def _parsegen(self): # this onto the input stream until we've scanned past the # preamble. separator = '--' + boundary - boundaryre = re.compile( - '(?P' + re.escape(separator) + - r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') + def boundarymatch(line): + if not line.startswith(separator): + return None + return boundaryendRE.match(line, len(separator)) capturing_preamble = True preamble = [] linesep = False @@ -341,7 +344,7 @@ def _parsegen(self): continue if line == '': break - mo = boundaryre.match(line) + mo = boundarymatch(line) if mo: # If we're looking at the end boundary, we're done with # this multipart. If there was a newline at the end of @@ -373,13 +376,13 @@ def _parsegen(self): if line is NeedMoreData: yield NeedMoreData continue - mo = boundaryre.match(line) + mo = boundarymatch(line) if not mo: self._input.unreadline(line) break # Recurse to parse this subpart; the input stream points # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) + self._input.push_eof_matcher(boundarymatch) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Misc/NEWS.d/next/Library/2023-07-11-16-36-22.gh-issue-106628.Kx8Zvc.rst b/Misc/NEWS.d/next/Library/2023-07-11-16-36-22.gh-issue-106628.Kx8Zvc.rst new file mode 100644 index 00000000000000..6fa276e901f648 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-11-16-36-22.gh-issue-106628.Kx8Zvc.rst @@ -0,0 +1,2 @@ +Speed up parsing of emails by about 20% by not compiling a new regular +expression for every single email. From 4b4a5b70aa8d47b1e2a0582b741c31b786da762a Mon Sep 17 00:00:00 2001 From: Chris Brett Date: Thu, 13 Jul 2023 08:51:13 +0100 Subject: [PATCH 08/75] gh-106634: Corrected minor asyncio doc issues (#106671) --- Lib/asyncio/base_events.py | 2 +- Lib/asyncio/events.py | 2 +- Misc/ACKS | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index f650e6b0488cf8..b092c9343634e2 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -727,7 +727,7 @@ def call_later(self, delay, callback, *args, context=None): always relative to the current time. Each callback will be called exactly once. If two callbacks - are scheduled for exactly the same time, it undefined which + are scheduled for exactly the same time, it is undefined which will be called first. Any positional arguments after the callback will be passed to diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index ce44942186b272..0ccf85105e6673 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -617,7 +617,7 @@ class AbstractEventLoopPolicy: def get_event_loop(self): """Get the event loop for the current context. - Returns an event loop object implementing the BaseEventLoop interface, + Returns an event loop object implementing the AbstractEventLoop interface, or raises an exception in case no event loop has been set for the current context and the current policy does not specify to create one. diff --git a/Misc/ACKS b/Misc/ACKS index ef0029a7e4119d..645ad5b700baaa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -226,6 +226,7 @@ Erik Bray Brian Brazil Demian Brecht Dave Brennan +Christopher Richard James Brett Tom Bridgman Anthony Briggs Keith Briggs From 32718f908cc92c474fd968912368b8a4500bd055 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 13 Jul 2023 14:30:35 +0100 Subject: [PATCH 09/75] gh-106309: Deprecate typing.no_type_check_decorator (#106312) --- Doc/library/typing.rst | 35 +++++++++++-------- Doc/whatsnew/3.13.rst | 4 +++ Lib/test/test_typing.py | 12 ++++--- Lib/typing.py | 2 ++ ...-07-01-16-51-55.gh-issue-106309.hSlB17.rst | 2 ++ 5 files changed, 36 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-01-16-51-55.gh-issue-106309.hSlB17.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 11af3ea3c9030a..0cf875582f7f42 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2849,6 +2849,9 @@ Functions and decorators This wraps the decorator with something that wraps the decorated function in :func:`no_type_check`. + .. deprecated-removed:: 3.13 3.15 + No type checker ever added support for ``@no_type_check_decorator``. It + is therefore deprecated, and will be removed in Python 3.15. .. decorator:: override @@ -3648,18 +3651,20 @@ Certain features in ``typing`` are deprecated and may be removed in a future version of Python. The following table summarizes major deprecations for your convenience. This is subject to change, and not all deprecations are listed. -+----------------------------------+---------------+-------------------+----------------+ -| Feature | Deprecated in | Projected removal | PEP/issue | -+==================================+===============+===================+================+ -| ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | -| collections | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | -| ``typing.Sized`` | | | | -+----------------------------------+---------------+-------------------+----------------+ -| ``typing.TypeAlias`` | 3.12 | Undecided | :pep:`695` | -+----------------------------------+---------------+-------------------+----------------+ ++-------------------------------------+---------------+-------------------+----------------+ +| Feature | Deprecated in | Projected removal | PEP/issue | ++=====================================+===============+===================+================+ +| ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | +| collections | | | | ++-------------------------------------+---------------+-------------------+----------------+ +| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | ++-------------------------------------+---------------+-------------------+----------------+ +| ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | ++-------------------------------------+---------------+-------------------+----------------+ +| ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | +| ``typing.Sized`` | | | | ++-------------------------------------+---------------+-------------------+----------------+ +| ``typing.TypeAlias`` | 3.12 | Undecided | :pep:`695` | ++-------------------------------------+---------------+-------------------+----------------+ +| ``typing.no_type_check_decorator`` | 3.13 | 3.15 | :gh:`106309` | ++-------------------------------------+---------------+-------------------+----------------+ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index b7c436fc151611..06fcaf4608cdcb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -161,6 +161,10 @@ Deprecated ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) +* :func:`typing.no_type_check_decorator` is deprecated, and scheduled for + removal in Python 3.15. After eight years in the :mod:`typing` module, it + has yet to be supported by any major type checkers. + (Contributed by Alex Waygood in :gh:`106309`.) * :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, emits :exc:`DeprecationWarning` since 3.13 diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1df21926d1f67e..0450a87577ecea 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5794,10 +5794,14 @@ class F: get_type_hints(clazz) def test_meta_no_type_check(self): - - @no_type_check_decorator - def magic_decorator(func): - return func + depr_msg = ( + "'typing.no_type_check_decorator' is deprecated " + "and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, depr_msg): + @no_type_check_decorator + def magic_decorator(func): + return func self.assertEqual(magic_decorator.__name__, 'magic_decorator') diff --git a/Lib/typing.py b/Lib/typing.py index 9187b74b0e2e1f..387b4c5ad5284b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2395,6 +2395,8 @@ def no_type_check_decorator(decorator): This wraps the decorator with something that wraps the decorated function in @no_type_check. """ + import warnings + warnings._deprecated("typing.no_type_check_decorator", remove=(3, 15)) @functools.wraps(decorator) def wrapped_decorator(*args, **kwds): func = decorator(*args, **kwds) diff --git a/Misc/NEWS.d/next/Library/2023-07-01-16-51-55.gh-issue-106309.hSlB17.rst b/Misc/NEWS.d/next/Library/2023-07-01-16-51-55.gh-issue-106309.hSlB17.rst new file mode 100644 index 00000000000000..5bd3880520871f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-01-16-51-55.gh-issue-106309.hSlB17.rst @@ -0,0 +1,2 @@ +Deprecate :func:`typing.no_type_check_decorator`. No major type checker ever +added support for this decorator. Patch by Alex Waygood. From 487861c6aef2fbcd92ccabb05ea1b57d18299b29 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 13 Jul 2023 16:36:19 +0100 Subject: [PATCH 10/75] GH-104909: Split `LOAD_ATTR_INSTANCE_VALUE` into micro-ops (GH-106678) --- Include/internal/pycore_opcode_metadata.h | 22 ++++--- ...-07-12-11-18-55.gh-issue-104909.DRUsuh.rst | 1 + Python/bytecodes.c | 19 +++++- Python/executor_cases.c.h | 18 ++++++ Python/generated_cases.c.h | 59 ++++++++++++------- Tools/cases_generator/generate_cases.py | 14 +---- 6 files changed, 87 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-12-11-18-55.gh-issue-104909.DRUsuh.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d2c1f9ad6e5fb3..79bbe9a916f596 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -39,10 +39,12 @@ #define _SKIP_CACHE 317 #define _GUARD_GLOBALS_VERSION 318 #define _GUARD_BUILTINS_VERSION 319 -#define IS_NONE 320 -#define _ITER_CHECK_RANGE 321 -#define _ITER_EXHAUSTED_RANGE 322 -#define _ITER_NEXT_RANGE 323 +#define _GUARD_TYPE_VERSION 320 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 321 +#define IS_NONE 322 +#define _ITER_CHECK_RANGE 323 +#define _ITER_EXHAUSTED_RANGE 324 +#define _ITER_NEXT_RANGE 325 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -932,7 +934,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC0, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; #define HAS_ARG_FLAG (1) #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) @@ -1321,9 +1323,11 @@ const char * const _PyOpcode_uop_name[512] = { [317] = "_SKIP_CACHE", [318] = "_GUARD_GLOBALS_VERSION", [319] = "_GUARD_BUILTINS_VERSION", - [320] = "IS_NONE", - [321] = "_ITER_CHECK_RANGE", - [322] = "_ITER_EXHAUSTED_RANGE", - [323] = "_ITER_NEXT_RANGE", + [320] = "_GUARD_TYPE_VERSION", + [321] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [322] = "IS_NONE", + [323] = "_ITER_CHECK_RANGE", + [324] = "_ITER_EXHAUSTED_RANGE", + [325] = "_ITER_NEXT_RANGE", }; #endif // NEED_OPCODE_METADATA diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-12-11-18-55.gh-issue-104909.DRUsuh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-12-11-18-55.gh-issue-104909.DRUsuh.rst new file mode 100644 index 00000000000000..e0c1e67515a62c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-12-11-18-55.gh-issue-104909.DRUsuh.rst @@ -0,0 +1 @@ +Split :opcode:`LOAD_ATTR_INSTANCE_VALUE` into micro-ops. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 18862f87b65fa0..176dbb584d0987 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1816,14 +1816,21 @@ dummy_func( LOAD_ATTR, }; - inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - assert(tp->tp_dictoffset < 0); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + } + + op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + } + + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); @@ -1832,6 +1839,12 @@ dummy_func( DECREF_INPUTS(); } + macro(LOAD_ATTR_INSTANCE_VALUE) = + _SKIP_CACHE + // Skip over the counter + _GUARD_TYPE_VERSION + + _CHECK_MANAGED_OBJECT_HAS_VALUES + + _LOAD_ATTR_INSTANCE_VALUE; + inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2c2dbf429cec11..805ea06e072167 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1425,6 +1425,24 @@ break; } + case _GUARD_TYPE_VERSION: { + PyObject *owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)operand; + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + PyObject *owner = stack_pointer[-1]; + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + break; + } + case COMPARE_OP: { static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 383432f51a89ac..d43c7386bd6f6d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2240,28 +2240,45 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *res2 = NULL; - PyObject *res; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t index = read_u16(&next_instr[3].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - assert(tp->tp_dictoffset < 0); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + PyObject *_tmp_1; + PyObject *_tmp_2 = stack_pointer[-1]; + { + } + { + PyObject *owner = _tmp_2; + uint32_t type_version = read_u32(&next_instr[1].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + _tmp_2 = owner; + } + { + PyObject *owner = _tmp_2; + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + _tmp_2 = owner; + } + { + PyObject *owner = _tmp_2; + PyObject *res2 = NULL; + PyObject *res; + uint16_t index = read_u16(&next_instr[3].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + res = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; + Py_DECREF(owner); + if (oparg & 1) { _tmp_2 = res2; } + _tmp_1 = res; + } next_instr += 9; + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = _tmp_1; + if (oparg & 1) { stack_pointer[-2] = _tmp_2; } DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index a20abcde85b7c7..ba4ef1ee97693e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -839,19 +839,7 @@ def map_families(self) -> None: ) else: member_instr.family = family - elif member_macro := self.macro_instrs.get(member): - for part in member_macro.parts: - if isinstance(part, Component): - if part.instr.family not in (family, None): - self.error( - f"Component {part.instr.name} of macro {member} " - f"is a member of multiple families " - f"({part.instr.family.name}, {family.name}).", - family, - ) - else: - part.instr.family = family - else: + elif not self.macro_instrs.get(member): self.error( f"Unknown instruction {member!r} referenced in family {family.name!r}", family, From 2f3ee02c22c4b42bf6075a75104c3cfbb4eb4c86 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Thu, 13 Jul 2023 11:45:21 -0400 Subject: [PATCH 11/75] gh-106690: Add a .coveragerc file to the CPython repository (#8150) The added file is the coverage default at some point in time + checking branches both ways + IDLE additions, labelled as such and somewhat designed to be unlikely to affect other files. Located in the CPython repository directory, it can be used where it is or copied elsewhere, depending on how one runs coverage. --------- Co-authored-by: Terry Jan Reedy Co-authored-by: Alex Waygood Co-authored-by: Erlend E. Aasland --- .coveragerc | 19 +++++++++++++++++++ ...-07-12-14-07-07.gh-issue-106690.NDz-oG.rst | 1 + 2 files changed, 20 insertions(+) create mode 100644 .coveragerc create mode 100644 Misc/NEWS.d/next/Tests/2023-07-12-14-07-07.gh-issue-106690.NDz-oG.rst diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000000..18bf2f40fe523f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,19 @@ +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + .*# pragma: no cover + .*# pragma: no branch + + # Additions for IDLE: + .*# htest # + if not (_htest or _utest): + if not .*_utest: + if .*_htest: + diff --git a/Misc/NEWS.d/next/Tests/2023-07-12-14-07-07.gh-issue-106690.NDz-oG.rst b/Misc/NEWS.d/next/Tests/2023-07-12-14-07-07.gh-issue-106690.NDz-oG.rst new file mode 100644 index 00000000000000..e7dc0ac2220502 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-12-14-07-07.gh-issue-106690.NDz-oG.rst @@ -0,0 +1 @@ +Add .coveragerc to cpython repository for use with coverage package. From e6e0ea0113748db1e9fe675be6db9041cd5cce1f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 13 Jul 2023 12:14:51 -0700 Subject: [PATCH 12/75] gh-106701: Move the hand-written Tier 2 uops to bytecodes.c (#106702) This moves EXIT_TRACE, SAVE_IP, JUMP_TO_TOP, and _POP_JUMP_IF_{FALSE,TRUE} from ceval.c to bytecodes.c. They are no less special than before, but this way they are discoverable o the copy-and-patch tooling. --- Include/internal/pycore_opcode_metadata.h | 100 +++++++++++----------- Python/bytecodes.c | 30 +++++++ Python/ceval.c | 40 --------- Python/executor_cases.c.h | 36 ++++++++ Tools/cases_generator/generate_cases.py | 12 ++- 5 files changed, 124 insertions(+), 94 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 79bbe9a916f596..c88640777e3fb0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -21,30 +21,30 @@ #define EXIT_TRACE 300 #define SAVE_IP 301 -#define _POP_JUMP_IF_FALSE 302 -#define _POP_JUMP_IF_TRUE 303 -#define JUMP_TO_TOP 304 -#define _GUARD_BOTH_INT 305 -#define _BINARY_OP_MULTIPLY_INT 306 -#define _BINARY_OP_ADD_INT 307 -#define _BINARY_OP_SUBTRACT_INT 308 -#define _GUARD_BOTH_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT 310 -#define _BINARY_OP_ADD_FLOAT 311 -#define _BINARY_OP_SUBTRACT_FLOAT 312 -#define _GUARD_BOTH_UNICODE 313 -#define _BINARY_OP_ADD_UNICODE 314 -#define _LOAD_LOCALS 315 -#define _LOAD_FROM_DICT_OR_GLOBALS 316 -#define _SKIP_CACHE 317 -#define _GUARD_GLOBALS_VERSION 318 -#define _GUARD_BUILTINS_VERSION 319 -#define _GUARD_TYPE_VERSION 320 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 321 -#define IS_NONE 322 -#define _ITER_CHECK_RANGE 323 -#define _ITER_EXHAUSTED_RANGE 324 -#define _ITER_NEXT_RANGE 325 +#define _GUARD_BOTH_INT 302 +#define _BINARY_OP_MULTIPLY_INT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_SUBTRACT_INT 305 +#define _GUARD_BOTH_FLOAT 306 +#define _BINARY_OP_MULTIPLY_FLOAT 307 +#define _BINARY_OP_ADD_FLOAT 308 +#define _BINARY_OP_SUBTRACT_FLOAT 309 +#define _GUARD_BOTH_UNICODE 310 +#define _BINARY_OP_ADD_UNICODE 311 +#define _LOAD_LOCALS 312 +#define _LOAD_FROM_DICT_OR_GLOBALS 313 +#define _SKIP_CACHE 314 +#define _GUARD_GLOBALS_VERSION 315 +#define _GUARD_BUILTINS_VERSION 316 +#define _GUARD_TYPE_VERSION 317 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 318 +#define IS_NONE 319 +#define _ITER_CHECK_RANGE 320 +#define _ITER_EXHAUSTED_RANGE 321 +#define _ITER_NEXT_RANGE 322 +#define _POP_JUMP_IF_FALSE 323 +#define _POP_JUMP_IF_TRUE 324 +#define JUMP_TO_TOP 325 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -1303,31 +1303,31 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; const char * const _PyOpcode_uop_name[512] = { - [300] = "EXIT_TRACE", - [301] = "SAVE_IP", - [302] = "_POP_JUMP_IF_FALSE", - [303] = "_POP_JUMP_IF_TRUE", - [304] = "JUMP_TO_TOP", - [305] = "_GUARD_BOTH_INT", - [306] = "_BINARY_OP_MULTIPLY_INT", - [307] = "_BINARY_OP_ADD_INT", - [308] = "_BINARY_OP_SUBTRACT_INT", - [309] = "_GUARD_BOTH_FLOAT", - [310] = "_BINARY_OP_MULTIPLY_FLOAT", - [311] = "_BINARY_OP_ADD_FLOAT", - [312] = "_BINARY_OP_SUBTRACT_FLOAT", - [313] = "_GUARD_BOTH_UNICODE", - [314] = "_BINARY_OP_ADD_UNICODE", - [315] = "_LOAD_LOCALS", - [316] = "_LOAD_FROM_DICT_OR_GLOBALS", - [317] = "_SKIP_CACHE", - [318] = "_GUARD_GLOBALS_VERSION", - [319] = "_GUARD_BUILTINS_VERSION", - [320] = "_GUARD_TYPE_VERSION", - [321] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", - [322] = "IS_NONE", - [323] = "_ITER_CHECK_RANGE", - [324] = "_ITER_EXHAUSTED_RANGE", - [325] = "_ITER_NEXT_RANGE", + [EXIT_TRACE] = "EXIT_TRACE", + [SAVE_IP] = "SAVE_IP", + [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", + [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", + [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", + [_SKIP_CACHE] = "_SKIP_CACHE", + [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [IS_NONE] = "IS_NONE", + [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_EXHAUSTED_RANGE] = "_ITER_EXHAUSTED_RANGE", + [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", + [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", + [JUMP_TO_TOP] = "JUMP_TO_TOP", }; #endif // NEED_OPCODE_METADATA diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 176dbb584d0987..1fe9970e53cdfe 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3654,6 +3654,36 @@ dummy_func( Py_UNREACHABLE(); } + ///////// Tier-2 only opcodes ///////// + + op(_POP_JUMP_IF_FALSE, (flag -- )) { + if (Py_IsFalse(flag)) { + pc = oparg; + } + } + + op(_POP_JUMP_IF_TRUE, (flag -- )) { + if (Py_IsTrue(flag)) { + pc = oparg; + } + } + + op(JUMP_TO_TOP, (--)) { + pc = 0; + CHECK_EVAL_BREAKER(); + } + + op(SAVE_IP, (--)) { + frame->prev_instr = ip_offset + oparg; + } + + op(EXIT_TRACE, (--)) { + frame->prev_instr--; // Back up to just before destination + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(self); + return frame; + } + // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index de44085d732cfa..d6c72fa3ff386c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2764,46 +2764,6 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject #define ENABLE_SPECIALIZATION 0 #include "executor_cases.c.h" - // NOTE: These pop-jumps move the uop pc, not the bytecode ip - case _POP_JUMP_IF_FALSE: - { - if (Py_IsFalse(stack_pointer[-1])) { - pc = oparg; - } - stack_pointer--; - break; - } - - case _POP_JUMP_IF_TRUE: - { - if (Py_IsTrue(stack_pointer[-1])) { - pc = oparg; - } - stack_pointer--; - break; - } - - case JUMP_TO_TOP: - { - pc = 0; - CHECK_EVAL_BREAKER(); - break; - } - - case SAVE_IP: - { - frame->prev_instr = ip_offset + oparg; - break; - } - - case EXIT_TRACE: - { - frame->prev_instr--; // Back up to just before destination - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(self); - return frame; - } - default: { fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 805ea06e072167..ce54755d5d25f1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1987,3 +1987,39 @@ stack_pointer[-(2 + (oparg-2))] = top; break; } + + case _POP_JUMP_IF_FALSE: { + PyObject *flag = stack_pointer[-1]; + if (Py_IsFalse(flag)) { + pc = oparg; + } + STACK_SHRINK(1); + break; + } + + case _POP_JUMP_IF_TRUE: { + PyObject *flag = stack_pointer[-1]; + if (Py_IsTrue(flag)) { + pc = oparg; + } + STACK_SHRINK(1); + break; + } + + case JUMP_TO_TOP: { + pc = 0; + break; + } + + case SAVE_IP: { + frame->prev_instr = ip_offset + oparg; + break; + } + + case EXIT_TRACE: { + frame->prev_instr--; // Back up to just before destination + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(self); + return frame; + break; + } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index ba4ef1ee97693e..ce16271097b955 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -410,6 +410,8 @@ def __init__(self, inst: parser.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" + if self.name == "EXIT_TRACE": + return True # This has 'return frame' but it's okay if self.always_exits: # print(f"Skipping {self.name} because it always exits") return False @@ -1278,7 +1280,7 @@ def write_metadata(self) -> None: typing.assert_never(thing) with self.out.block("const char * const _PyOpcode_uop_name[512] =", ";"): - self.write_uop_items(lambda name, counter: f"[{counter}] = \"{name}\",") + self.write_uop_items(lambda name, counter: f"[{name}] = \"{name}\",") self.out.emit("#endif // NEED_OPCODE_METADATA") @@ -1324,17 +1326,19 @@ def write_pseudo_instrs(self) -> None: def write_uop_items(self, make_text: typing.Callable[[str, int], str]) -> None: """Write '#define XXX NNN' for each uop""" counter = 300 # TODO: Avoid collision with pseudo instructions + seen = set() def add(name: str) -> None: + if name in seen: + return nonlocal counter self.out.emit(make_text(name, counter)) counter += 1 + seen.add(name) + # These two are first by convention add("EXIT_TRACE") add("SAVE_IP") - add("_POP_JUMP_IF_FALSE") - add("_POP_JUMP_IF_TRUE") - add("JUMP_TO_TOP") for instr in self.instrs.values(): if instr.kind == "op" and instr.is_viable_uop(): From 8d2f3c36caf9ecdee1176314b18388aef6e7f2c2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 13 Jul 2023 09:18:53 -1000 Subject: [PATCH 13/75] gh-106664: selectors: add get() method to _SelectorMapping (#106665) It can be used to avoid raising and catching KeyError twice via __getitem__. Co-authored-by: Inada Naoki --- Lib/selectors.py | 14 +++++++++----- Lib/test/test_selectors.py | 6 ++++++ .../2023-07-12-03-04-45.gh-issue-106664.ZeUG78.rst | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-12-03-04-45.gh-issue-106664.ZeUG78.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index af6a4f94b5008a..dfcc125dcd94ef 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -66,12 +66,16 @@ def __init__(self, selector): def __len__(self): return len(self._selector._fd_to_key) + def get(self, fileobj, default=None): + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key.get(fd, default) + def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None + fd = self._selector._fileobj_lookup(fileobj) + key = self._selector._fd_to_key.get(fd) + if key is None: + raise KeyError("{!r} is not registered".format(fileobj)) + return key def __iter__(self): return iter(self._selector._fd_to_key) diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index c2db88c203920a..4545cbadb796fd 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -223,6 +223,8 @@ def test_close(self): self.assertRaises(RuntimeError, s.get_key, wr) self.assertRaises(KeyError, mapping.__getitem__, rd) self.assertRaises(KeyError, mapping.__getitem__, wr) + self.assertEqual(mapping.get(rd), None) + self.assertEqual(mapping.get(wr), None) def test_get_key(self): s = self.SELECTOR() @@ -241,13 +243,17 @@ def test_get_map(self): self.addCleanup(s.close) rd, wr = self.make_socketpair() + sentinel = object() keys = s.get_map() self.assertFalse(keys) self.assertEqual(len(keys), 0) self.assertEqual(list(keys), []) + self.assertEqual(keys.get(rd), None) + self.assertEqual(keys.get(rd, sentinel), sentinel) key = s.register(rd, selectors.EVENT_READ, "data") self.assertIn(rd, keys) + self.assertEqual(key, keys.get(rd)) self.assertEqual(key, keys[rd]) self.assertEqual(len(keys), 1) self.assertEqual(list(keys), [rd.fileno()]) diff --git a/Misc/NEWS.d/next/Library/2023-07-12-03-04-45.gh-issue-106664.ZeUG78.rst b/Misc/NEWS.d/next/Library/2023-07-12-03-04-45.gh-issue-106664.ZeUG78.rst new file mode 100644 index 00000000000000..c278cad74bd049 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-12-03-04-45.gh-issue-106664.ZeUG78.rst @@ -0,0 +1 @@ +:mod:`selectors`: Add ``_SelectorMapping.get()`` method and optimize ``_SelectorMapping.__getitem__()``. From f014f1567c081542aaf5cecce118edf35e09bcc5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 13 Jul 2023 12:24:54 -0700 Subject: [PATCH 14/75] docs: clarify Path.suffix (GH-106650) --- Doc/library/pathlib.rst | 5 +++-- Doc/library/zipfile.rst | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 338575404ff0ad..af81df217eea92 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -432,7 +432,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.suffix - The file extension of the final component, if any:: + The last dot-separated portion of the final component, if any:: >>> PurePosixPath('my/library/setup.py').suffix '.py' @@ -441,10 +441,11 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffix '' + This is commonly called the file extension. .. attribute:: PurePath.suffixes - A list of the path's file extensions:: + A list of the path's suffixes, often called file extensions:: >>> PurePosixPath('my/library.tar.gar').suffixes ['.tar', '.gar'] diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 45f3d340bd82d3..bd951e4872f113 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -577,7 +577,8 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. .. data:: Path.suffix - The file extension of the final component. + The last dot-separated portion of the final component, if any. + This is commonly called the file extension. .. versionadded:: 3.11 Added :data:`Path.suffix` property. @@ -591,7 +592,7 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. .. data:: Path.suffixes - A list of the path’s file extensions. + A list of the path’s suffixes, commonly called file extensions. .. versionadded:: 3.11 Added :data:`Path.suffixes` property. From ec45c513d389510930a62631a21a1dbb3f3aabb7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 14 Jul 2023 00:18:32 +0200 Subject: [PATCH 15/75] gh-106368: Increase Argument Clinic test coverage (#106728) - improve output_parameter() coverage - improve coverage for Function.kind --- Lib/test/clinic.test.c | 553 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_clinic.py | 37 +++ 2 files changed, 590 insertions(+) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index da97c4bdd7e8e5..2fd8760415dc72 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -3,6 +3,10 @@ output preset block [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3c81ac2402d06a8b]*/ +/*[clinic input] +class Test "TestObj *" "TestType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=fc7e50384d12b83f]*/ /*[clinic input] test_object_converter @@ -61,6 +65,58 @@ test_object_converter_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic end generated code: output=886f4f9b598726b6 input=005e6a8a711a869b]*/ +/*[clinic input] +cloned = test_object_converter +Check the clone feature. +[clinic start generated code]*/ + +PyDoc_STRVAR(cloned__doc__, +"cloned($module, a, b, c, d, /)\n" +"--\n" +"\n" +"Check the clone feature."); + +#define CLONED_METHODDEF \ + {"cloned", _PyCFunction_CAST(cloned), METH_FASTCALL, cloned__doc__}, + +static PyObject * +cloned_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyUnicode_Object *d); + +static PyObject * +cloned(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + PyObject *c; + PyUnicode_Object *d; + + if (!_PyArg_CheckPositional("cloned", nargs, 4, 4)) { + goto exit; + } + a = args[0]; + if (!PyUnicode_FSConverter(args[1], &b)) { + goto exit; + } + if (!PyUnicode_Check(args[2])) { + _PyArg_BadArgument("cloned", "argument 3", "str", args[2]); + goto exit; + } + c = args[2]; + d = (PyUnicode_Object *)args[3]; + return_value = cloned_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +cloned_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyUnicode_Object *d) +/*[clinic end generated code: output=026b483e27c38065 input=0543614019d6fcc7]*/ + + /*[clinic input] test_object_converter_one_arg @@ -4265,3 +4321,500 @@ static PyObject * mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, PyObject *return_value) /*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ + + +/*[clinic input] +Test.cls_with_param + cls: defining_class + / + a: int +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_cls_with_param__doc__, +"cls_with_param($self, /, a)\n" +"--\n" +"\n"); + +#define TEST_CLS_WITH_PARAM_METHODDEF \ + {"cls_with_param", _PyCFunction_CAST(Test_cls_with_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test_cls_with_param__doc__}, + +static PyObject * +Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a); + +static PyObject * +Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cls_with_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = _PyLong_AsInt(args[0]); + if (a == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = Test_cls_with_param_impl(self, cls, a); + +exit: + return return_value; +} + +static PyObject * +Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) +/*[clinic end generated code: output=00218e7f583e6c81 input=af158077bd237ef9]*/ + + +/*[clinic input] +Test.__init__ +Empty init method. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test()\n" +"--\n" +"\n" +"Empty init method."); + +static int +Test___init___impl(TestObj *self); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + return_value = Test___init___impl((TestObj *)self); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self) +/*[clinic end generated code: output=f6a35c85bc5b408f input=4ea79fee54d0c3ff]*/ + + +/*[clinic input] +@classmethod +Test.__new__ +Empty new method. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test()\n" +"--\n" +"\n" +"Empty new method."); + +static PyObject * +Test_impl(PyTypeObject *type); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + return_value = Test_impl(type); + +exit: + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type) +/*[clinic end generated code: output=68a117adc057940f input=6fe98a19f097907f]*/ + + +/*[clinic input] +Test.cls_no_params + cls: defining_class + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_cls_no_params__doc__, +"cls_no_params($self, /)\n" +"--\n" +"\n"); + +#define TEST_CLS_NO_PARAMS_METHODDEF \ + {"cls_no_params", _PyCFunction_CAST(Test_cls_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test_cls_no_params__doc__}, + +static PyObject * +Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); + +static PyObject * +Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); + return NULL; + } + return Test_cls_no_params_impl(self, cls); +} + +static PyObject * +Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) +/*[clinic end generated code: output=cc8845f22cff3dcb input=e7e2e4e344e96a11]*/ + + +/*[clinic input] +Test.metho_not_default_return_converter -> int + a: object + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_metho_not_default_return_converter__doc__, +"metho_not_default_return_converter($self, a, /)\n" +"--\n" +"\n"); + +#define TEST_METHO_NOT_DEFAULT_RETURN_CONVERTER_METHODDEF \ + {"metho_not_default_return_converter", (PyCFunction)Test_metho_not_default_return_converter, METH_O, Test_metho_not_default_return_converter__doc__}, + +static int +Test_metho_not_default_return_converter_impl(TestObj *self, PyObject *a); + +static PyObject * +Test_metho_not_default_return_converter(TestObj *self, PyObject *a) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = Test_metho_not_default_return_converter_impl(self, a); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +static int +Test_metho_not_default_return_converter_impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=3350de11bd538007 input=428657129b521177]*/ + + +/*[clinic input] +Test.an_metho_arg_named_arg + arg: int + Name should be mangled to 'arg_' in generated output. + / +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_an_metho_arg_named_arg__doc__, +"an_metho_arg_named_arg($self, arg, /)\n" +"--\n" +"\n" +"\n" +"\n" +" arg\n" +" Name should be mangled to \'arg_\' in generated output."); + +#define TEST_AN_METHO_ARG_NAMED_ARG_METHODDEF \ + {"an_metho_arg_named_arg", (PyCFunction)Test_an_metho_arg_named_arg, METH_O, Test_an_metho_arg_named_arg__doc__}, + +static PyObject * +Test_an_metho_arg_named_arg_impl(TestObj *self, int arg); + +static PyObject * +Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) +{ + PyObject *return_value = NULL; + int arg; + + arg = _PyLong_AsInt(arg_); + if (arg == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = Test_an_metho_arg_named_arg_impl(self, arg); + +exit: + return return_value; +} + +static PyObject * +Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) +/*[clinic end generated code: output=7d590626642194ae input=2a53a57cf5624f95]*/ + + +/*[clinic input] +Test.__init__ + *args: object + / +Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static int +Test___init___impl(TestObj *self, PyObject *args); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *__clinic_args = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_GetSlice(0, -1); + return_value = Test___init___impl((TestObj *)self, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *args) +/*[clinic end generated code: output=0ed1009fe0dcf98d input=96c3ddc0cd38fc0c]*/ + + +/*[clinic input] +@classmethod +Test.__new__ + *args: object + / +Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *args); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { + goto exit; + } + __clinic_args = PyTuple_GetSlice(0, -1); + return_value = Test_impl(type, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *args) +/*[clinic end generated code: output=8b219f6633e2a2e9 input=26a672e2e9750120]*/ + + +/*[clinic input] +Test.__init__ + a: object +Init method with positional or keyword arguments. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(a)\n" +"--\n" +"\n" +"Init method with positional or keyword arguments."); + +static int +Test___init___impl(TestObj *self, PyObject *a); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Test", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = Test___init___impl((TestObj *)self, a); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/ + + +/*[clinic input] +@classmethod +Test.class_method +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_class_method__doc__, +"class_method($type, /)\n" +"--\n" +"\n"); + +#define TEST_CLASS_METHOD_METHODDEF \ + {"class_method", (PyCFunction)Test_class_method, METH_NOARGS|METH_CLASS, Test_class_method__doc__}, + +static PyObject * +Test_class_method_impl(PyTypeObject *type); + +static PyObject * +Test_class_method(PyTypeObject *type, PyObject *Py_UNUSED(ignored)) +{ + return Test_class_method_impl(type); +} + +static PyObject * +Test_class_method_impl(PyTypeObject *type) +/*[clinic end generated code: output=47fb7ecca1abcaaa input=43bc4a0494547b80]*/ + + +/*[clinic input] +@staticmethod +Test.static_method +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_static_method__doc__, +"static_method()\n" +"--\n" +"\n"); + +#define TEST_STATIC_METHOD_METHODDEF \ + {"static_method", (PyCFunction)Test_static_method, METH_NOARGS|METH_STATIC, Test_static_method__doc__}, + +static PyObject * +Test_static_method_impl(); + +static PyObject * +Test_static_method(void *null, PyObject *Py_UNUSED(ignored)) +{ + return Test_static_method_impl(); +} + +static PyObject * +Test_static_method_impl() +/*[clinic end generated code: output=82524a63025cf7ab input=dae892fac55ae72b]*/ + + +/*[clinic input] +@coexist +Test.meth_coexist +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_meth_coexist__doc__, +"meth_coexist($self, /)\n" +"--\n" +"\n"); + +#define TEST_METH_COEXIST_METHODDEF \ + {"meth_coexist", (PyCFunction)Test_meth_coexist, METH_NOARGS|METH_COEXIST, Test_meth_coexist__doc__}, + +static PyObject * +Test_meth_coexist_impl(TestObj *self); + +static PyObject * +Test_meth_coexist(TestObj *self, PyObject *Py_UNUSED(ignored)) +{ + return Test_meth_coexist_impl(self); +} + +static PyObject * +Test_meth_coexist_impl(TestObj *self) +/*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 685ba58642a5ae..975840333e5901 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1013,6 +1013,43 @@ def test_defining_class_param_cannot_be_optional(self): out = self.parse_function_should_fail(block) self.assertEqual(out, expected_error_msg) + def test_slot_methods_cannot_access_defining_class(self): + block = """ + module foo + class Foo "" "" + Foo.__init__ + cls: defining_class + a: object + """ + msg = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, msg): + self.parse_function(block) + + def test_new_must_be_a_class_method(self): + expected_error_msg = ( + "Error on line 0:\n" + "__new__ must be a class method!\n" + ) + out = self.parse_function_should_fail(""" + module foo + class Foo "" "" + Foo.__new__ + """) + self.assertEqual(out, expected_error_msg) + + def test_init_must_be_a_normal_method(self): + expected_error_msg = ( + "Error on line 0:\n" + "__init__ must be a normal method, not a class or static method!\n" + ) + out = self.parse_function_should_fail(""" + module foo + class Foo "" "" + @classmethod + Foo.__init__ + """) + self.assertEqual(out, expected_error_msg) + def test_unused_param(self): block = self.parse(""" module foo From 128a6c1d889639012c83c122b82c9cdf0b62d587 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 13 Jul 2023 23:54:05 +0100 Subject: [PATCH 16/75] gh-104683: Argument clinic: use an enum to describe the different kinds of functions (#106721) Argument clinic: use an enum to describe the different kinds of functions --- Tools/clinic/clinic.py | 69 +++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 8a75aba1e4de93..2dce33690993dd 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -807,7 +807,7 @@ def output_templates(self, f): default_return_converter = (not f.return_converter or f.return_converter.type == 'PyObject *') - new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) + new_or_init = f.kind.new_or_init vararg = NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 @@ -1250,7 +1250,7 @@ def parser_body( if new_or_init: methoddef_define = '' - if f.kind == METHOD_NEW: + if f.kind is METHOD_NEW: parser_prototype = parser_prototype_keyword else: return_value_declaration = "int return_value = -1;" @@ -1475,7 +1475,7 @@ def render_function( last_group = 0 first_optional = len(selfless) positional = selfless and selfless[-1].is_positional_only() - new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) + new_or_init = f.kind.new_or_init has_option_groups = False # offset i by -1 because first_optional needs to ignore self @@ -2441,9 +2441,28 @@ def __repr__(self) -> str: """.strip().split()) -INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW = """ -INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW -""".replace(",", "").strip().split() +class FunctionKind(enum.Enum): + INVALID = enum.auto() + CALLABLE = enum.auto() + STATIC_METHOD = enum.auto() + CLASS_METHOD = enum.auto() + METHOD_INIT = enum.auto() + METHOD_NEW = enum.auto() + + @functools.cached_property + def new_or_init(self) -> bool: + return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} + + def __repr__(self) -> str: + return f"" + + +INVALID: Final = FunctionKind.INVALID +CALLABLE: Final = FunctionKind.CALLABLE +STATIC_METHOD: Final = FunctionKind.STATIC_METHOD +CLASS_METHOD: Final = FunctionKind.CLASS_METHOD +METHOD_INIT: Final = FunctionKind.METHOD_INIT +METHOD_NEW: Final = FunctionKind.METHOD_NEW ParamDict = dict[str, "Parameter"] ReturnConverterType = Callable[..., "CReturnConverter"] @@ -2471,7 +2490,7 @@ class Function: return_converter: CReturnConverter return_annotation: object = inspect.Signature.empty docstring: str = '' - kind: str = CALLABLE + kind: FunctionKind = CALLABLE coexist: bool = False # docstring_only means "don't generate a machine-readable # signature, just a normal docstring". it's True for @@ -2497,15 +2516,16 @@ def render_parameters(self) -> list[Parameter]: @property def methoddef_flags(self) -> str | None: - if self.kind in (METHOD_INIT, METHOD_NEW): + if self.kind.new_or_init: return None flags = [] - if self.kind == CLASS_METHOD: - flags.append('METH_CLASS') - elif self.kind == STATIC_METHOD: - flags.append('METH_STATIC') - else: - assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind) + match self.kind: + case FunctionKind.CLASS_METHOD: + flags.append('METH_CLASS') + case FunctionKind.STATIC_METHOD: + flags.append('METH_STATIC') + case _ as kind: + assert kind is FunctionKind.CALLABLE, f"unknown kind: {kind!r}" if self.coexist: flags.append('METH_COEXIST') return '|'.join(flags) @@ -3888,7 +3908,7 @@ def correct_name_for_self( if f.cls: return "PyObject *", "self" return "PyObject *", "module" - if f.kind == STATIC_METHOD: + if f.kind is STATIC_METHOD: return "void *", "null" if f.kind in (CLASS_METHOD, METHOD_NEW): return "PyTypeObject *", "type" @@ -3921,9 +3941,8 @@ def pre_render(self): self.type = self.specified_type or self.type or default_type kind = self.function.kind - new_or_init = kind in (METHOD_NEW, METHOD_INIT) - if (kind == STATIC_METHOD) or new_or_init: + if kind is STATIC_METHOD or kind.new_or_init: self.show_in_signature = False # tp_new (METHOD_NEW) functions are of type newfunc: @@ -3973,7 +3992,7 @@ def render(self, parameter, data): parameter is a clinic.Parameter instance. data is a CRenderData instance. """ - if self.function.kind == STATIC_METHOD: + if self.function.kind is STATIC_METHOD: return self._render_self(parameter, data) @@ -3992,8 +4011,8 @@ def set_template_dict(self, template_dict): kind = self.function.kind cls = self.function.cls - if ((kind in (METHOD_NEW, METHOD_INIT)) and cls and cls.typedef): - if kind == METHOD_NEW: + if kind.new_or_init and cls and cls.typedef: + if kind is METHOD_NEW: type_check = ( '({0} == base_tp || {0}->tp_init == base_tp->tp_init)' ).format(self.name) @@ -4337,7 +4356,7 @@ class DSLParser: parameter_state: int seen_positional_with_default: bool indent: IndentStack - kind: str + kind: FunctionKind coexist: bool parameter_continuation: str preserve_output: bool @@ -4626,7 +4645,7 @@ def state_modulename_name(self, line: str | None) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - if not (existing_function.kind == self.kind and existing_function.coexist == self.coexist): + if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): fail("'kind' of function and cloned function don't match! (@classmethod/@staticmethod/@coexist)") function = existing_function.copy( name=function_name, full_name=full_name, module=module, @@ -4679,11 +4698,11 @@ def state_modulename_name(self, line: str | None) -> None: fail(f"{fields[-1]} is a special method and cannot be converted to Argument Clinic! (Yet.)") if fields[-1] == '__new__': - if (self.kind != CLASS_METHOD) or (not cls): + if (self.kind is not CLASS_METHOD) or (not cls): fail("__new__ must be a class method!") self.kind = METHOD_NEW elif fields[-1] == '__init__': - if (self.kind != CALLABLE) or (not cls): + if (self.kind is not CALLABLE) or (not cls): fail("__init__ must be a normal method, not a class or static method!") self.kind = METHOD_INIT if not return_converter: @@ -5203,7 +5222,7 @@ def state_function_docstring(self, line): def format_docstring(self): f = self.function - new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) + new_or_init = f.kind.new_or_init if new_or_init and not f.docstring: # don't render a docstring at all, no signature, nothing. return f.docstring From 025995feadaeebeef5d808f2564f0fd65b704ea5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 13 Jul 2023 17:27:35 -0700 Subject: [PATCH 17/75] gh-106529: Split FOR_ITER_{LIST,TUPLE} into uops (#106696) Also rename `_ITER_EXHAUSTED_XXX` to `_IS_ITER_EXHAUSTED_XXX` to make it clear this is a test. --- Include/internal/pycore_opcode_metadata.h | 26 +++-- Lib/test/test_capi/test_misc.py | 47 +++++++- Python/bytecodes.c | 129 ++++++++++++++++------ Python/executor_cases.c.h | 76 ++++++++++++- Python/generated_cases.c.h | 124 +++++++++++++-------- Python/optimizer.c | 87 +++++++++++---- 6 files changed, 378 insertions(+), 111 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c88640777e3fb0..e94732b64384b5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -39,12 +39,18 @@ #define _GUARD_TYPE_VERSION 317 #define _CHECK_MANAGED_OBJECT_HAS_VALUES 318 #define IS_NONE 319 -#define _ITER_CHECK_RANGE 320 -#define _ITER_EXHAUSTED_RANGE 321 -#define _ITER_NEXT_RANGE 322 -#define _POP_JUMP_IF_FALSE 323 -#define _POP_JUMP_IF_TRUE 324 -#define JUMP_TO_TOP 325 +#define _ITER_CHECK_LIST 320 +#define _IS_ITER_EXHAUSTED_LIST 321 +#define _ITER_NEXT_LIST 322 +#define _ITER_CHECK_TUPLE 323 +#define _IS_ITER_EXHAUSTED_TUPLE 324 +#define _ITER_NEXT_TUPLE 325 +#define _ITER_CHECK_RANGE 326 +#define _IS_ITER_EXHAUSTED_RANGE 327 +#define _ITER_NEXT_RANGE 328 +#define _POP_JUMP_IF_FALSE 329 +#define _POP_JUMP_IF_TRUE 330 +#define JUMP_TO_TOP 331 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -1323,8 +1329,14 @@ const char * const _PyOpcode_uop_name[512] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [IS_NONE] = "IS_NONE", + [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", + [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_IS_ITER_EXHAUSTED_TUPLE] = "_IS_ITER_EXHAUSTED_TUPLE", + [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", - [_ITER_EXHAUSTED_RANGE] = "_ITER_EXHAUSTED_RANGE", + [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index abdf7ed8976350..43c04463236a2a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2590,7 +2590,6 @@ def testfunc(n): for i in range(n): total += i return total - # import dis; dis.dis(testfunc) opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): @@ -2602,7 +2601,51 @@ def testfunc(n): # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") uops = {opname for opname, _ in ex} - self.assertIn("_ITER_EXHAUSTED_RANGE", uops) + self.assertIn("_IS_ITER_EXHAUSTED_RANGE", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_list(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = list(range(10)) + total = testfunc(a) + self.assertEqual(total, 45) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _ in ex} + self.assertIn("_IS_ITER_EXHAUSTED_LIST", uops) + # Verification that the jump goes past END_FOR + # is done by manual inspection of the output + + def test_for_iter_tuple(self): + def testfunc(a): + total = 0 + for i in a: + total += i + return total + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = tuple(range(10)) + total = testfunc(a) + self.assertEqual(total, 45) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + # for i, (opname, oparg) in enumerate(ex): + # print(f"{i:4d}: {opname:<20s} {oparg:3d}") + uops = {opname for opname, _ in ex} + self.assertIn("_IS_ITER_EXHAUSTED_TUPLE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1fe9970e53cdfe..15b48ae9d82672 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -17,6 +17,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES +#include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -55,13 +56,14 @@ static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2; static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys; -static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter; +static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted; static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals; static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from; static PyObject **pieces, **values; static size_t jump; // Dummy variables for cache effects static uint16_t invert, counter, index, hint; +#define unused 0 // Used in a macro def, can't be static static uint32_t type_version; static PyObject * @@ -2418,52 +2420,108 @@ dummy_func( INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); } - inst(FOR_ITER_LIST, (unused/1, iter -- iter, next)) { + op(_ITER_CHECK_LIST, (iter -- iter)) { DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + } + + op(_ITER_JUMP_LIST, (iter -- iter)) { _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyList_GET_SIZE(seq)) { - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_list; // End of this instruction + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); } - it->it_seq = NULL; - Py_DECREF(seq); + Py_DECREF(iter); + STACK_SHRINK(1); + SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - end_for_iter_list: - // Common case: no jump, leave it to the code generator } - inst(FOR_ITER_TUPLE, (unused/1, iter -- iter, next)) { + // Only used by Tier 2 + op(_IS_ITER_EXHAUSTED_LIST, (iter -- iter, exhausted)) { + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + exhausted = Py_True; + } + else { + exhausted = Py_False; + } + } + + op(_ITER_NEXT_LIST, (iter -- iter, next)) { + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + + macro(FOR_ITER_LIST) = + unused/1 + // Skip over the counter + _ITER_CHECK_LIST + + _ITER_JUMP_LIST + + _ITER_NEXT_LIST; + + op(_ITER_CHECK_TUPLE, (iter -- iter)) { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + + op(_ITER_JUMP_TUPLE, (iter -- iter)) { _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyTuple_GET_SIZE(seq)) { - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_tuple; // End of this instruction + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); } - it->it_seq = NULL; - Py_DECREF(seq); + Py_DECREF(iter); + STACK_SHRINK(1); + SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - end_for_iter_tuple: - // Common case: no jump, leave it to the code generator } + // Only used by Tier 2 + op(_IS_ITER_EXHAUSTED_TUPLE, (iter -- iter, exhausted)) { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + exhausted = Py_True; + } + else { + exhausted = Py_False; + } + } + + op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + + macro(FOR_ITER_TUPLE) = + unused/1 + // Skip over the counter + _ITER_CHECK_TUPLE + + _ITER_JUMP_TUPLE + + _ITER_NEXT_TUPLE; + op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); @@ -2484,7 +2542,7 @@ dummy_func( } // Only used by Tier 2 - op(_ITER_EXHAUSTED_RANGE, (iter -- iter, exhausted)) { + op(_IS_ITER_EXHAUSTED_RANGE, (iter -- iter, exhausted)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); exhausted = r->len <= 0 ? Py_True : Py_False; @@ -2502,7 +2560,10 @@ dummy_func( } macro(FOR_ITER_RANGE) = - unused/1 + _ITER_CHECK_RANGE + _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; + unused/1 + // Skip over the counter + _ITER_CHECK_RANGE + + _ITER_JUMP_RANGE + + _ITER_NEXT_RANGE; inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ce54755d5d25f1..626baece814607 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1738,6 +1738,80 @@ break; } + case _ITER_CHECK_LIST: { + PyObject *iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + break; + } + + case _IS_ITER_EXHAUSTED_LIST: { + PyObject *iter = stack_pointer[-1]; + PyObject *exhausted; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + exhausted = Py_True; + } + else { + exhausted = Py_False; + } + STACK_GROW(1); + stack_pointer[-1] = exhausted; + break; + } + + case _ITER_NEXT_LIST: { + PyObject *iter = stack_pointer[-1]; + PyObject *next; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + STACK_GROW(1); + stack_pointer[-1] = next; + break; + } + + case _ITER_CHECK_TUPLE: { + PyObject *iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + break; + } + + case _IS_ITER_EXHAUSTED_TUPLE: { + PyObject *iter = stack_pointer[-1]; + PyObject *exhausted; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + exhausted = Py_True; + } + else { + exhausted = Py_False; + } + STACK_GROW(1); + stack_pointer[-1] = exhausted; + break; + } + + case _ITER_NEXT_TUPLE: { + PyObject *iter = stack_pointer[-1]; + PyObject *next; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + STACK_GROW(1); + stack_pointer[-1] = next; + break; + } + case _ITER_CHECK_RANGE: { PyObject *iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; @@ -1745,7 +1819,7 @@ break; } - case _ITER_EXHAUSTED_RANGE: { + case _IS_ITER_EXHAUSTED_RANGE: { PyObject *iter = stack_pointer[-1]; PyObject *exhausted; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d43c7386bd6f6d..68531dc074769e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3051,60 +3051,96 @@ } TARGET(FOR_ITER_LIST) { - PyObject *iter = stack_pointer[-1]; - PyObject *next; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - _PyListIterObject *it = (_PyListIterObject *)iter; - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyList_GET_SIZE(seq)) { - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_list; // End of this instruction + PyObject *_tmp_1; + PyObject *_tmp_2 = stack_pointer[-1]; + { + PyObject *iter = _tmp_2; + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + _tmp_2 = iter; + } + { + PyObject *iter = _tmp_2; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - it->it_seq = NULL; - Py_DECREF(seq); + _tmp_2 = iter; + } + { + PyObject *iter = _tmp_2; + PyObject *next; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + _tmp_2 = iter; + _tmp_1 = next; } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - end_for_iter_list: - // Common case: no jump, leave it to the code generator - STACK_GROW(1); - stack_pointer[-1] = next; next_instr += 1; + STACK_GROW(1); + stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = _tmp_2; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - PyObject *iter = stack_pointer[-1]; - PyObject *next; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq) { - if (it->it_index < PyTuple_GET_SIZE(seq)) { - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - goto end_for_iter_tuple; // End of this instruction + PyObject *_tmp_1; + PyObject *_tmp_2 = stack_pointer[-1]; + { + PyObject *iter = _tmp_2; + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + _tmp_2 = iter; + } + { + PyObject *iter = _tmp_2; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - it->it_seq = NULL; - Py_DECREF(seq); + _tmp_2 = iter; + } + { + PyObject *iter = _tmp_2; + PyObject *next; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + _tmp_2 = iter; + _tmp_1 = next; } - Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - end_for_iter_tuple: - // Common case: no jump, leave it to the code generator - STACK_GROW(1); - stack_pointer[-1] = next; next_instr += 1; + STACK_GROW(1); + stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = _tmp_2; DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index abd2351f6b78bd..289b202f806ae1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -378,6 +378,7 @@ translate_bytecode_to_trace( _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; + int reserved = 0; #ifdef Py_DEBUG char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); @@ -385,6 +386,9 @@ translate_bytecode_to_trace( if (uop_debug != NULL && *uop_debug >= '0') { lltrace = *uop_debug - '0'; // TODO: Parse an int and all that } +#endif + +#ifdef Py_DEBUG #define DPRINTF(level, ...) \ if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } #else @@ -397,6 +401,8 @@ translate_bytecode_to_trace( uop_name(OPCODE), \ (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ + assert(reserved > 0); \ + reserved--; \ trace[trace_length].opcode = (OPCODE); \ trace[trace_length].operand = (OPERAND); \ trace_length++; @@ -409,9 +415,23 @@ translate_bytecode_to_trace( (INDEX), \ uop_name(OPCODE), \ (uint64_t)(OPERAND)); \ + assert(reserved > 0); \ + reserved--; \ trace[(INDEX)].opcode = (OPCODE); \ trace[(INDEX)].operand = (OPERAND); +// Reserve space for n uops +#define RESERVE_RAW(n, opname) \ + if (trace_length + (n) > max_length) { \ + DPRINTF(2, "No room for %s (need %d, got %d)\n", \ + (opname), (n), max_length - trace_length); \ + goto done; \ + } \ + reserved = (n); // Keep ADD_TO_TRACE / ADD_TO_STUB honest + +// Reserve space for main+stub uops, plus 2 for SAVE_IP and EXIT_TRACE +#define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 2, uop_name(opcode)) + DPRINTF(4, "Optimizing %s (%s:%d) at byte offset %ld\n", PyUnicode_AsUTF8(code->co_qualname), @@ -420,16 +440,20 @@ translate_bytecode_to_trace( 2 * INSTR_IP(initial_instr, code)); for (;;) { + RESERVE_RAW(2, "epilogue"); // Always need space for SAVE_IP and EXIT_TRACE ADD_TO_TRACE(SAVE_IP, INSTR_IP(instr, code)); + int opcode = instr->op.code; int oparg = instr->op.arg; int extras = 0; + while (opcode == EXTENDED_ARG) { instr++; extras += 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; } + if (opcode == ENTER_EXECUTOR) { _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; @@ -437,17 +461,14 @@ translate_bytecode_to_trace( DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); oparg = (oparg & 0xffffff00) | executor->vm_data.oparg; } + switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { // Assume jump unlikely (TODO: handle jump likely case) - // Reserve 5 entries (1 here, 2 stub, plus SAVE_IP + EXIT_TRACE) - if (trace_length + 5 > max_length) { - DPRINTF(1, "Ran out of space for POP_JUMP_IF_FALSE\n"); - goto done; - } + RESERVE(1, 2); _Py_CODEUNIT *target_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg; max_length -= 2; // Really the start of the stubs @@ -461,9 +482,8 @@ translate_bytecode_to_trace( case JUMP_BACKWARD: { - if (instr + 2 - oparg == initial_instr - && trace_length + 3 <= max_length) - { + if (instr + 2 - oparg == initial_instr) { + RESERVE(1, 0); ADD_TO_TRACE(JUMP_TO_TOP, 0); } else { @@ -474,26 +494,45 @@ translate_bytecode_to_trace( case JUMP_FORWARD: { + RESERVE(0, 0); // This will emit two SAVE_IP instructions; leave it to the optimizer instr += oparg; break; } + case FOR_ITER_LIST: + case FOR_ITER_TUPLE: case FOR_ITER_RANGE: { - // Assume jump unlikely (can a for-loop exit be likely?) - // Reserve 9 entries (4 here, 3 stub, plus SAVE_IP + EXIT_TRACE) - if (trace_length + 9 > max_length) { - DPRINTF(1, "Ran out of space for FOR_ITER_RANGE\n"); - goto done; + RESERVE(4, 3); + int check_op, exhausted_op, next_op; + switch (opcode) { + case FOR_ITER_LIST: + check_op = _ITER_CHECK_LIST; + exhausted_op = _IS_ITER_EXHAUSTED_LIST; + next_op = _ITER_NEXT_LIST; + break; + case FOR_ITER_TUPLE: + check_op = _ITER_CHECK_TUPLE; + exhausted_op = _IS_ITER_EXHAUSTED_TUPLE; + next_op = _ITER_NEXT_TUPLE; + break; + case FOR_ITER_RANGE: + check_op = _ITER_CHECK_RANGE; + exhausted_op = _IS_ITER_EXHAUSTED_RANGE; + next_op = _ITER_NEXT_RANGE; + break; + default: + Py_UNREACHABLE(); } + // Assume jump unlikely (can a for-loop exit be likely?) _Py_CODEUNIT *target_instr = // +1 at the end skips over END_FOR instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg + 1; max_length -= 3; // Really the start of the stubs - ADD_TO_TRACE(_ITER_CHECK_RANGE, 0); - ADD_TO_TRACE(_ITER_EXHAUSTED_RANGE, 0); + ADD_TO_TRACE(check_op, 0); + ADD_TO_TRACE(exhausted_op, 0); ADD_TO_TRACE(_POP_JUMP_IF_TRUE, max_length); - ADD_TO_TRACE(_ITER_NEXT_RANGE, 0); + ADD_TO_TRACE(next_op, 0); ADD_TO_STUB(max_length + 0, POP_TOP, 0); ADD_TO_STUB(max_length + 1, SAVE_IP, INSTR_IP(target_instr, code)); @@ -507,10 +546,7 @@ translate_bytecode_to_trace( if (expansion->nuops > 0) { // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE) int nuops = expansion->nuops; - if (trace_length + nuops + 2 > max_length) { - DPRINTF(1, "Ran out of space for %s\n", uop_name(opcode)); - goto done; - } + RESERVE(nuops, 0); for (int i = 0; i < nuops; i++) { uint64_t operand; int offset = expansion->uops[i].offset; @@ -556,12 +592,14 @@ translate_bytecode_to_trace( } DPRINTF(2, "Unsupported opcode %s\n", uop_name(opcode)); goto done; // Break out of loop - } - } + } // End default + + } // End switch (opcode) + instr++; // Add cache size for opcode instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - } + } // End for (;;) done: // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE @@ -610,6 +648,9 @@ translate_bytecode_to_trace( } return 0; +#undef RESERVE +#undef RESERVE_RAW +#undef INSTR_IP #undef ADD_TO_TRACE #undef DPRINTF } From 490295d651d04ec3b3eff2a2cda7501191bad78a Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 14 Jul 2023 09:55:49 +0300 Subject: [PATCH 18/75] gh-105626: Change the default return value of `HTTPConnection.get_proxy_response_headers` (#105628) --- Doc/library/http.client.rst | 2 +- Lib/http/client.py | 5 ++--- Lib/test/test_httplib.py | 13 +++++++++++++ .../2023-06-10-12-20-17.gh-issue-105626.XyZein.rst | 3 +++ 4 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 45291933d635b9..b9ceab699cef63 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -390,7 +390,7 @@ HTTPConnection Objects Returns a dictionary with the headers of the response received from the proxy server to the CONNECT request. - If the CONNECT request was not sent, the method returns an empty dictionary. + If the CONNECT request was not sent, the method returns ``None``. .. versionadded:: 3.12 diff --git a/Lib/http/client.py b/Lib/http/client.py index 3d98e4eb54bb45..b35b1d6368aae7 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -970,13 +970,12 @@ def get_proxy_response_headers(self): received from the proxy server to the CONNECT request sent to set the tunnel. - If the CONNECT request was not sent, the method returns - an empty dictionary. + If the CONNECT request was not sent, the method returns None. """ return ( _parse_header_lines(self._raw_proxy_headers) if self._raw_proxy_headers is not None - else {} + else None ) def connect(self): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8955d45fa93dd4..fe8105ee2bb3fa 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2404,6 +2404,19 @@ def test_proxy_response_headers(self): headers = self.conn.get_proxy_response_headers() self.assertIn(expected_header, headers.items()) + def test_no_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIsNone(headers) + def test_tunnel_leak(self): sock = None diff --git a/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst b/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst new file mode 100644 index 00000000000000..2a48361fa596c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst @@ -0,0 +1,3 @@ +Change the default return value of +:meth:`http.client.HTTPConnection.get_proxy_response_headers` to be ``None`` +and not ``{}``. From 21d98be42289369ccfbdcc38574cb9ab50ce1c02 Mon Sep 17 00:00:00 2001 From: Grant Ramsay Date: Fri, 14 Jul 2023 19:10:54 +1200 Subject: [PATCH 19/75] gh-105293: Do not call SSL_CTX_set_session_id_context on client side SSL context (#105295) * gh-105293: Do not call SSL_CTX_set_session_id_context on client side SSL context Openssl states this is a "server side only" operation. Calling this on a client side socket can result in unexpected behavior * Add news entry on SSL "set session id context" changes --- .../2023-07-14-14-53-58.gh-issue-105293.kimf_i.rst | 2 ++ Modules/_ssl.c | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-14-14-53-58.gh-issue-105293.kimf_i.rst diff --git a/Misc/NEWS.d/next/Library/2023-07-14-14-53-58.gh-issue-105293.kimf_i.rst b/Misc/NEWS.d/next/Library/2023-07-14-14-53-58.gh-issue-105293.kimf_i.rst new file mode 100644 index 00000000000000..c263c8524aa962 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-14-14-53-58.gh-issue-105293.kimf_i.rst @@ -0,0 +1,2 @@ +Remove call to ``SSL_CTX_set_session_id_context`` during client side context +creation in the :mod:`ssl` module. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index df1496925f678e..571de331e92cd9 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -847,6 +847,15 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; } + + if (socket_type == PY_SSL_SERVER) { +#define SID_CTX "Python" + /* Set the session id context (server-side only) */ + SSL_set_session_id_context(self->ssl, (const unsigned char *) SID_CTX, + sizeof(SID_CTX)); +#undef SID_CTX + } + /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ #if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION < 0x101010cf X509_VERIFY_PARAM *ssl_params = SSL_get0_param(self->ssl); @@ -3186,11 +3195,6 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) usage for no cost at all. */ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -#define SID_CTX "Python" - SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, - sizeof(SID_CTX)); -#undef SID_CTX - params = SSL_CTX_get0_param(self->ctx); /* Improve trust chain building when cross-signed intermediate certificates are present. See https://bugs.python.org/issue23476. */ From 89867d2491c0c3ef77bc237899b2f0762f43c03c Mon Sep 17 00:00:00 2001 From: Charlie Zhao Date: Fri, 14 Jul 2023 15:38:03 +0800 Subject: [PATCH 20/75] gh-106446: Fix failed doctest in stdtypes (#106447) --------- Co-authored-by: Terry Jan Reedy --- Doc/library/stdtypes.rst | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8e049d9645c0b6..fd51b1187576b1 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3953,7 +3953,7 @@ copying. >>> m = memoryview(bytearray(b'abc')) >>> mm = m.toreadonly() >>> mm.tolist() - [89, 98, 99] + [97, 98, 99] >>> mm[0] = 42 Traceback (most recent call last): File "", line 1, in @@ -4009,6 +4009,7 @@ copying. :mod:`struct` syntax. One of the formats must be a byte format ('B', 'b' or 'c'). The byte length of the result must be the same as the original length. + Note that all byte lengths may depend on the operating system. Cast 1D/long to 1D/unsigned bytes:: @@ -4039,8 +4040,8 @@ copying. >>> x = memoryview(b) >>> x[0] = b'a' Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview: invalid value for format "B" + ... + TypeError: memoryview: invalid type for format 'B' >>> y = x.cast('c') >>> y[0] = b'a' >>> b @@ -4789,10 +4790,10 @@ An example of dictionary view usage:: >>> # set operations >>> keys & {'eggs', 'bacon', 'salad'} {'bacon'} - >>> keys ^ {'sausage', 'juice'} - {'juice', 'sausage', 'bacon', 'spam'} - >>> keys | ['juice', 'juice', 'juice'] - {'juice', 'sausage', 'bacon', 'spam', 'eggs'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True >>> # get back a read-only proxy for the original dictionary >>> values.mapping @@ -4999,8 +5000,8 @@ exception to disallow mistakes like ``dict[str][str]``:: >>> dict[str][str] Traceback (most recent call last): - File "", line 1, in - TypeError: There are no type variables left in dict[str] + ... + TypeError: dict[str] is not a generic class However, such expressions are valid when :ref:`type variables ` are used. The index must have as many elements as there are type variable items @@ -5206,13 +5207,15 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. >>> isinstance("", int | str) True - However, union objects containing :ref:`parameterized generics - ` cannot be used:: + However, :ref:`parameterized generics ` in + union objects cannot be checked:: - >>> isinstance(1, int | list[int]) + >>> isinstance(1, int | list[int]) # short-circuit evaluation + True + >>> isinstance([1], int | list[int]) Traceback (most recent call last): - File "", line 1, in - TypeError: isinstance() argument 2 cannot contain a parameterized generic + ... + TypeError: isinstance() argument 2 cannot be a parameterized generic The user-exposed type for the union object can be accessed from :data:`types.UnionType` and used for :func:`isinstance` checks. An object cannot be @@ -5515,7 +5518,7 @@ types, where they are relevant. Some of these are not reported by the definition order. Example:: >>> int.__subclasses__() - [] + [, , , ] .. _int_max_str_digits: @@ -5551,7 +5554,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> _ = int('2' * 5432) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit >>> i = int('2' * 4300) >>> len(str(i)) 4300 @@ -5559,7 +5562,7 @@ When an operation would exceed the limit, a :exc:`ValueError` is raised: >>> len(str(i_squared)) Traceback (most recent call last): ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 8599 digits; use sys.set_int_max_str_digits() to increase the limit. + ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit >>> len(hex(i_squared)) 7144 >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. From 7c95345e4f93f4a2475418f17df5aae39dea861f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 14 Jul 2023 14:20:58 +0200 Subject: [PATCH 21/75] gh-104050: Argument Clinic: Annotate `output_templates()` (#106732) Co-authored-by: AlexWaygood --- Tools/clinic/clinic.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2dce33690993dd..726ebc04f55bd5 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -792,11 +792,14 @@ def docstring_for_c_string( add('"') return ''.join(text) - def output_templates(self, f): + def output_templates( + self, + f: Function + ) -> dict[str, str]: parameters = list(f.parameters.values()) assert parameters - assert isinstance(parameters[0].converter, self_converter) - del parameters[0] + first_param = parameters.pop(0) + assert isinstance(first_param.converter, self_converter) requires_defining_class = False if parameters and isinstance(parameters[0].converter, defining_class_converter): requires_defining_class = True @@ -809,7 +812,7 @@ def output_templates(self, f): new_or_init = f.kind.new_or_init - vararg = NO_VARARG + vararg: int | str = NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): if p.is_keyword_only(): @@ -897,7 +900,7 @@ def output_templates(self, f): # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. - parser_body_fields = () + parser_body_fields: tuple[str, ...] = () def parser_body( prototype: str, *fields: str, @@ -932,6 +935,7 @@ def parser_body( return linear_format(output(), parser_declarations=declarations) if not parameters: + parser_code: list[str] | None if not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" @@ -1165,7 +1169,7 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = parser_prototype_def_class - add_label = None + add_label: str | None = None for i, p in enumerate(parameters): if isinstance(p.converter, defining_class_converter): raise ValueError("defining_class should be the first " @@ -1308,6 +1312,8 @@ def parser_body( cpp_if = "#if " + conditional cpp_endif = "#endif /* " + conditional + " */" + assert clinic is not None + assert f.full_name is not None if methoddef_define and f.full_name not in clinic.ifndef_symbols: clinic.ifndef_symbols.add(f.full_name) methoddef_ifndef = normalize_snippet(""" From 243fdcb40ebeb177ce723911c1f7fad8a1fdf6cb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 14 Jul 2023 13:38:28 -0400 Subject: [PATCH 22/75] gh-106531: Remove importlib.resources._legacy (#106532) * gh-106531: Remove importlib.resources._legacy Syncs with importlib_resources 6.0. * Remove documentation for removed functionality. --- Doc/library/importlib.resources.rst | 156 ------------------ Lib/importlib/resources/__init__.py | 19 --- Lib/importlib/resources/_legacy.py | 120 -------------- ...-07-07-16-19-59.gh-issue-106531.eMfNm8.rst | 3 + 4 files changed, 3 insertions(+), 295 deletions(-) delete mode 100644 Lib/importlib/resources/_legacy.py create mode 100644 Misc/NEWS.d/next/Library/2023-07-07-16-19-59.gh-issue-106531.eMfNm8.rst diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 755693840fecd8..76faf731144779 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -94,159 +94,3 @@ for example, a package and its resources can be imported from a zip file using the file system is required. .. versionadded:: 3.9 - - -Deprecated functions -^^^^^^^^^^^^^^^^^^^^ - -An older, deprecated set of functions is still available, but is -scheduled for removal in a future version of Python. -The main drawback of these functions is that they do not support -directories: they assume all resources are located directly within a *package*. - -.. data:: Package - - Whenever a function accepts a ``Package`` argument, you can pass in - either a :class:`module object ` or a module name - as a string. You can only pass module objects whose - ``__spec__.submodule_search_locations`` is not ``None``. - - The ``Package`` type is defined as ``Union[str, ModuleType]``. - - .. deprecated:: 3.12 - - -.. data:: Resource - - For *resource* arguments of the functions below, you can pass in - the name of a resource as a string or - a :class:`path-like object `. - - The ``Resource`` type is defined as ``Union[str, os.PathLike]``. - - -.. function:: open_binary(package, resource) - - Open for binary reading the *resource* within *package*. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). This function returns a - ``typing.BinaryIO`` instance, a binary I/O stream open for reading. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).open('rb') - - -.. function:: open_text(package, resource, encoding='utf-8', errors='strict') - - Open for text reading the *resource* within *package*. By default, the - resource is opened for reading as UTF-8. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). *encoding* and *errors* - have the same meaning as with built-in :func:`open`. - - This function returns a ``typing.TextIO`` instance, a text I/O stream open - for reading. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).open('r', encoding=encoding) - - -.. function:: read_binary(package, resource) - - Read and return the contents of the *resource* within *package* as - ``bytes``. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). This function returns the - contents of the resource as :class:`bytes`. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).read_bytes() - - -.. function:: read_text(package, resource, encoding='utf-8', errors='strict') - - Read and return the contents of *resource* within *package* as a ``str``. - By default, the contents are read as strict UTF-8. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). *encoding* and *errors* - have the same meaning as with built-in :func:`open`. This function - returns the contents of the resource as :class:`str`. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).read_text(encoding=encoding) - - -.. function:: path(package, resource) - - Return the path to the *resource* as an actual file system path. This - function returns a context manager for use in a :keyword:`with` statement. - The context manager provides a :class:`pathlib.Path` object. - - Exiting the context manager cleans up any temporary file created when the - resource needs to be extracted from e.g. a zip file. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. *resource* is the name of the resource to open - within *package*; it may not contain path separators and it may not have - sub-resources (i.e. it cannot be a directory). - - .. deprecated:: 3.11 - - Calls to this function can be replaced using :func:`as_file`:: - - as_file(files(package).joinpath(resource)) - - -.. function:: is_resource(package, name) - - Return ``True`` if there is a resource named *name* in the package, - otherwise ``False``. - This function does not consider directories to be resources. - *package* is either a name or a module object which conforms to the - ``Package`` requirements. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - files(package).joinpath(resource).is_file() - - -.. function:: contents(package) - - Return an iterable over the named items within the package. The iterable - returns :class:`str` resources (e.g. files) and non-resources - (e.g. directories). The iterable does not recurse into subdirectories. - - *package* is either a name or a module object which conforms to the - ``Package`` requirements. - - .. deprecated:: 3.11 - - Calls to this function can be replaced by:: - - (resource.name for resource in files(package).iterdir() if resource.is_file()) diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 34e3a9950cc557..e6b60c18caa052 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -6,31 +6,12 @@ Package, ) -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, - Resource, -) - from .abc import ResourceReader __all__ = [ 'Package', - 'Resource', 'ResourceReader', 'as_file', - 'contents', 'files', - 'is_resource', - 'open_binary', - 'open_text', - 'path', - 'read_binary', - 'read_text', ] diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105dad6e2..00000000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Misc/NEWS.d/next/Library/2023-07-07-16-19-59.gh-issue-106531.eMfNm8.rst b/Misc/NEWS.d/next/Library/2023-07-07-16-19-59.gh-issue-106531.eMfNm8.rst new file mode 100644 index 00000000000000..a52107103c4576 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-16-19-59.gh-issue-106531.eMfNm8.rst @@ -0,0 +1,3 @@ +Removed ``_legacy`` and the names it provided from ``importlib.resources``: +``Resource``, ``contents``, ``is_resource``, ``open_binary``, ``open_text``, +``path``, ``read_binary``, and ``read_text``. From 6a70edf24ca217c5ed4a556d0df5748fc775c762 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 14 Jul 2023 18:41:52 +0100 Subject: [PATCH 23/75] gh-105481: expose opcode metadata via the _opcode module (#106688) --- Include/cpython/compile.h | 6 + Include/internal/pycore_opcode_metadata.h | 32 +- Lib/test/test__opcode.py | 64 ++++ ...-07-13-16-04-15.gh-issue-105481.pYSwMj.rst | 1 + Modules/_opcode.c | 91 +++++ Modules/clinic/_opcode.c.h | 317 +++++++++++++++++- Python/compile.c | 32 ++ Python/flowgraph.c | 4 +- Tools/cases_generator/generate_cases.py | 32 +- 9 files changed, 558 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-13-16-04-15.gh-issue-105481.pYSwMj.rst diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index f5a62a8ec6dd0c..cd7fd7bd377663 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -67,3 +67,9 @@ typedef struct { #define PY_INVALID_STACK_EFFECT INT_MAX PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); + +PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e94732b64384b5..8373f56653b1c7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -3,6 +3,8 @@ // Python/bytecodes.c // Do not edit! +#include + #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ @@ -941,14 +943,20 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { #endif enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC0, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; + +#define IS_VALID_OPCODE(OP) \ + (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ + (_PyOpcode_opcode_metadata[(OP)].valid_entry)) + #define HAS_ARG_FLAG (1) #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) #define HAS_JUMP_FLAG (8) -#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_ARG_FLAG)) -#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_CONST_FLAG)) -#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_NAME_FLAG)) -#define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[(OP)].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) +#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) +#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) +#define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) + struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -971,12 +979,16 @@ struct opcode_macro_expansion { #define SAME_OPCODE_METADATA(OP1, OP2) \ (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2)) +#define OPCODE_METADATA_SIZE 512 +#define OPCODE_UOP_NAME_SIZE 512 +#define OPCODE_MACRO_EXPANSION_SIZE 256 + #ifndef NEED_OPCODE_METADATA -extern const struct opcode_metadata _PyOpcode_opcode_metadata[512]; -extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; -extern const char * const _PyOpcode_uop_name[512]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; +extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; #else // if NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { +const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1194,7 +1206,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { [CACHE] = { true, INSTR_FMT_IX, 0 }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, }; -const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { +const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, @@ -1308,7 +1320,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; -const char * const _PyOpcode_uop_name[512] = { +const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [EXIT_TRACE] = "EXIT_TRACE", [SAVE_IP] = "SAVE_IP", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 3e084928844d2f..7d9553d9e383a3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -9,6 +9,70 @@ class OpcodeTests(unittest.TestCase): + def check_bool_function_result(self, func, ops, expected): + for op in ops: + if isinstance(op, str): + op = dis.opmap[op] + with self.subTest(opcode=op, func=func): + self.assertIsInstance(func(op), bool) + self.assertEqual(func(op), expected) + + def test_invalid_opcodes(self): + invalid = [-100, -1, 255, 512, 513, 1000] + self.check_bool_function_result(_opcode.is_valid, invalid, False) + self.check_bool_function_result(_opcode.has_arg, invalid, False) + self.check_bool_function_result(_opcode.has_const, invalid, False) + self.check_bool_function_result(_opcode.has_name, invalid, False) + self.check_bool_function_result(_opcode.has_jump, invalid, False) + + def test_is_valid(self): + names = [ + 'CACHE', + 'POP_TOP', + 'IMPORT_NAME', + 'JUMP', + 'INSTRUMENTED_RETURN_VALUE', + ] + opcodes = [dis.opmap[opname] for opname in names] + self.check_bool_function_result(_opcode.is_valid, opcodes, True) + + def test_has_arg(self): + has_arg = ['SWAP', 'LOAD_FAST', 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'JUMP'] + no_arg = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_arg, has_arg, True) + self.check_bool_function_result(_opcode.has_arg, no_arg, False) + + def test_has_const(self): + has_const = ['LOAD_CONST', 'RETURN_CONST', 'KW_NAMES'] + no_const = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_const, has_const, True) + self.check_bool_function_result(_opcode.has_const, no_const, False) + + def test_has_name(self): + has_name = ['STORE_NAME', 'DELETE_ATTR', 'STORE_GLOBAL', 'IMPORT_FROM', + 'LOAD_FROM_DICT_OR_GLOBALS'] + no_name = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_name, has_name, True) + self.check_bool_function_result(_opcode.has_name, no_name, False) + + def test_has_jump(self): + has_jump = ['FOR_ITER', 'JUMP_FORWARD', 'JUMP', 'POP_JUMP_IF_TRUE', 'SEND'] + no_jump = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_jump, has_jump, True) + self.check_bool_function_result(_opcode.has_jump, no_jump, False) + + # the following test is part of the refactor, it will be removed soon + def test_against_legacy_bool_values(self): + # limiting to ops up to ENTER_EXECUTOR, because everything after that + # is not currently categorized correctly in opcode.py. + for op in range(0, opcode.opmap['ENTER_EXECUTOR']): + with self.subTest(op=op): + if opcode.opname[op] != f'<{op}>': + self.assertEqual(op in dis.hasarg, _opcode.has_arg(op)) + self.assertEqual(op in dis.hasconst, _opcode.has_const(op)) + self.assertEqual(op in dis.hasname, _opcode.has_name(op)) + self.assertEqual(op in dis.hasjrel, _opcode.has_jump(op)) + def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) diff --git a/Misc/NEWS.d/next/Library/2023-07-13-16-04-15.gh-issue-105481.pYSwMj.rst b/Misc/NEWS.d/next/Library/2023-07-13-16-04-15.gh-issue-105481.pYSwMj.rst new file mode 100644 index 00000000000000..bc2ba51d31aa9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-13-16-04-15.gh-issue-105481.pYSwMj.rst @@ -0,0 +1 @@ +Expose opcode metadata through :mod:`_opcode`. diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 3c0fce48770ac4..b3b9873d21a5be 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "compile.h" #include "opcode.h" #include "internal/pycore_code.h" @@ -61,6 +62,91 @@ _opcode_stack_effect_impl(PyObject *module, int opcode, PyObject *oparg, /*[clinic input] +_opcode.is_valid -> bool + + opcode: int + +Return True if opcode is valid, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_is_valid_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ +{ + return PyUnstable_OpcodeIsValid(opcode); +} + +/*[clinic input] + +_opcode.has_arg -> bool + + opcode: int + +Return True if the opcode uses its oparg, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_arg_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasArg(opcode); +} + +/*[clinic input] + +_opcode.has_const -> bool + + opcode: int + +Return True if the opcode accesses a constant, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_const_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasConst(opcode); +} + +/*[clinic input] + +_opcode.has_name -> bool + + opcode: int + +Return True if the opcode accesses an attribute by name, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_name_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasName(opcode); +} + +/*[clinic input] + +_opcode.has_jump -> bool + + opcode: int + +Return True if the opcode has a jump target, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_jump_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasJump(opcode); + +} + +/*[clinic input] + _opcode.get_specialization_stats Return the specialization stats @@ -80,6 +166,11 @@ _opcode_get_specialization_stats_impl(PyObject *module) static PyMethodDef opcode_functions[] = { _OPCODE_STACK_EFFECT_METHODDEF + _OPCODE_IS_VALID_METHODDEF + _OPCODE_HAS_ARG_METHODDEF + _OPCODE_HAS_CONST_METHODDEF + _OPCODE_HAS_NAME_METHODDEF + _OPCODE_HAS_JUMP_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 3bd3ba02387435..3eb050e470c343 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -86,6 +86,321 @@ _opcode_stack_effect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_opcode_is_valid__doc__, +"is_valid($module, /, opcode)\n" +"--\n" +"\n" +"Return True if opcode is valid, False otherwise."); + +#define _OPCODE_IS_VALID_METHODDEF \ + {"is_valid", _PyCFunction_CAST(_opcode_is_valid), METH_FASTCALL|METH_KEYWORDS, _opcode_is_valid__doc__}, + +static int +_opcode_is_valid_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_is_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_valid", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_is_valid_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_arg__doc__, +"has_arg($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode uses its oparg, False otherwise."); + +#define _OPCODE_HAS_ARG_METHODDEF \ + {"has_arg", _PyCFunction_CAST(_opcode_has_arg), METH_FASTCALL|METH_KEYWORDS, _opcode_has_arg__doc__}, + +static int +_opcode_has_arg_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_arg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_arg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_arg_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_const__doc__, +"has_const($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a constant, False otherwise."); + +#define _OPCODE_HAS_CONST_METHODDEF \ + {"has_const", _PyCFunction_CAST(_opcode_has_const), METH_FASTCALL|METH_KEYWORDS, _opcode_has_const__doc__}, + +static int +_opcode_has_const_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_const(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_const", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_const_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_name__doc__, +"has_name($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses an attribute by name, False otherwise."); + +#define _OPCODE_HAS_NAME_METHODDEF \ + {"has_name", _PyCFunction_CAST(_opcode_has_name), METH_FASTCALL|METH_KEYWORDS, _opcode_has_name__doc__}, + +static int +_opcode_has_name_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_name", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_name_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_jump__doc__, +"has_jump($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode has a jump target, False otherwise."); + +#define _OPCODE_HAS_JUMP_METHODDEF \ + {"has_jump", _PyCFunction_CAST(_opcode_has_jump), METH_FASTCALL|METH_KEYWORDS, _opcode_has_jump__doc__}, + +static int +_opcode_has_jump_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_jump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_jump_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_get_specialization_stats__doc__, "get_specialization_stats($module, /)\n" "--\n" @@ -103,4 +418,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=21e3d53a659c651a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae2b2ef56d582180 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index d9e38cfdefaf23..9e86e06777ffa4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -36,7 +36,9 @@ #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() +#define NEED_OPCODE_METADATA #include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed +#undef NEED_OPCODE_METADATA #define COMP_GENEXP 0 #define COMP_LISTCOMP 1 @@ -864,6 +866,36 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return stack_effect(opcode, oparg, -1); } +int +PyUnstable_OpcodeIsValid(int opcode) +{ + return IS_VALID_OPCODE(opcode); +} + +int +PyUnstable_OpcodeHasArg(int opcode) +{ + return OPCODE_HAS_ARG(opcode); +} + +int +PyUnstable_OpcodeHasConst(int opcode) +{ + return OPCODE_HAS_CONST(opcode); +} + +int +PyUnstable_OpcodeHasName(int opcode) +{ + return OPCODE_HAS_NAME(opcode); +} + +int +PyUnstable_OpcodeHasJump(int opcode) +{ + return OPCODE_HAS_JUMP(opcode); +} + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 04f269c5853835..e485ed103147a1 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -7,9 +7,7 @@ #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_opcode_utils.h" -#define NEED_OPCODE_METADATA -#include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed -#undef NEED_OPCODE_METADATA +#include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc #undef SUCCESS diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index ce16271097b955..6589289121863b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -308,7 +308,7 @@ def emit_macros(cls, out: Formatter): for name, value in flags.bitmask.items(): out.emit( f"#define OPCODE_{name[:-len('_FLAG')]}(OP) " - f"(_PyOpcode_opcode_metadata[(OP)].flags & ({name}))") + f"(_PyOpcode_opcode_metadata[OP].flags & ({name}))") @dataclasses.dataclass @@ -1192,6 +1192,8 @@ def write_metadata(self) -> None: self.write_provenance_header() + self.out.emit("\n#include ") + self.write_pseudo_instrs() self.out.emit("") @@ -1202,8 +1204,16 @@ def write_metadata(self) -> None: # Write type definitions self.out.emit(f"enum InstructionFormat {{ {', '.join(format_enums)} }};") + self.out.emit("") + self.out.emit( + "#define IS_VALID_OPCODE(OP) \\\n" + " (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \\\n" + " (_PyOpcode_opcode_metadata[(OP)].valid_entry))") + + self.out.emit("") InstructionFlags.emit_macros(self.out) + self.out.emit("") with self.out.block("struct opcode_metadata", ";"): self.out.emit("bool valid_entry;") self.out.emit("enum InstructionFormat instr_format;") @@ -1226,13 +1236,20 @@ def write_metadata(self) -> None: self.out.emit("") # Write metadata array declaration + self.out.emit("#define OPCODE_METADATA_SIZE 512") + self.out.emit("#define OPCODE_UOP_NAME_SIZE 512") + self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") + self.out.emit("") self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit("extern const struct opcode_metadata _PyOpcode_opcode_metadata[512];") - self.out.emit("extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256];") - self.out.emit("extern const char * const _PyOpcode_uop_name[512];") + self.out.emit("extern const struct opcode_metadata " + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];") + self.out.emit("extern const struct opcode_macro_expansion " + "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];") + self.out.emit("extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];") self.out.emit("#else // if NEED_OPCODE_METADATA") - self.out.emit("const struct opcode_metadata _PyOpcode_opcode_metadata[512] = {") + self.out.emit("const struct opcode_metadata " + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {") # Write metadata for each instruction for thing in self.everything: @@ -1253,7 +1270,8 @@ def write_metadata(self) -> None: self.out.emit("};") with self.out.block( - "const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] =", + "const struct opcode_macro_expansion " + "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] =", ";", ): # Write macro expansion for each non-pseudo instruction @@ -1279,7 +1297,7 @@ def write_metadata(self) -> None: case _: typing.assert_never(thing) - with self.out.block("const char * const _PyOpcode_uop_name[512] =", ";"): + with self.out.block("const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";"): self.write_uop_items(lambda name, counter: f"[{name}] = \"{name}\",") self.out.emit("#endif // NEED_OPCODE_METADATA") From aeef8591e41b68341af308e56a744396c66879cc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 14 Jul 2023 08:46:30 -1000 Subject: [PATCH 24/75] gh-106554: replace `_BaseSelectorImpl._key_from_fd` with `dict.get` (#106555) --- Lib/selectors.py | 21 ++++--------------- ...-07-09-01-59-24.gh-issue-106554.37c53J.rst | 1 + 2 files changed, 5 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-09-01-59-24.gh-issue-106554.37c53J.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index dfcc125dcd94ef..6d82935445d4b1 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -276,19 +276,6 @@ def close(self): def get_map(self): return self._map - def _key_from_fd(self, fd): - """Return the key associated to a given file descriptor. - - Parameters: - fd -- file descriptor - - Returns: - corresponding key, or None if not found - """ - try: - return self._fd_to_key[fd] - except KeyError: - return None class SelectSelector(_BaseSelectorImpl): @@ -336,7 +323,7 @@ def select(self, timeout=None): if fd in w: events |= EVENT_WRITE - key = self._key_from_fd(fd) + key = self._fd_to_key.get(fd) if key: ready.append((key, events & key.events)) return ready @@ -426,7 +413,7 @@ def select(self, timeout=None): if event & ~self._EVENT_WRITE: events |= EVENT_READ - key = self._key_from_fd(fd) + key = self._fd_to_key.get(fd) if key: ready.append((key, events & key.events)) return ready @@ -479,7 +466,7 @@ def select(self, timeout=None): if event & ~select.EPOLLOUT: events |= EVENT_READ - key = self._key_from_fd(fd) + key = self._fd_to_key.get(fd) if key: ready.append((key, events & key.events)) return ready @@ -574,7 +561,7 @@ def select(self, timeout=None): if flag == select.KQ_FILTER_WRITE: events |= EVENT_WRITE - key = self._key_from_fd(fd) + key = self._fd_to_key.get(fd) if key: ready.append((key, events & key.events)) return ready diff --git a/Misc/NEWS.d/next/Library/2023-07-09-01-59-24.gh-issue-106554.37c53J.rst b/Misc/NEWS.d/next/Library/2023-07-09-01-59-24.gh-issue-106554.37c53J.rst new file mode 100644 index 00000000000000..2136f3aa5a8eb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-09-01-59-24.gh-issue-106554.37c53J.rst @@ -0,0 +1 @@ +:mod:`selectors`: Reduce Selector overhead by using a ``dict.get()`` to lookup file descriptors. From 89ec0e952965b6a1be40e26c3ddc4131599e5ee9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 14 Jul 2023 19:49:02 +0100 Subject: [PATCH 25/75] gh-106745: typing docs: Clarify that removal of PEP-585 aliases is not currently planned (#106748) --- Doc/library/typing.rst | 70 +++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 0cf875582f7f42..0265a39ce646f4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3101,6 +3101,7 @@ Constant .. versionadded:: 3.5.2 .. _generic-concrete-collections: +.. _deprecated-aliases: Deprecated aliases ------------------ @@ -3109,16 +3110,21 @@ This module defines several deprecated aliases to pre-existing standard library classes. These were originally included in the typing module in order to support parameterizing these generic classes using ``[]``. However, the aliases became redundant in Python 3.9 when the -corresponding pre-existing classes were enhanced to support ``[]``. +corresponding pre-existing classes were enhanced to support ``[]`` (see +:pep:`585`). -The redundant types are deprecated as of Python 3.9 but no -deprecation warnings are issued by the interpreter. -It is expected that type checkers will flag the deprecated types -when the checked program targets Python 3.9 or newer. +The redundant types are deprecated as of Python 3.9. However, while the aliases +may be removed at some point, removal of these aliases is not currently +planned. As such, no deprecation warnings are currently issued by the +interpreter for these aliases. -The deprecated types will be removed from the :mod:`typing` module -no sooner than the first Python version released 5 years after the release of Python 3.9.0. -See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. +If at some point it is decided to remove these deprecated aliases, a +deprecation warning will be issued by the interpreter for at least two releases +prior to removal. The aliases are guaranteed to remain in the typing module +without deprecation warnings until at least Python 3.14. + +Type checkers are encouraged to flag uses of the deprecated types if the +program they are checking targets a minimum Python version of 3.9 or newer. .. _corresponding-to-built-in-types: @@ -3651,20 +3657,34 @@ Certain features in ``typing`` are deprecated and may be removed in a future version of Python. The following table summarizes major deprecations for your convenience. This is subject to change, and not all deprecations are listed. -+-------------------------------------+---------------+-------------------+----------------+ -| Feature | Deprecated in | Projected removal | PEP/issue | -+=====================================+===============+===================+================+ -| ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | -| collections | | | | -+-------------------------------------+---------------+-------------------+----------------+ -| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | -+-------------------------------------+---------------+-------------------+----------------+ -| ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | -+-------------------------------------+---------------+-------------------+----------------+ -| ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | -| ``typing.Sized`` | | | | -+-------------------------------------+---------------+-------------------+----------------+ -| ``typing.TypeAlias`` | 3.12 | Undecided | :pep:`695` | -+-------------------------------------+---------------+-------------------+----------------+ -| ``typing.no_type_check_decorator`` | 3.13 | 3.15 | :gh:`106309` | -+-------------------------------------+---------------+-------------------+----------------+ +.. list-table:: + :header-rows: 1 + + * - Feature + - Deprecated in + - Projected removal + - PEP/issue + * - ``typing`` versions of standard collections + - 3.9 + - Undecided (see :ref:`deprecated-aliases` for more information) + - :pep:`585` + * - :class:`typing.ByteString` + - 3.9 + - 3.14 + - :gh:`91896` + * - :data:`typing.Text` + - 3.11 + - Undecided + - :gh:`92332` + * - :class:`typing.Hashable` and :class:`typing.Sized` + - 3.12 + - Undecided + - :gh:`94309` + * - :data:`typing.TypeAlias` + - 3.12 + - Undecided + - :pep:`695` + * - :func:`@typing.no_type_check_decorator ` + - 3.13 + - 3.15 + - :gh:`106309` From fb32f35c0585b1dbb87b6f254818e1f485a50f65 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 14 Jul 2023 20:41:24 +0100 Subject: [PATCH 26/75] gh-102799: replace internal sys.exc_info() call by sys.exception() (#106746) --- Lib/logging/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 2a011b6d2ff15d..46e86cb87ecfcb 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -662,7 +662,7 @@ def formatException(self, ei): # See issues #9427, #1553375. Commented out for now. #if getattr(self, 'fullstack', False): # traceback.print_stack(tb.tb_frame.f_back, file=sio) - traceback.print_exception(ei[0], ei[1], tb, None, sio) + traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": @@ -1080,14 +1080,14 @@ def handleError(self, record): The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 - t, v, tb = sys.exc_info() + exc = sys.exception() try: sys.stderr.write('--- Logging error ---\n') - traceback.print_exception(t, v, tb, None, sys.stderr) + traceback.print_exception(exc, limit=None, file=sys.stderr) sys.stderr.write('Call stack:\n') # Walk the stack frame up until we're out of logging, # so as to print the calling context. - frame = tb.tb_frame + frame = exc.__traceback__.tb_frame while (frame and os.path.dirname(frame.f_code.co_filename) == __path__[0]): frame = frame.f_back @@ -1112,7 +1112,7 @@ def handleError(self, record): except OSError: #pragma: no cover pass # see issue 5971 finally: - del t, v, tb + del exc def __repr__(self): level = getLevelName(self.level) From 03185f0c150ebc52d41dd5ea6f369c7b5ba9fc16 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 14 Jul 2023 16:40:46 -0400 Subject: [PATCH 27/75] gh-106752: Move zipfile._path into its own package (#106753) * gh-106752: Move zipfile._path into its own package so it may have supplementary behavior. * Add blurb --- .github/CODEOWNERS | 2 +- Lib/test/test_zipfile/_path/__init__.py | 0 Lib/test/test_zipfile/{ => _path}/_functools.py | 0 Lib/test/test_zipfile/{ => _path}/_itertools.py | 0 Lib/test/test_zipfile/{ => _path}/_support.py | 0 Lib/test/test_zipfile/{ => _path}/_test_params.py | 0 Lib/test/test_zipfile/{ => _path}/test_complexity.py | 0 Lib/test/test_zipfile/{ => _path}/test_path.py | 0 Lib/zipfile/{_path.py => _path/__init__.py} | 0 Makefile.pre.in | 3 ++- .../next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst | 2 ++ 11 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_zipfile/_path/__init__.py rename Lib/test/test_zipfile/{ => _path}/_functools.py (100%) rename Lib/test/test_zipfile/{ => _path}/_itertools.py (100%) rename Lib/test/test_zipfile/{ => _path}/_support.py (100%) rename Lib/test/test_zipfile/{ => _path}/_test_params.py (100%) rename Lib/test/test_zipfile/{ => _path}/test_complexity.py (100%) rename Lib/test/test_zipfile/{ => _path}/test_path.py (100%) rename Lib/zipfile/{_path.py => _path/__init__.py} (100%) create mode 100644 Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5b471c79f75eea..234a954cc7662f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -172,7 +172,7 @@ Doc/c-api/stable.rst @encukou **/*pathlib* @barneygale # zipfile.Path -**/*zipfile/*_path.py @jaraco +**/*zipfile/_path/* @jaraco # Argument Clinic /Tools/clinic/** @erlend-aasland @AlexWaygood diff --git a/Lib/test/test_zipfile/_path/__init__.py b/Lib/test/test_zipfile/_path/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_zipfile/_functools.py b/Lib/test/test_zipfile/_path/_functools.py similarity index 100% rename from Lib/test/test_zipfile/_functools.py rename to Lib/test/test_zipfile/_path/_functools.py diff --git a/Lib/test/test_zipfile/_itertools.py b/Lib/test/test_zipfile/_path/_itertools.py similarity index 100% rename from Lib/test/test_zipfile/_itertools.py rename to Lib/test/test_zipfile/_path/_itertools.py diff --git a/Lib/test/test_zipfile/_support.py b/Lib/test/test_zipfile/_path/_support.py similarity index 100% rename from Lib/test/test_zipfile/_support.py rename to Lib/test/test_zipfile/_path/_support.py diff --git a/Lib/test/test_zipfile/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py similarity index 100% rename from Lib/test/test_zipfile/_test_params.py rename to Lib/test/test_zipfile/_path/_test_params.py diff --git a/Lib/test/test_zipfile/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py similarity index 100% rename from Lib/test/test_zipfile/test_complexity.py rename to Lib/test/test_zipfile/_path/test_complexity.py diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/_path/test_path.py similarity index 100% rename from Lib/test/test_zipfile/test_path.py rename to Lib/test/test_zipfile/_path/test_path.py diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path/__init__.py similarity index 100% rename from Lib/zipfile/_path.py rename to Lib/zipfile/_path/__init__.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 4b655513f656d7..ddf524ac17d72a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2125,7 +2125,7 @@ LIBSUBDIRS= asyncio \ wsgiref \ $(XMLLIBSUBDIRS) \ xmlrpc \ - zipfile \ + zipfile zipfile/_path \ zoneinfo \ __phello__ TESTSUBDIRS= idlelib/idle_test \ @@ -2229,6 +2229,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_warnings \ test/test_warnings/data \ test/test_zipfile \ + test/test_zipfile/_path \ test/test_zoneinfo \ test/test_zoneinfo/data \ test/tkinterdata \ diff --git a/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst b/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst new file mode 100644 index 00000000000000..ba7257e3610808 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst @@ -0,0 +1,2 @@ +Moved tests for ``zipfile.Path`` into ``Lib/test/test_zipfile/_path``. Made +``zipfile._path`` a package. From 0db85eeba762e72f9f3c027e432cdebc627aac6c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 14 Jul 2023 17:22:06 -0700 Subject: [PATCH 28/75] gh-106529: Fix subtle Tier 2 edge case with list iterator (#106756) The Tier 2 opcode _IS_ITER_EXHAUSTED_LIST (and _TUPLE) didn't set it->it_seq to NULL, causing a subtle bug that resulted in test_exhausted_iterator in list_tests.py to fail when running all tests with -Xuops. The bug was introduced in gh-106696. Added this as an explicit test. Also fixed the dependencies for ceval.o -- it depends on executor_cases.c.h. --- Lib/test/test_capi/test_misc.py | 13 +++++++++++++ Makefile.pre.in | 1 + Python/bytecodes.c | 14 ++++++++++++-- Python/executor_cases.c.h | 14 ++++++++++++-- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 43c04463236a2a..6df918997b2b19 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2649,6 +2649,19 @@ def testfunc(a): # Verification that the jump goes past END_FOR # is done by manual inspection of the output + def test_list_edge_case(self): + def testfunc(it): + for x in it: + pass + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index ddf524ac17d72a..553b2aa480c184 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1564,6 +1564,7 @@ Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/condvar.h \ $(srcdir)/Python/generated_cases.c.h \ + $(srcdir)/Python/executor_cases.c.h \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Python/opcode_targets.h diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 15b48ae9d82672..3432b027713462 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2448,7 +2448,12 @@ dummy_func( _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq == NULL) { + exhausted = Py_True; + } + else if (it->it_index >= PyList_GET_SIZE(seq)) { + Py_DECREF(seq); + it->it_seq = NULL; exhausted = Py_True; } else { @@ -2499,7 +2504,12 @@ dummy_func( _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq == NULL) { + exhausted = Py_True; + } + else if (it->it_index >= PyTuple_GET_SIZE(seq)) { + Py_DECREF(seq); + it->it_seq = NULL; exhausted = Py_True; } else { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 626baece814607..ae21ffad94d801 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1750,7 +1750,12 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq == NULL) { + exhausted = Py_True; + } + else if (it->it_index >= PyList_GET_SIZE(seq)) { + Py_DECREF(seq); + it->it_seq = NULL; exhausted = Py_True; } else { @@ -1787,7 +1792,12 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq == NULL) { + exhausted = Py_True; + } + else if (it->it_index >= PyTuple_GET_SIZE(seq)) { + Py_DECREF(seq); + it->it_seq = NULL; exhausted = Py_True; } else { From 2d7d1aa4bcd5da0177458b22b1b856db76aa20d4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 15 Jul 2023 11:28:57 +0200 Subject: [PATCH 29/75] gh-106368: Increase Argument Clinic BlockParser test coverage (#106759) --- Lib/test/test_clinic.py | 100 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 975840333e5901..b5744f7013d6ad 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -18,6 +18,19 @@ from clinic import DSLParser +class _ParserBase(TestCase): + maxDiff = None + + def expect_parser_failure(self, parser, _input): + with support.captured_stdout() as stdout: + with self.assertRaises(SystemExit): + parser(_input) + return stdout.getvalue() + + def parse_function_should_fail(self, _input): + return self.expect_parser_failure(self.parse_function, _input) + + class FakeConverter: def __init__(self, name, args): self.name = name @@ -88,7 +101,15 @@ def directive(self, name, args): _module_and_class = clinic.Clinic._module_and_class -class ClinicWholeFileTest(TestCase): + +class ClinicWholeFileTest(_ParserBase): + def setUp(self): + self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") + + def expect_failure(self, raw): + _input = dedent(raw).strip() + return self.expect_parser_failure(self.clinic.parse, _input) + def test_eol(self): # regression test: # clinic's block parser didn't recognize @@ -98,15 +119,86 @@ def test_eol(self): # so it would spit out an end line for you. # and since you really already had one, # the last line of the block got corrupted. - c = clinic.Clinic(clinic.CLanguage(None), filename="file") raw = "/*[clinic]\nfoo\n[clinic]*/" - cooked = c.parse(raw).splitlines() + cooked = self.clinic.parse(raw).splitlines() end_line = cooked[2].rstrip() # this test is redundant, it's just here explicitly to catch # the regression test so we don't forget what it looked like self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") self.assertEqual(end_line, "[clinic]*/") + def test_mangled_marker_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: foo]*/ + """ + msg = ( + 'Error in file "test.c" on line 3:\n' + "Mangled Argument Clinic marker line: '/*[clinic end generated code: foo]*/'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_checksum_mismatch(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ + """ + msg = ( + 'Error in file "test.c" on line 3:\n' + 'Checksum mismatch!\n' + 'Expected: 0123456789abcdef\n' + 'Computed: da39a3ee5e6b4b0d\n' + ) + out = self.expect_failure(raw) + self.assertIn(msg, out) + + def test_garbage_after_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/foobarfoobar! + """ + msg = ( + 'Error in file "test.c" on line 2:\n' + "Garbage after stop line: 'foobarfoobar!'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_whitespace_before_stop_line(self): + raw = """ + /*[clinic input] + [clinic start generated code]*/ + """ + msg = ( + 'Error in file "test.c" on line 2:\n' + "Whitespace is not allowed before the stop line: ' [clinic start generated code]*/'\n" + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_parse_with_body_prefix(self): + clang = clinic.CLanguage(None) + clang.body_prefix = "//" + clang.start_line = "//[{dsl_name} start]" + clang.stop_line = "//[{dsl_name} stop]" + cl = clinic.Clinic(clang, filename="test.c") + raw = dedent(""" + //[clinic start] + //module test + //[clinic stop] + """).strip() + out = cl.parse(raw) + expected = dedent(""" + //[clinic start] + //module test + // + //[clinic stop] + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=65fab8adff58cf08]*/ + """).lstrip() # Note, lstrip() because of the newline + self.assertEqual(out, expected) class ClinicGroupPermuterTest(TestCase): @@ -285,7 +377,7 @@ def test_clinic_1(self): """) -class ClinicParserTest(TestCase): +class ClinicParserTest(_ParserBase): def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), From bbf62979851283b601b2dac0073ab331ebeb3be9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 15 Jul 2023 12:11:32 +0200 Subject: [PATCH 30/75] gh-104050: Argument Clinic: Annotate BlockParser (#106750) --- Tools/clinic/clinic.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 726ebc04f55bd5..861a6507eae753 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1712,7 +1712,13 @@ class BlockParser: Iterator, yields Block objects. """ - def __init__(self, input, language, *, verify=True): + def __init__( + self, + input: str, + language: Language, + *, + verify: bool = True + ) -> None: """ "input" should be a str object with embedded \n characters. @@ -1730,15 +1736,15 @@ def __init__(self, input, language, *, verify=True): self.find_start_re = create_regex(before, after, whole_line=False) self.start_re = create_regex(before, after) self.verify = verify - self.last_checksum_re = None - self.last_dsl_name = None - self.dsl_name = None + self.last_checksum_re: re.Pattern[str] | None = None + self.last_dsl_name: str | None = None + self.dsl_name: str | None = None self.first_block = True - def __iter__(self): + def __iter__(self) -> BlockParser: return self - def __next__(self): + def __next__(self) -> Block: while True: if not self.input: raise StopIteration @@ -1755,18 +1761,18 @@ def __next__(self): return block - def is_start_line(self, line): + def is_start_line(self, line: str) -> str | None: match = self.start_re.match(line.lstrip()) return match.group(1) if match else None - def _line(self, lookahead=False): + def _line(self, lookahead: bool = False) -> str: self.line_number += 1 line = self.input.pop() if not lookahead: self.language.parse_line(line) return line - def parse_verbatim_block(self): + def parse_verbatim_block(self) -> Block: add, output = text_accumulator() self.block_start_line_number = self.line_number @@ -1780,13 +1786,13 @@ def parse_verbatim_block(self): return Block(output()) - def parse_clinic_block(self, dsl_name): + def parse_clinic_block(self, dsl_name: str) -> Block: input_add, input_output = text_accumulator() self.block_start_line_number = self.line_number + 1 stop_line = self.language.stop_line.format(dsl_name=dsl_name) body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) - def is_stop_line(line): + def is_stop_line(line: str) -> bool: # make sure to recognize stop line even if it # doesn't end with EOL (it could be the very end of the file) if line.startswith(stop_line): @@ -1820,6 +1826,7 @@ def is_stop_line(line): checksum_re = create_regex(before, after, word=False) self.last_dsl_name = dsl_name self.last_checksum_re = checksum_re + assert checksum_re is not None # scan forward for checksum line output_add, output_output = text_accumulator() @@ -1834,6 +1841,7 @@ def is_stop_line(line): if self.is_start_line(line): break + output: str | None output = output_output() if arguments: d = {} From 2566b74b26bcce24199427acea392aed644f4b17 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 15 Jul 2023 19:33:32 +0900 Subject: [PATCH 31/75] gh-81283: compiler: remove indent from docstring (#106411) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Éric --- Doc/whatsnew/3.13.rst | 7 ++ Include/internal/pycore_compile.h | 2 + Lib/inspect.py | 45 +++++---- Lib/test/test_doctest.py | 4 +- Lib/test/test_inspect.py | 35 ++++++- ...3-07-04-20-42-54.gh-issue-81283.hfh_MD.rst | 3 + Modules/_testinternalcapi.c | 20 +++- Modules/clinic/_testinternalcapi.c.h | 61 +++++++++++- Python/compile.c | 99 ++++++++++++++++++- 9 files changed, 246 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-04-20-42-54.gh-issue-81283.hfh_MD.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 06fcaf4608cdcb..161d5fb1c59a30 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -79,6 +79,13 @@ Other Language Changes * Allow the *count* argument of :meth:`str.replace` to be a keyword. (Contributed by Hugo van Kemenade in :gh:`106487`.) +* Compiler now strip indents from docstrings. + This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). + For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 + is reduced by about 5%. + This change will affect tools using docstrings, like :mod:`doctest`. + (Contributed by Inada Naoki in :gh:`81283`.) + New Modules =========== diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index e204d4d2457a16..beb37cced06dba 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -91,6 +91,8 @@ int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); /* Access compiler internals for unit testing */ +PyAPI_FUNC(PyObject*) _PyCompile_CleanDoc(PyObject *doc); + PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, diff --git a/Lib/inspect.py b/Lib/inspect.py index a550202bb0d49b..15f94a194856ac 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -881,29 +881,28 @@ def cleandoc(doc): Any whitespace that can be uniformly removed from the second line onwards is removed.""" - try: - lines = doc.expandtabs().split('\n') - except UnicodeError: - return None - else: - # Find minimum indentation of any non-blank lines after first line. - margin = sys.maxsize - for line in lines[1:]: - content = len(line.lstrip()) - if content: - indent = len(line) - content - margin = min(margin, indent) - # Remove indentation. - if lines: - lines[0] = lines[0].lstrip() - if margin < sys.maxsize: - for i in range(1, len(lines)): lines[i] = lines[i][margin:] - # Remove any trailing or leading blank lines. - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) - return '\n'.join(lines) + lines = doc.expandtabs().split('\n') + + # Find minimum indentation of any non-blank lines after first line. + margin = sys.maxsize + for line in lines[1:]: + content = len(line.lstrip(' ')) + if content: + indent = len(line) - content + margin = min(margin, indent) + # Remove indentation. + if lines: + lines[0] = lines[0].lstrip(' ') + if margin < sys.maxsize: + for i in range(1, len(lines)): + lines[i] = lines[i][margin:] + # Remove any trailing or leading blank lines. + while lines and not lines[-1]: + lines.pop() + while lines and not lines[0]: + lines.pop(0) + return '\n'.join(lines) + def getfile(object): """Work out which source or compiled file an object was defined in.""" diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 542fcdb5cf6f66..bea52c6de7ec6d 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -1287,14 +1287,14 @@ def optionflags(): r""" treated as equal: >>> def f(x): - ... '>>> print(1, 2, 3)\n 1 2\n 3' + ... '\n>>> print(1, 2, 3)\n 1 2\n 3' >>> # Without the flag: >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) ... # doctest: +ELLIPSIS ********************************************************************** - File ..., line 2, in f + File ..., line 3, in f Failed example: print(1, 2, 3) Expected: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index d89953ab60f022..64afeec351b353 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -596,9 +596,40 @@ def test_finddoc(self): self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) self.assertEqual(finddoc(int.real), int.real.__doc__) + cleandoc_testdata = [ + # first line should have different margin + (' An\n indented\n docstring.', 'An\nindented\n docstring.'), + # trailing whitespace are not removed. + (' An \n \n indented \n docstring. ', + 'An \n \nindented \n docstring. '), + # NUL is not termination. + ('doc\0string\n\n second\0line\n third\0line\0', + 'doc\0string\n\nsecond\0line\nthird\0line\0'), + # first line is lstrip()-ped. other lines are kept when no margin.[w: + (' ', ''), + # compiler.cleandoc() doesn't strip leading/trailing newlines + # to keep maximum backward compatibility. + # inspect.cleandoc() removes them. + ('\n\n\n first paragraph\n\n second paragraph\n\n', + '\n\n\nfirst paragraph\n\n second paragraph\n\n'), + (' \n \n \n ', '\n \n \n '), + ] + def test_cleandoc(self): - self.assertEqual(inspect.cleandoc('An\n indented\n docstring.'), - 'An\nindented\ndocstring.') + func = inspect.cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + # only inspect.cleandoc() strip \n + expected = expected.strip('\n') + with self.subTest(i=i): + self.assertEqual(func(input), expected) + + @cpython_only + def test_c_cleandoc(self): + import _testinternalcapi + func = _testinternalcapi.compiler_cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + with self.subTest(i=i): + self.assertEqual(func(input), expected) def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-04-20-42-54.gh-issue-81283.hfh_MD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-20-42-54.gh-issue-81283.hfh_MD.rst new file mode 100644 index 00000000000000..f673c665fe3277 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-20-42-54.gh-issue-81283.hfh_MD.rst @@ -0,0 +1,3 @@ +Compiler now strips indents from docstrings. It reduces ``pyc`` file size 5% +when the module is heavily documented. This change affects to ``__doc__`` so +tools like doctest will be affected. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7745dd5abc22f0..271ad6cfcaee32 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,7 +15,7 @@ #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_ceval.h" // _PyEval_AddPendingCall #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame @@ -704,6 +704,23 @@ set_eval_frame_record(PyObject *self, PyObject *list) Py_RETURN_NONE; } +/*[clinic input] + +_testinternalcapi.compiler_cleandoc -> object + + doc: unicode + +C implementation of inspect.cleandoc(). +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) +/*[clinic end generated code: output=2dd203a80feff5bc input=2de03fab931d9cdc]*/ +{ + return _PyCompile_CleanDoc(doc); +} + + /*[clinic input] _testinternalcapi.compiler_codegen -> object @@ -1448,6 +1465,7 @@ static PyMethodDef module_functions[] = { {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, + _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index f5124125874503..9419dcd751a0e9 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -8,6 +8,65 @@ preserve #endif +PyDoc_STRVAR(_testinternalcapi_compiler_cleandoc__doc__, +"compiler_cleandoc($module, /, doc)\n" +"--\n" +"\n" +"C implementation of inspect.cleandoc()."); + +#define _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF \ + {"compiler_cleandoc", _PyCFunction_CAST(_testinternalcapi_compiler_cleandoc), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_compiler_cleandoc__doc__}, + +static PyObject * +_testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc); + +static PyObject * +_testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(doc), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"doc", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compiler_cleandoc", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *doc; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("compiler_cleandoc", "argument 'doc'", "str", args[0]); + goto exit; + } + doc = args[0]; + return_value = _testinternalcapi_compiler_cleandoc_impl(module, doc); + +exit: + return return_value; +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -206,4 +265,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=2965f1578b986218 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=811d50772c8f285a input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 9e86e06777ffa4..b80f7c01bcd90e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1704,10 +1704,16 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(stmts); if (docstring) { + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } i = 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); assert(st->kind == Expr_kind); - VISIT(c, expr, st->v.Expr.value); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } @@ -2252,11 +2258,19 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f /* if not -OO mode, add docstring */ if (c->c_optimize < 2) { docstring = _PyAST_GetDocString(body); + if (docstring) { + docstring = _PyCompile_CleanDoc(docstring); + if (docstring == NULL) { + compiler_exit_scope(c); + return ERROR; + } + } } if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { compiler_exit_scope(c); return ERROR; } + Py_XDECREF(docstring); c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); @@ -7967,6 +7981,89 @@ cfg_to_instructions(cfg_builder *g) return NULL; } +// C implementation of inspect.cleandoc() +// +// Difference from inspect.cleandoc(): +// - Do not remove leading and trailing blank lines to keep lineno. +PyObject * +_PyCompile_CleanDoc(PyObject *doc) +{ + doc = PyObject_CallMethod(doc, "expandtabs", NULL); + if (doc == NULL) { + return NULL; + } + + Py_ssize_t doc_size; + const char *doc_utf8 = PyUnicode_AsUTF8AndSize(doc, &doc_size); + if (doc_utf8 == NULL) { + Py_DECREF(doc); + return NULL; + } + const char *p = doc_utf8; + const char *pend = p + doc_size; + + // First pass: find minimum indentation of any non-blank lines + // after first line. + while (p < pend && *p++ != '\n') { + } + + Py_ssize_t margin = PY_SSIZE_T_MAX; + while (p < pend) { + const char *s = p; + while (*p == ' ') p++; + if (p < pend && *p != '\n') { + margin = Py_MIN(margin, p - s); + } + while (p < pend && *p++ != '\n') { + } + } + if (margin == PY_SSIZE_T_MAX) { + margin = 0; + } + + // Second pass: write cleandoc into buff. + + // copy first line without leading spaces. + p = doc_utf8; + while (*p == ' ') { + p++; + } + if (p == doc_utf8 && margin == 0 ) { + // doc is already clean. + return doc; + } + + char *buff = PyMem_Malloc(doc_size); + char *w = buff; + + while (p < pend) { + int ch = *w++ = *p++; + if (ch == '\n') { + break; + } + } + + // copy subsequent lines without margin. + while (p < pend) { + for (Py_ssize_t i = 0; i < margin; i++, p++) { + if (*p != ' ') { + assert(*p == '\n' || *p == '\0'); + break; + } + } + while (p < pend) { + int ch = *w++ = *p++; + if (ch == '\n') { + break; + } + } + } + + Py_DECREF(doc); + return PyUnicode_FromStringAndSize(buff, w - buff); +} + + PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize, int compile_mode) From 22980dc7c9dcec4b74fea815542601ef582c230e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Jul 2023 09:21:17 -0400 Subject: [PATCH 32/75] gh-106752: Sync with zipp 3.16.2 (#106757) * gh-106752: Sync with zipp 3.16.2 * Add blurb --- .../test_zipfile/_path/test_complexity.py | 81 ++++++++++++++++++- Lib/test/test_zipfile/_path/test_path.py | 70 ++++++++++++++-- Lib/test/test_zipfile/_path/write-alpharep.py | 4 + Lib/zipfile/_path/__init__.py | 31 +++---- Lib/zipfile/_path/glob.py | 40 +++++++++ ...-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst | 5 ++ 6 files changed, 204 insertions(+), 27 deletions(-) create mode 100644 Lib/test/test_zipfile/_path/write-alpharep.py create mode 100644 Lib/zipfile/_path/glob.py create mode 100644 Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index 3432dc39e56c4e..7050937738af18 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -1,5 +1,9 @@ -import unittest +import io +import itertools +import math +import re import string +import unittest import zipfile from ._functools import compose @@ -9,9 +13,11 @@ big_o = import_or_skip('big_o') +pytest = import_or_skip('pytest') class TestComplexity(unittest.TestCase): + @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( compose(consume, zipfile.CompleteDirs._implied_dirs), @@ -22,3 +28,76 @@ def test_implied_dirs_performance(self): min_n=1, ) assert best <= big_o.complexities.Linear + + def make_zip_path(self, depth=1, width=1) -> zipfile.Path: + """ + Construct a Path with width files at every level of depth. + """ + zf = zipfile.ZipFile(io.BytesIO(), mode='w') + pairs = itertools.product(self.make_deep_paths(depth), self.make_names(width)) + for path, name in pairs: + zf.writestr(f"{path}{name}.txt", b'') + zf.filename = "big un.zip" + return zipfile.Path(zf) + + @classmethod + def make_names(cls, width, letters=string.ascii_lowercase): + """ + >>> list(TestComplexity.make_names(2)) + ['a', 'b'] + >>> list(TestComplexity.make_names(30)) + ['aa', 'ab', ..., 'bd'] + """ + # determine how many products are needed to produce width + n_products = math.ceil(math.log(width, len(letters))) + inputs = (letters,) * n_products + combinations = itertools.product(*inputs) + names = map(''.join, combinations) + return itertools.islice(names, width) + + @classmethod + def make_deep_paths(cls, depth): + return map(cls.make_deep_path, range(depth)) + + @classmethod + def make_deep_path(cls, depth): + return ''.join(('d/',) * depth) + + def test_baseline_regex_complexity(self): + best, others = big_o.big_o( + lambda path: re.fullmatch(r'[^/]*\\.txt', path), + self.make_deep_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Constant + + @pytest.mark.flaky + def test_glob_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + self.make_zip_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Quadratic + + @pytest.mark.flaky + def test_glob_width(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(width=size), + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width_and_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(depth=size, width=size), + max_n=10, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index aff91e53995875..c66cb3cba69ebd 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -41,9 +41,13 @@ def build_alpharep_fixture(): │ ├── d │ │ └── e.txt │ └── f.txt - └── g - └── h - └── i.txt + ├── g + │ └── h + │ └── i.txt + └── j + ├── k.bin + ├── l.baz + └── m.bar This fixture has the following key characteristics: @@ -51,6 +55,7 @@ def build_alpharep_fixture(): - a file two levels deep (b/d/e) - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) + - a directory with files of different extensions (j/klm) "alpha" because it uses alphabet "rep" because it's a representative example @@ -62,6 +67,9 @@ def build_alpharep_fixture(): zf.writestr("b/d/e.txt", b"content of e") zf.writestr("b/f.txt", b"content of f") zf.writestr("g/h/i.txt", b"content of i") + zf.writestr("j/k.bin", b"content of k") + zf.writestr("j/l.baz", b"content of l") + zf.writestr("j/m.bar", b"content of m") zf.filename = "alpharep.zip" return zf @@ -92,7 +100,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -112,7 +120,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -127,7 +135,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -229,7 +237,7 @@ def test_open_missing_directory(self): @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -295,7 +303,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g = root.iterdir() + a, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -394,6 +402,13 @@ def test_suffixes(self, alpharep): e = root / '.hgrc' assert e.suffixes == [] + @pass_alpharep + def test_suffix_no_filename(self, alpharep): + alpharep.filename = None + root = zipfile.Path(alpharep) + assert root.joinpath('example').suffix == "" + assert root.joinpath('example').suffixes == [] + @pass_alpharep def test_stem(self, alpharep): """ @@ -411,6 +426,8 @@ def test_stem(self, alpharep): d = root / "d" assert d.stem == "d" + assert (root / ".gitignore").stem == ".gitignore" + @pass_alpharep def test_root_parent(self, alpharep): root = zipfile.Path(alpharep) @@ -442,12 +459,49 @@ def test_match_and_glob(self, alpharep): assert not root.match("*.txt") assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + assert list(root.glob("b/*.txt")) == [ + zipfile.Path(alpharep, "b/c.txt"), + zipfile.Path(alpharep, "b/f.txt"), + ] + @pass_alpharep + def test_glob_recursive(self, alpharep): + root = zipfile.Path(alpharep) files = root.glob("**/*.txt") assert all(each.match("*.txt") for each in files) assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + @pass_alpharep + def test_glob_subdirs(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*/i.txt")) == [] + assert list(root.rglob("*/i.txt")) == [zipfile.Path(alpharep, "g/h/i.txt")] + + @pass_alpharep + def test_glob_does_not_overmatch_dot(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*.xt")) == [] + + @pass_alpharep + def test_glob_single_char(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("a?txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[.]txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[?]txt")) == [] + + @pass_alpharep + def test_glob_chars(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("j/?.b[ai][nz]")) == [ + zipfile.Path(alpharep, "j/k.bin"), + zipfile.Path(alpharep, "j/l.baz"), + ] + def test_glob_empty(self): root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) with self.assertRaises(ValueError): diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py new file mode 100644 index 00000000000000..48c09b537179fd --- /dev/null +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -0,0 +1,4 @@ +from . import test_path + + +__name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index fd49a3ea91db59..78c413563bb2b1 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -5,7 +5,8 @@ import contextlib import pathlib import re -import fnmatch + +from .glob import translate __all__ = ['Path'] @@ -296,21 +297,24 @@ def open(self, mode='r', *args, pwd=None, **kwargs): encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) return io.TextIOWrapper(stream, encoding, *args, **kwargs) + def _base(self): + return pathlib.PurePosixPath(self.at or self.root.filename) + @property def name(self): - return pathlib.Path(self.at).name or self.filename.name + return self._base().name @property def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix + return self._base().suffix @property def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes + return self._base().suffixes @property def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem + return self._base().stem @property def filename(self): @@ -347,7 +351,7 @@ def iterdir(self): return filter(self._is_child, subs) def match(self, path_pattern): - return pathlib.Path(self.at).match(path_pattern) + return pathlib.PurePosixPath(self.at).match(path_pattern) def is_symlink(self): """ @@ -355,22 +359,13 @@ def is_symlink(self): """ return False - def _descendants(self): - for child in self.iterdir(): - yield child - if child.is_dir(): - yield from child._descendants() - def glob(self, pattern): if not pattern: raise ValueError(f"Unacceptable pattern: {pattern!r}") - matches = re.compile(fnmatch.translate(pattern)).fullmatch - return ( - child - for child in self._descendants() - if matches(str(child.relative_to(self))) - ) + prefix = re.escape(self.at) + matches = re.compile(prefix + translate(pattern)).fullmatch + return map(self._next, filter(matches, self.root.namelist())) def rglob(self, pattern): return self.glob(f'**/{pattern}') diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py new file mode 100644 index 00000000000000..4a2e665e27078a --- /dev/null +++ b/Lib/zipfile/_path/glob.py @@ -0,0 +1,40 @@ +import re + + +def translate(pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> translate('*.txt') + '[^/]*\\.txt' + >>> translate('a?txt') + 'a.txt' + >>> translate('**/*') + '.*/[^/]*' + """ + return ''.join(map(replace, separate(pattern))) + + +def separate(pattern): + """ + Separate out character sets to avoid translating their contents. + + >>> [m.group(0) for m in separate('*.txt')] + ['*.txt'] + >>> [m.group(0) for m in separate('a[?]txt')] + ['a', '[?]', 'txt'] + """ + return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) + + +def replace(match): + """ + Perform the replacements for a match from :func:`separate`. + """ + + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', r'[^/]*') + .replace('\\?', r'.') + ) diff --git a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst new file mode 100644 index 00000000000000..bbc53d76decbc3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst @@ -0,0 +1,5 @@ +Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows +separators are no longer honored (and never were meant to be); Fixed +``name``/``suffix``/``suffixes``/``stem`` operations when no filename is +present and the Path is not at the root of the zipfile; Reworked glob for +performance and more correct matching behavior. From e2ec0bad67552e27174255db86dda90fc72e6694 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Jul 2023 14:43:09 -0500 Subject: [PATCH 33/75] Add more examples to the recipe docs (GH-106782) Demonstrate that factor() works for large composites and large primes. --- Doc/library/itertools.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index a2d1798a2c6da1..f88525456ff939 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1045,6 +1045,8 @@ The following recipes have a more mathematical flavor: def factor(n): "Prime factors of n." # factor(99) --> 3 3 11 + # factor(1_000_000_000_000_007) --> 47 59 360620266859 + # factor(1_000_000_000_000_403) --> 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while True: quotient, remainder = divmod(n, prime) From d46a42fd8e8915e03cc211ab9163058b6365ab0f Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Sat, 15 Jul 2023 22:23:10 +0200 Subject: [PATCH 34/75] faq/library: remove outdated section (#105996) --- Doc/faq/library.rst | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 597caaa778e1c8..22f7f846d261d8 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -669,41 +669,6 @@ and client-side web systems. A summary of available frameworks is maintained by Paul Boddie at https://wiki.python.org/moin/WebProgramming\ . -Cameron Laird maintains a useful set of pages about Python web technologies at -https://web.archive.org/web/20210224183619/http://phaseit.net/claird/comp.lang.python/web_python. - - -How can I mimic CGI form submission (METHOD=POST)? --------------------------------------------------- - -I would like to retrieve web pages that are the result of POSTing a form. Is -there existing code that would let me do this easily? - -Yes. Here's a simple example that uses :mod:`urllib.request`:: - - #!/usr/local/bin/python - - import urllib.request - - # build the query string - qs = "First=Josephine&MI=Q&Last=Public" - - # connect and send the server a path - req = urllib.request.urlopen('http://www.some-server.out-there' - '/cgi-bin/some-cgi-script', data=qs) - with req: - msg, hdrs = req.read(), req.info() - -Note that in general for percent-encoded POST operations, query strings must be -quoted using :func:`urllib.parse.urlencode`. For example, to send -``name=Guy Steele, Jr.``:: - - >>> import urllib.parse - >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'}) - 'name=Guy+Steele%2C+Jr.' - -.. seealso:: :ref:`urllib-howto` for extensive examples. - What module should I use to help with generating HTML? ------------------------------------------------------ From 8c177294899b621fe04ae755abd41b4d319dd4b5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 16 Jul 2023 00:42:58 +0200 Subject: [PATCH 35/75] Docs: Normalize Argument Clinic How-To section capitalization (#106788) --- Doc/howto/clinic.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 4620b4617e3450..0f99cb64994ab2 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -27,7 +27,8 @@ Argument Clinic How-To version of Argument Clinic that ships with the next version of CPython *could* be totally incompatible and break all your code. -The Goals Of Argument Clinic + +The goals of Argument Clinic ============================ Argument Clinic's primary goal @@ -78,7 +79,7 @@ and it should be able to do many interesting and smart things with all the information you give it. -Basic Concepts And Usage +Basic concepts and usage ======================== Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. @@ -141,7 +142,7 @@ For the sake of clarity, here's the terminology we'll use with Argument Clinic: a block.) -Converting Your First Function +Converting your first function ============================== The best way to get a sense of how Argument Clinic works is to @@ -558,7 +559,8 @@ Let's dive in! Congratulations, you've ported your first function to work with Argument Clinic! -Advanced Topics + +Advanced topics =============== Now that you've had some experience working with Argument Clinic, it's time @@ -636,7 +638,8 @@ after the last argument). Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this will change soon. -Optional Groups + +Optional groups --------------- Some legacy functions have a tricky approach to parsing their arguments: @@ -899,6 +902,7 @@ available. For each converter it'll show you all the parameters it accepts, along with the default value for each parameter. Just run ``Tools/clinic/clinic.py --converters`` to see the full list. + Py_buffer --------- @@ -908,7 +912,6 @@ you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. Argument Clinic generates code that does it for you (in the parsing function). - Advanced converters ------------------- @@ -975,6 +978,7 @@ value called ``NULL`` for just this reason: from Python's perspective it behaves like a default value of ``None``, but the C variable is initialized with ``NULL``. + Expressions specified as default values --------------------------------------- @@ -1032,7 +1036,6 @@ you're not permitted to use: * Tuple/list/set/dict literals. - Using a return converter ------------------------ @@ -1146,6 +1149,7 @@ then modifying it. Cloning is an all-or nothing proposition. Also, the function you are cloning from must have been previously defined in the current file. + Calling Python code ------------------- @@ -1380,6 +1384,7 @@ handle initialization and cleanup. You can see more examples of custom converters in the CPython source tree; grep the C files for the string ``CConverter``. + Writing a custom return converter --------------------------------- @@ -1394,8 +1399,9 @@ write your own return converter, please read ``Tools/clinic/clinic.py``, specifically the implementation of ``CReturnConverter`` and all its subclasses. + METH_O and METH_NOARGS ----------------------------------------------- +---------------------- To convert a function using ``METH_O``, make sure the function's single argument is using the ``object`` converter, and mark the @@ -1415,8 +1421,9 @@ any arguments. You can still use a self converter, a return converter, and specify a ``type`` argument to the object converter for ``METH_O``. + tp_new and tp_init functions ----------------------------------------------- +---------------------------- You can convert ``tp_new`` and ``tp_init`` functions. Just name them ``__new__`` or ``__init__`` as appropriate. Notes: @@ -1437,6 +1444,7 @@ them ``__new__`` or ``__init__`` as appropriate. Notes: (If your function doesn't support keywords, the parsing function generated will throw an exception if it receives any.) + Changing and redirecting Clinic's output ---------------------------------------- @@ -1721,7 +1729,7 @@ the file was not modified by hand before it gets overwritten. The #ifdef trick ----------------------------------------------- +---------------- If you're converting a function that isn't available on all platforms, there's a trick you can use to make life a little easier. The existing @@ -1801,7 +1809,6 @@ Argument Clinic added to your file (it'll be at the very bottom), then move it above the ``PyMethodDef`` structure where that macro is used. - Using Argument Clinic in Python files ------------------------------------- From c02ee4503151105dc892018ebc7f633e7f3f62f8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 16 Jul 2023 10:26:26 +0300 Subject: [PATCH 36/75] Docs search: Replace jQuery with vanilla JavaScript (#106743) * Replace jQuery with vanilla JavaScript * Switch 'var' to 'const' or 'let' --- Doc/tools/templates/search.html | 74 ++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index f2ac2ea0f09873..852974461380f2 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -1,48 +1,62 @@ {% extends "!search.html" %} {% block extrahead %} {{ super() }} + -{% endblock %} \ No newline at end of file +{% endblock %} From 83bd568d2b57337a91ef046c1f52f9ebb03a7803 Mon Sep 17 00:00:00 2001 From: Simone Rubino Date: Sun, 16 Jul 2023 09:29:58 +0200 Subject: [PATCH 37/75] Doc: devmode: add -Xdev option to example (#106253) --- Doc/library/devmode.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index 977735990ffe92..b2bad48a07e27e 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -198,7 +198,7 @@ descriptor" error when finalizing the file object: .. code-block:: shell-session - $ python script.py + $ python -X dev script.py import os script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> main() From e58960160fcb4fce63177fcd9ef605f887377767 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 16 Jul 2023 21:23:54 +0900 Subject: [PATCH 38/75] Doc: fix section levels of devmode doc (#106801) --- Doc/library/devmode.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index b2bad48a07e27e..80ac13b116c1d2 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -16,7 +16,7 @@ setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. See also :ref:`Python debug build `. Effects of the Python Development Mode -====================================== +-------------------------------------- Enabling the Python Development Mode is similar to the following command, but with additional effects described below:: @@ -107,7 +107,7 @@ value can be read from :data:`sys.flags.dev_mode `. ResourceWarning Example -======================= +----------------------- Example of a script counting the number of lines of the text file specified in the command line:: @@ -171,7 +171,7 @@ application more deterministic and more reliable. Bad file descriptor error example -================================= +--------------------------------- Script displaying the first line of itself:: From cc25ca16ee406db936dfbd2337cbd14b12ccc4b7 Mon Sep 17 00:00:00 2001 From: Kevin Diem Date: Sun, 16 Jul 2023 11:16:34 -0400 Subject: [PATCH 39/75] gh-106706: Streamline family syntax in cases generator DSL (#106716) From `family(opname, STRUCTSIZE) = OPNAME + SPEC1 + ... + SPECn;` to `family(OPNAME, STRUCTSIZE) = SPEC1 + ... + SPECn;` --- ...-07-13-12-08-35.gh-issue-106706.29zp8E.rst | 3 ++ Python/bytecodes.c | 39 +++++++------------ Tools/cases_generator/generate_cases.py | 27 +++++++------ .../cases_generator/interpreter_definition.md | 2 +- Tools/cases_generator/test_generator.py | 2 +- 5 files changed, 31 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-07-13-12-08-35.gh-issue-106706.29zp8E.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-13-12-08-35.gh-issue-106706.29zp8E.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-13-12-08-35.gh-issue-106706.29zp8E.rst new file mode 100644 index 00000000000000..bbd8e8eddda607 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-07-13-12-08-35.gh-issue-106706.29zp8E.rst @@ -0,0 +1,3 @@ +Change bytecode syntax for families +to remove redundant name matching +pseudo syntax. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3432b027713462..3c3992c068b063 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -284,8 +284,7 @@ dummy_func( res = Py_IsFalse(value) ? Py_True : Py_False; } - family(to_bool, INLINE_CACHE_ENTRIES_TO_BOOL) = { - TO_BOOL, + family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = { TO_BOOL_ALWAYS_TRUE, TO_BOOL_BOOL, TO_BOOL_INT, @@ -372,8 +371,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - family(binary_op, INLINE_CACHE_ENTRIES_BINARY_OP) = { - BINARY_OP, + family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = { BINARY_OP_MULTIPLY_INT, BINARY_OP_ADD_INT, BINARY_OP_SUBTRACT_INT, @@ -507,8 +505,7 @@ dummy_func( macro(BINARY_OP_INPLACE_ADD_UNICODE) = _GUARD_BOTH_UNICODE + _BINARY_OP_INPLACE_ADD_UNICODE; - family(binary_subscr, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { - BINARY_SUBSCR, + family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { BINARY_SUBSCR_DICT, BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, @@ -643,8 +640,7 @@ dummy_func( ERROR_IF(err, error); } - family(store_subscr, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { - STORE_SUBSCR, + family(STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, }; @@ -921,8 +917,7 @@ dummy_func( ERROR_IF(iter == NULL, error); } - family(send, INLINE_CACHE_ENTRIES_SEND) = { - SEND, + family(SEND, INLINE_CACHE_ENTRIES_SEND) = { SEND_GEN, }; @@ -1134,8 +1129,7 @@ dummy_func( } } - family(unpack_sequence, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE) = { - UNPACK_SEQUENCE, + family(UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE) = { UNPACK_SEQUENCE_TWO_TUPLE, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_LIST, @@ -1198,8 +1192,7 @@ dummy_func( ERROR_IF(res == 0, error); } - family(store_attr, INLINE_CACHE_ENTRIES_STORE_ATTR) = { - STORE_ATTR, + family(STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR) = { STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT, @@ -1298,8 +1291,7 @@ dummy_func( macro(LOAD_FROM_DICT_OR_GLOBALS) = _LOAD_FROM_DICT_OR_GLOBALS; - family(load_global, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { - LOAD_GLOBAL, + family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { LOAD_GLOBAL_MODULE, LOAD_GLOBAL_BUILTIN, }; @@ -1647,8 +1639,7 @@ dummy_func( GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - family(load_super_attr, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { - LOAD_SUPER_ATTR, + family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { LOAD_SUPER_ATTR_ATTR, LOAD_SUPER_ATTR_METHOD, }; @@ -1750,8 +1741,7 @@ dummy_func( } } - family(load_attr, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { - LOAD_ATTR, + family(LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_MODULE, LOAD_ATTR_WITH_HINT, @@ -2048,8 +2038,7 @@ dummy_func( Py_DECREF(owner); } - family(compare_op, INLINE_CACHE_ENTRIES_COMPARE_OP) = { - COMPARE_OP, + family(COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP) = { COMPARE_OP_FLOAT, COMPARE_OP_INT, COMPARE_OP_STR, @@ -2350,8 +2339,7 @@ dummy_func( // This is optimized by skipping that instruction and combining // its effect (popping 'iter' instead of pushing 'next'.) - family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = { - FOR_ITER, + family(FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER) = { FOR_ITER_LIST, FOR_ITER_TUPLE, FOR_ITER_RANGE, @@ -2810,8 +2798,7 @@ dummy_func( // Cache layout: counter/1, func_version/2 // Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members! - family(call, INLINE_CACHE_ENTRIES_CALL) = { - CALL, + family(CALL, INLINE_CACHE_ENTRIES_CALL) = { CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 6589289121863b..3edd8ee51ba64e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -438,7 +438,7 @@ def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: """Write one instruction, sans prologue and epilogue.""" # Write a static assertion that a family's cache size is correct if family := self.family: - if self.name == family.members[0]: + if self.name == family.name: if cache_size := family.size: out.emit( f"static_assert({cache_size} == " @@ -831,7 +831,7 @@ def find_predictions(self) -> None: def map_families(self) -> None: """Link instruction names back to their family, if they have one.""" for family in self.families.values(): - for member in family.members: + for member in [family.name] + family.members: if member_instr := self.instrs.get(member): if member_instr.family not in (family, None): self.error( @@ -855,8 +855,11 @@ def check_families(self) -> None: - All members must have the same cache, input and output effects """ for family in self.families.values(): - if len(family.members) < 2: - self.error(f"Family {family.name!r} has insufficient members", family) + if family.name not in self.macro_instrs and family.name not in self.instrs: + self.error( + f"Family {family.name!r} has unknown instruction {family.name!r}", + family, + ) members = [ member for member in family.members @@ -867,10 +870,8 @@ def check_families(self) -> None: self.error( f"Family {family.name!r} has unknown members: {unknown}", family ) - if len(members) < 2: - continue - expected_effects = self.effect_counts(members[0]) - for member in members[1:]: + expected_effects = self.effect_counts(family.name) + for member in members: member_effects = self.effect_counts(member) if member_effects != expected_effects: self.error( @@ -1311,11 +1312,10 @@ def write_metadata(self) -> None: self.out.emit("") self.out.emit("_specializations = {") for name, family in self.families.items(): - assert len(family.members) > 1 with self.out.indent(): - self.out.emit(f"\"{family.members[0]}\": [") + self.out.emit(f"\"{family.name}\": [") with self.out.indent(): - for m in family.members[1:]: + for m in family.members: self.out.emit(f"\"{m}\",") self.out.emit(f"],") self.out.emit("}") @@ -1551,9 +1551,8 @@ def write_macro(self, mac: MacroInstruction) -> None: self.out.emit(f"next_instr += {cache_adjust};") if ( - last_instr - and (family := last_instr.family) - and mac.name == family.members[0] + (family := self.families.get(mac.name)) + and mac.name == family.name and (cache_size := family.size) ): self.out.emit( diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index c03870ef59eb49..f141848631d04a 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -347,7 +347,7 @@ For explanations see "Generating the interpreter" below.) ### Defining an instruction family -A _family_ represents a specializable instruction and its specializations. +A _family_ maps a specializable instruction to its specializations. Example: These opcodes all share the same instruction format): ```C diff --git a/Tools/cases_generator/test_generator.py b/Tools/cases_generator/test_generator.py index e374ac41e6a94d..e44273429b7405 100644 --- a/Tools/cases_generator/test_generator.py +++ b/Tools/cases_generator/test_generator.py @@ -287,7 +287,7 @@ def test_macro_instruction(): inst(OP3, (unused/5, arg2, left, right -- res)) { res = op3(arg2, left, right); } - family(op, INLINE_CACHE_ENTRIES_OP) = { OP, OP3 }; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ output = """ TARGET(OP1) { From 55408f86d78259f18c56c5e1ea51e0f8dcdbeb67 Mon Sep 17 00:00:00 2001 From: Grigoriev Semyon <33061489+grigoriev-semyon@users.noreply.github.com> Date: Sun, 16 Jul 2023 18:30:39 +0300 Subject: [PATCH 40/75] gh-105726: Add `__slots__` to `AbstractContextManager` and `AbstractAsyncContextManager` (#106771) Co-authored-by: Kumar Aditya --- Lib/contextlib.py | 4 ++++ Lib/test/test_contextlib.py | 10 ++++++++++ Lib/test/test_contextlib_async.py | 12 ++++++++++++ .../2023-07-15-12-52-50.gh-issue-105726.NGthO8.rst | 3 +++ 4 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-15-12-52-50.gh-issue-105726.NGthO8.rst diff --git a/Lib/contextlib.py b/Lib/contextlib.py index b5acbcb9e6d77c..95947aceccc304 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 0f8351ab8108a6..ecc5a43dad43da 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -24,6 +24,16 @@ def __exit__(self, *args): manager = DefaultEnter() self.assertIs(manager.__enter__(), manager) + def test_slots(self): + class DefaultContextManager(AbstractContextManager): + __slots__ = () + + def __exit__(self, *args): + super().__exit__(*args) + + with self.assertRaises(AttributeError): + DefaultContextManager().var = 42 + def test_exit_is_abstract(self): class MissingExit(AbstractContextManager): pass diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 3d43ed0fcab168..bb72ae74e5845f 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -37,6 +37,18 @@ async def __aexit__(self, *args): async with manager as context: self.assertIs(manager, context) + @_async_test + async def test_slots(self): + class DefaultAsyncContextManager(AbstractAsyncContextManager): + __slots__ = () + + async def __aexit__(self, *args): + await super().__aexit__(*args) + + with self.assertRaises(AttributeError): + manager = DefaultAsyncContextManager() + manager.var = 42 + @_async_test async def test_async_gen_propagates_generator_exit(self): # A regression test for https://bugs.python.org/issue33786. diff --git a/Misc/NEWS.d/next/Library/2023-07-15-12-52-50.gh-issue-105726.NGthO8.rst b/Misc/NEWS.d/next/Library/2023-07-15-12-52-50.gh-issue-105726.NGthO8.rst new file mode 100644 index 00000000000000..434f93240eccdf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-15-12-52-50.gh-issue-105726.NGthO8.rst @@ -0,0 +1,3 @@ +Added ``__slots__`` to :class:`contextlib.AbstractContextManager` and :class:`contextlib.AbstractAsyncContextManager` +so that child classes can use ``__slots__``. + From 4dc593477a2e8a5c22e3e2346aaae05ca46b12cb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Poupon Date: Sun, 16 Jul 2023 20:14:08 +0300 Subject: [PATCH 41/75] Fix the french used in the email documentation (GH-106279) * Fix the french used in the email documentation The french used in one of the example was either machine translated a while ago or written by someone who does not speak french. Fixed it by using grammatically correct french. --- Doc/includes/email-alternative.py | 10 +++++----- Doc/library/email.examples.rst | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py index df7ca6f3faa332..26b302b495c7ac 100644 --- a/Doc/includes/email-alternative.py +++ b/Doc/includes/email-alternative.py @@ -8,14 +8,14 @@ # Create the base text message. msg = EmailMessage() -msg['Subject'] = "Ayons asperges pour le déjeuner" +msg['Subject'] = "Pourquoi pas des asperges pour ce midi ?" msg['From'] = Address("Pepé Le Pew", "pepe", "example.com") msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"), Address("Fabrette Pussycat", "fabrette", "example.com")) msg.set_content("""\ Salut! -Cela ressemble à un excellent recipie[1] déjeuner. +Cette recette [1] sera sûrement un très bon repas. [1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 @@ -31,10 +31,10 @@

Salut!

-

Cela ressemble à un excellent +

Cette - recipie - déjeuner. + recette + sera sûrement un très bon repas.

diff --git a/Doc/library/email.examples.rst b/Doc/library/email.examples.rst index fc964622809d0e..492a8354d8bf85 100644 --- a/Doc/library/email.examples.rst +++ b/Doc/library/email.examples.rst @@ -55,11 +55,11 @@ Up to the prompt, the output from the above is: To: Penelope Pussycat , Fabrette Pussycat From: Pepé Le Pew - Subject: Ayons asperges pour le déjeuner + Subject: Pourquoi pas des asperges pour ce midi ? Salut! - Cela ressemble à un excellent recipie[1] déjeuner. + Cette recette [1] sera sûrement un très bon repas. .. rubric:: Footnotes From c41320701b28904064c89a0a29775efed6b6d053 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 17 Jul 2023 02:05:24 +0300 Subject: [PATCH 42/75] gh-105540: Convert `pytest` tests of `cases_generator` to regular tests (#106713) --- .../test/test_generated_cases.py | 349 ++++++++++-------- Tools/cases_generator/generate_cases.py | 32 +- 2 files changed, 218 insertions(+), 163 deletions(-) rename Tools/cases_generator/test_generator.py => Lib/test/test_generated_cases.py (58%) diff --git a/Tools/cases_generator/test_generator.py b/Lib/test/test_generated_cases.py similarity index 58% rename from Tools/cases_generator/test_generator.py rename to Lib/test/test_generated_cases.py index e44273429b7405..ba0e5e8b0f6954 100644 --- a/Tools/cases_generator/test_generator.py +++ b/Lib/test/test_generated_cases.py @@ -1,97 +1,148 @@ -# Sorry for using pytest, these tests are mostly just for me. -# Use pytest -vv for best results. - import tempfile +import unittest +import os + +from test import test_tools + +test_tools.skip_if_missing('cases_generator') +with test_tools.imports_under_tool('cases_generator'): + import generate_cases + from parser import StackEffect + + +class TestEffects(unittest.TestCase): + def test_effect_sizes(self): + input_effects = [ + x := StackEffect("x", "", "", ""), + y := StackEffect("y", "", "", "oparg"), + z := StackEffect("z", "", "", "oparg*2"), + ] + output_effects = [ + StackEffect("a", "", "", ""), + StackEffect("b", "", "", "oparg*4"), + StackEffect("c", "", "", ""), + ] + other_effects = [ + StackEffect("p", "", "", "oparg<<1"), + StackEffect("q", "", "", ""), + StackEffect("r", "", "", ""), + ] + self.assertEqual(generate_cases.effect_size(x), (1, "")) + self.assertEqual(generate_cases.effect_size(y), (0, "oparg")) + self.assertEqual(generate_cases.effect_size(z), (0, "oparg*2")) + + self.assertEqual( + generate_cases.list_effect_size(input_effects), + (1, "oparg + oparg*2"), + ) + self.assertEqual( + generate_cases.list_effect_size(output_effects), + (2, "oparg*4"), + ) + self.assertEqual( + generate_cases.list_effect_size(other_effects), + (2, "(oparg<<1)"), + ) + + self.assertEqual( + generate_cases.string_effect_size( + generate_cases.list_effect_size(input_effects), + ), "1 + oparg + oparg*2", + ) + self.assertEqual( + generate_cases.string_effect_size( + generate_cases.list_effect_size(output_effects), + ), + "2 + oparg*4", + ) + self.assertEqual( + generate_cases.string_effect_size( + generate_cases.list_effect_size(other_effects), + ), + "2 + (oparg<<1)", + ) + + +class TestGeneratedCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() -import generate_cases -from parser import StackEffect - - -def test_effect_sizes(): - input_effects = [ - x := StackEffect("x", "", "", ""), - y := StackEffect("y", "", "", "oparg"), - z := StackEffect("z", "", "", "oparg*2"), - ] - output_effects = [ - StackEffect("a", "", "", ""), - StackEffect("b", "", "", "oparg*4"), - StackEffect("c", "", "", ""), - ] - other_effects = [ - StackEffect("p", "", "", "oparg<<1"), - StackEffect("q", "", "", ""), - StackEffect("r", "", "", ""), - ] - assert generate_cases.effect_size(x) == (1, "") - assert generate_cases.effect_size(y) == (0, "oparg") - assert generate_cases.effect_size(z) == (0, "oparg*2") - - assert generate_cases.list_effect_size(input_effects) == (1, "oparg + oparg*2") - assert generate_cases.list_effect_size(output_effects) == (2, "oparg*4") - assert generate_cases.list_effect_size(other_effects) == (2, "(oparg<<1)") - - assert generate_cases.string_effect_size(generate_cases.list_effect_size(input_effects)) == "1 + oparg + oparg*2" - assert generate_cases.string_effect_size(generate_cases.list_effect_size(output_effects)) == "2 + oparg*4" - assert generate_cases.string_effect_size(generate_cases.list_effect_size(other_effects)) == "2 + (oparg<<1)" - - -def run_cases_test(input: str, expected: str): - temp_input = tempfile.NamedTemporaryFile("w+") - temp_input.write(generate_cases.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(generate_cases.END_MARKER) - temp_input.flush() - temp_output = tempfile.NamedTemporaryFile("w+") - temp_metadata = tempfile.NamedTemporaryFile("w+") - temp_pymetadata = tempfile.NamedTemporaryFile("w+") - temp_executor = tempfile.NamedTemporaryFile("w+") - a = generate_cases.Analyzer( - [temp_input.name], - temp_output.name, - temp_metadata.name, - temp_pymetadata.name, - temp_executor.name, - ) - a.parse() - a.analyze() - if a.errors: - raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions() - temp_output.seek(0) - lines = temp_output.readlines() - while lines and lines[0].startswith("// "): - lines.pop(0) - actual = "".join(lines) - # if actual.rstrip() != expected.rstrip(): - # print("Actual:") - # print(actual) - # print("Expected:") - # print(expected) - # print("End") - assert actual.rstrip() == expected.rstrip() - -def test_inst_no_args(): - input = """ + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(generate_cases.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(generate_cases.END_MARKER) + temp_input.flush() + + a = generate_cases.Analyzer( + [self.temp_input_filename], + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ) + a.parse() + a.analyze() + if a.errors: + raise RuntimeError(f"Found {a.errors} errors") + a.write_instructions() + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith("// "): + lines.pop(0) + actual = "".join(lines) + # if actual.rstrip() != expected.rstrip(): + # print("Actual:") + # print(actual) + # print("Expected:") + # print(expected) + # print("End") + + self.assertEqual(actual.rstrip(), expected.rstrip()) + + def test_inst_no_args(self): + input = """ inst(OP, (--)) { spam(); } """ - output = """ + output = """ TARGET(OP) { spam(); DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_inst_one_pop(): - input = """ + def test_inst_one_pop(self): + input = """ inst(OP, (value --)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *value = stack_pointer[-1]; spam(); @@ -99,15 +150,15 @@ def test_inst_one_pop(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_inst_one_push(): - input = """ + def test_inst_one_push(self): + input = """ inst(OP, (-- res)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *res; spam(); @@ -116,15 +167,15 @@ def test_inst_one_push(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_inst_one_push_one_pop(): - input = """ + def test_inst_one_push_one_pop(self): + input = """ inst(OP, (value -- res)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *value = stack_pointer[-1]; PyObject *res; @@ -133,15 +184,15 @@ def test_inst_one_push_one_pop(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_binary_op(): - input = """ + def test_binary_op(self): + input = """ inst(OP, (left, right -- res)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; @@ -152,15 +203,15 @@ def test_binary_op(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_overlap(): - input = """ + def test_overlap(self): + input = """ inst(OP, (left, right -- left, result)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; @@ -170,10 +221,10 @@ def test_overlap(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_predictions_and_eval_breaker(): - input = """ + def test_predictions_and_eval_breaker(self): + input = """ inst(OP1, (--)) { } inst(OP3, (arg -- res)) { @@ -181,7 +232,7 @@ def test_predictions_and_eval_breaker(): CHECK_EVAL_BREAKER(); } """ - output = """ + output = """ TARGET(OP1) { PREDICTED(OP1); DISPATCH(); @@ -196,43 +247,43 @@ def test_predictions_and_eval_breaker(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_error_if_plain(): - input = """ + def test_error_if_plain(self): + input = """ inst(OP, (--)) { ERROR_IF(cond, label); } """ - output = """ + output = """ TARGET(OP) { if (cond) goto label; DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_error_if_plain_with_comment(): - input = """ + def test_error_if_plain_with_comment(self): + input = """ inst(OP, (--)) { ERROR_IF(cond, label); // Comment is ok } """ - output = """ + output = """ TARGET(OP) { if (cond) goto label; DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_error_if_pop(): - input = """ + def test_error_if_pop(self): + input = """ inst(OP, (left, right -- res)) { ERROR_IF(cond, label); } """ - output = """ + output = """ TARGET(OP) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; @@ -243,14 +294,14 @@ def test_error_if_pop(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_cache_effect(): - input = """ + def test_cache_effect(self): + input = """ inst(OP, (counter/1, extra/2, value --)) { } """ - output = """ + output = """ TARGET(OP) { PyObject *value = stack_pointer[-1]; uint16_t counter = read_u16(&next_instr[0].cache); @@ -260,23 +311,23 @@ def test_cache_effect(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_suppress_dispatch(): - input = """ + def test_suppress_dispatch(self): + input = """ inst(OP, (--)) { goto somewhere; } """ - output = """ + output = """ TARGET(OP) { goto somewhere; } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_macro_instruction(): - input = """ + def test_macro_instruction(self): + input = """ inst(OP1, (counter/1, left, right -- left, right)) { op1(left, right); } @@ -289,7 +340,7 @@ def test_macro_instruction(): } family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ - output = """ + output = """ TARGET(OP1) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; @@ -339,15 +390,15 @@ def test_macro_instruction(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_array_input(): - input = """ + def test_array_input(self): + input = """ inst(OP, (below, values[oparg*2], above --)) { spam(); } """ - output = """ + output = """ TARGET(OP) { PyObject *above = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg*2)); @@ -358,15 +409,15 @@ def test_array_input(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_array_output(): - input = """ + def test_array_output(self): + input = """ inst(OP, (unused, unused -- below, values[oparg*3], above)) { spam(values, oparg); } """ - output = """ + output = """ TARGET(OP) { PyObject *below; PyObject **values = stack_pointer - (2) + 1; @@ -378,15 +429,15 @@ def test_array_output(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_array_input_output(): - input = """ + def test_array_input_output(self): + input = """ inst(OP, (values[oparg] -- values[oparg], above)) { spam(values, oparg); } """ - output = """ + output = """ TARGET(OP) { PyObject **values = (stack_pointer - oparg); PyObject *above; @@ -396,15 +447,15 @@ def test_array_input_output(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_array_error_if(): - input = """ + def test_array_error_if(self): + input = """ inst(OP, (extra, values[oparg] --)) { ERROR_IF(oparg == 0, somewhere); } """ - output = """ + output = """ TARGET(OP) { PyObject **values = (stack_pointer - oparg); PyObject *extra = stack_pointer[-(1 + oparg)]; @@ -414,15 +465,15 @@ def test_array_error_if(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_cond_effect(): - input = """ + def test_cond_effect(self): + input = """ inst(OP, (aa, input if ((oparg & 1) == 1), cc -- xx, output if (oparg & 2), zz)) { output = spam(oparg, input); } """ - output = """ + output = """ TARGET(OP) { PyObject *cc = stack_pointer[-1]; PyObject *input = ((oparg & 1) == 1) ? stack_pointer[-(1 + (((oparg & 1) == 1) ? 1 : 0))] : NULL; @@ -439,10 +490,10 @@ def test_cond_effect(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) -def test_macro_cond_effect(): - input = """ + def test_macro_cond_effect(self): + input = """ op(A, (left, middle, right --)) { # Body of A } @@ -451,7 +502,7 @@ def test_macro_cond_effect(): } macro(M) = A + B; """ - output = """ + output = """ TARGET(M) { PyObject *_tmp_1 = stack_pointer[-1]; PyObject *_tmp_2 = stack_pointer[-2]; @@ -479,4 +530,8 @@ def test_macro_cond_effect(): DISPATCH(); } """ - run_cases_test(input, output) + self.run_cases_test(input, output) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 3edd8ee51ba64e..df5de6e299aaa9 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -156,16 +156,9 @@ def __init__( self.emit_line_directives = emit_line_directives self.comment = comment self.lineno = 1 - filename = os.path.relpath(self.stream.name, ROOT) - # Make filename more user-friendly and less platform-specific - filename = filename.replace("\\", "/") - if filename.startswith("./"): - filename = filename[2:] - if filename.endswith(".new"): - filename = filename[:-4] - self.filename = filename + self.filename = prettify_filename(self.stream.name) self.nominal_lineno = 1 - self.nominal_filename = filename + self.nominal_filename = self.filename def write_raw(self, s: str) -> None: self.stream.write(s) @@ -737,12 +730,8 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: with open(filename) as file: src = file.read() - filename = os.path.relpath(filename, ROOT) - # Make filename more user-friendly and less platform-specific - filename = filename.replace("\\", "/") - if filename.startswith("./"): - filename = filename[2:] - psr = parser.Parser(src, filename=filename) + + psr = parser.Parser(src, filename=prettify_filename(filename)) # Skip until begin marker while tkn := psr.next(raw=True): @@ -1149,7 +1138,7 @@ def write_function( def from_source_files(self) -> str: paths = f"\n{self.out.comment} ".join( - os.path.relpath(filename, ROOT).replace(os.path.sep, posixpath.sep) + prettify_filename(filename) for filename in self.input_filenames ) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" @@ -1597,6 +1586,17 @@ def wrap_macro(self, mac: MacroInstruction): self.out.emit(f"DISPATCH();") +def prettify_filename(filename: str) -> str: + # Make filename more user-friendly and less platform-specific, + # it is only used for error reporting at this point. + filename = filename.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] + if filename.endswith(".new"): + filename = filename[:-4] + return filename + + def extract_block_text(block: parser.Block) -> tuple[list[str], bool, int]: # Get lines of text with proper dedent blocklines = block.text.splitlines(True) From 383dcbebcda576e3a3fd18c9246364f67bb65df5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 17 Jul 2023 02:04:10 +0200 Subject: [PATCH 43/75] gh-104050: Argument Clinic: Annotate Clinic.parse() (#106760) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 861a6507eae753..9b7069e9b8fcb0 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -42,6 +42,7 @@ Literal, NamedTuple, NoReturn, + Protocol, TypeGuard, overload, ) @@ -2055,7 +2056,12 @@ def write_file(filename: str, new_contents: str) -> None: ClassDict = dict[str, "Class"] DestinationDict = dict[str, Destination] ModuleDict = dict[str, "Module"] -ParserDict = dict[str, "DSLParser"] + + +class Parser(Protocol): + def __init__(self, clinic: Clinic) -> None: ... + def parse(self, block: Block) -> None: ... + clinic = None class Clinic: @@ -2113,7 +2119,7 @@ def __init__( ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) - self.parsers: ParserDict = {} + self.parsers: dict[str, Parser] = {} self.language: CLanguage = language if printer: fail("Custom printers are broken right now") @@ -2205,7 +2211,7 @@ def get_destination_buffer( d = self.get_destination(name) return d.buffers[item] - def parse(self, input): + def parse(self, input: str) -> str: printer = self.printer self.block_parser = BlockParser(input, self.language, verify=self.verify) for block in self.block_parser: @@ -5521,7 +5527,10 @@ def state_terminal(self, line): # "clinic", handles the Clinic DSL # "python", handles running Python code # -parsers = {'clinic' : DSLParser, 'python': PythonParser} +parsers: dict[str, Callable[[Clinic], Parser]] = { + 'clinic': DSLParser, + 'python': PythonParser, +} clinic = None From 48956cc60ea05bc50b6cd73e53dd9a7d4b1dac9f Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 17 Jul 2023 09:09:11 +0900 Subject: [PATCH 44/75] gh-106797: Remove warning logs from Python/generated_cases.c.h (gh-106798) --- Include/internal/pycore_opcode_metadata.h | 10 +++++----- Python/generated_cases.c.h | 20 ++++++++------------ Tools/cases_generator/generate_cases.py | 14 +++++++++++--- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8373f56653b1c7..3b2eab23e092ff 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -839,15 +839,15 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case PUSH_EXC_INFO: return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return (1 ? 1 : 0) + 1; + return 1 + 1; case LOAD_ATTR_METHOD_NO_DICT: - return (1 ? 1 : 0) + 1; + return 1 + 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return (0 ? 1 : 0) + 1; + return 0 + 1; case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return (0 ? 1 : 0) + 1; + return 0 + 1; case LOAD_ATTR_METHOD_LAZY_DICT: - return (1 ? 1 : 0) + 1; + return 1 + 1; case KW_NAMES: return 0; case INSTRUMENTED_CALL: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 68531dc074769e..392914c0521e9d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3360,9 +3360,9 @@ res2 = Py_NewRef(descr); assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; - STACK_GROW((1 ? 1 : 0)); + STACK_GROW(1); stack_pointer[-1] = res; - if (1) { stack_pointer[-(1 + (1 ? 1 : 0))] = res2; } + stack_pointer[-(1 + 1)] = res2; next_instr += 9; DISPATCH(); } @@ -3382,16 +3382,15 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); res = self; - STACK_GROW((1 ? 1 : 0)); + STACK_GROW(1); stack_pointer[-1] = res; - if (1) { stack_pointer[-(1 + (1 ? 1 : 0))] = res2; } + stack_pointer[-(1 + 1)] = res2; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); @@ -3410,16 +3409,14 @@ assert(descr != NULL); Py_DECREF(self); res = Py_NewRef(descr); - STACK_GROW((0 ? 1 : 0)); + STACK_GROW(0); stack_pointer[-1] = res; - if (0) { stack_pointer[-(1 + (0 ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3432,9 +3429,8 @@ assert(descr != NULL); Py_DECREF(self); res = Py_NewRef(descr); - STACK_GROW((0 ? 1 : 0)); + STACK_GROW(0); stack_pointer[-1] = res; - if (0) { stack_pointer[-(1 + (0 ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } @@ -3458,9 +3454,9 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); res2 = Py_NewRef(descr); res = self; - STACK_GROW((1 ? 1 : 0)); + STACK_GROW(1); stack_pointer[-1] = res; - if (1) { stack_pointer[-(1 + (1 ? 1 : 0))] = res2; } + stack_pointer[-(1 + 1)] = res2; next_instr += 9; DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index df5de6e299aaa9..a0a8b8cbe4baba 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -98,6 +98,8 @@ def effect_size(effect: StackEffect) -> tuple[int, str]: assert not effect.cond, "Array effects cannot have a condition" return 0, effect.size elif effect.cond: + if effect.cond in ("0", "1"): + return 0, effect.cond return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" else: return 1, "" @@ -217,7 +219,7 @@ def stack_adjust( self.emit(f"STACK_GROW({osym});") def declare(self, dst: StackEffect, src: StackEffect | None): - if dst.name == UNUSED: + if dst.name == UNUSED or dst.cond == "0": return typ = f"{dst.type}" if dst.type else "PyObject *" if src: @@ -241,7 +243,10 @@ def assign(self, dst: StackEffect, src: StackEffect): self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") else: stmt = f"{dst.name} = {cast}{src.name};" - if src.cond: + if src.cond and src.cond != "1": + if src.cond == "0": + # It will not be executed + return stmt = f"if ({src.cond}) {{ {stmt} }}" self.emit(stmt) @@ -1067,7 +1072,10 @@ def effect_str(effects: list[StackEffect]) -> str: for effect in comp.instr.output_effects: assert not effect.size, effect if effect.cond: - pushed_symbolic.append(maybe_parenthesize(f"{maybe_parenthesize(effect.cond)} ? 1 : 0")) + if effect.cond in ("0", "1"): + pushed_symbolic.append(effect.cond) + else: + pushed_symbolic.append(maybe_parenthesize(f"{maybe_parenthesize(effect.cond)} ? 1 : 0")) sp += 1 high = max(sp, high) if high != max(0, sp): From babb22da5a25c18a2d203bf72ba35e7861ca60ee Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Jul 2023 21:37:07 -0500 Subject: [PATCH 45/75] Add more recipe tests. Make the factor recipe a bit faster and clearer. (GH-106817) --- Doc/library/itertools.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index f88525456ff939..730736bbb59ed9 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1049,11 +1049,10 @@ The following recipes have a more mathematical flavor: # factor(1_000_000_000_000_403) --> 1000000000000403 for prime in sieve(math.isqrt(n) + 1): while True: - quotient, remainder = divmod(n, prime) - if remainder: + if n % prime: break yield prime - n = quotient + n //= prime if n == 1: return if n > 1: @@ -1354,6 +1353,12 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + >>> list(factor(99)) # Code example 1 + [3, 3, 11] + >>> list(factor(1_000_000_000_000_007)) # Code example 2 + [47, 59, 360620266859] + >>> list(factor(1_000_000_000_000_403)) # Code example 3 + [1000000000000403] >>> list(factor(0)) [] >>> list(factor(1)) From 7aa89e505d893cd5e6f33b84d66e5fa769089931 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 16 Jul 2023 23:36:03 -0400 Subject: [PATCH 46/75] gh-106780: Add __match_args__ to tutorial example (#106784) Add Point definition with this attribute before example that needs it. --- Doc/tutorial/controlflow.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 4336bf50df40a7..e140f51f1dda78 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -343,7 +343,13 @@ Dotted names (like ``foo.bar``), attribute names (the ``x=`` and ``y=`` above) o (recognized by the "(...)" next to them like ``Point`` above) are never assigned to. Patterns can be arbitrarily nested. For example, if we have a short -list of points, we could match it like this:: +list of Points, with ``__match_args__`` added, we could match it like this:: + + class Point: + __match_args__ = ('x', 'y') + def __init__(self, x, y): + self.x = x + self.y = y match points: case []: From 5ecedbd26692b9fbdd7aad81b991869bf650f929 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:28:33 +0100 Subject: [PATCH 47/75] gh-106789: avoid importing pprint from sysconfig (#106790) --- Lib/opcode.py | 26 +++++-------------- Lib/sysconfig.py | 8 ++++-- ...-07-16-10-40-34.gh-issue-106789.NvyE3C.rst | 1 + 3 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst diff --git a/Lib/opcode.py b/Lib/opcode.py index bc885051c6454e..1b36300785aaea 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -6,26 +6,14 @@ __all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] - -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 - -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass - -# _opcode_metadata may not be ready during early stages of the build -try: + "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] + +from _opcode import stack_effect + +import sys +# The build uses older versions of Python which do not have _opcode_metadata +if sys.version_info[:2] >= (3, 13): from _opcode_metadata import _specializations, _specialized_instructions -except ModuleNotFoundError: - pass cmp_op = ('<', '<=', '==', '!=', '>', '>=') diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 122d441bd19f5e..a8b5c5f7dfba5b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -465,10 +465,14 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) +def _print_config_dict(d, stream): + print ("{", file=stream) + for k, v in sorted(d.items()): + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" - import pprint vars = {} # load the installed Makefile: makefile = get_makefile_filename() @@ -523,7 +527,7 @@ def _generate_posix_vars(): f.write('# system configuration generated and used by' ' the sysconfig module\n') f.write('build_time_vars = ') - pprint.pprint(vars, stream=f) + _print_config_dict(vars, stream=f) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: diff --git a/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst new file mode 100644 index 00000000000000..532f8059740daf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst @@ -0,0 +1 @@ +Remove import of :mod:``pprint`` from :mod:``sysconfig``. From 036bb7365607ab7e5cf901f1ac4256f9ae1be82c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 17 Jul 2023 13:47:08 +0200 Subject: [PATCH 48/75] gh-104050: Improve Argument Clinic type annotation coverage (#106810) Add various missing annotations in the following classes: - BlockPrinter - CConverter - CLanguage - FormatCounterFormatter - Language - _TextAccumulator Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 49 ++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 9b7069e9b8fcb0..311f0a1a56a038 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -109,7 +109,7 @@ class _TextAccumulator(NamedTuple): def _text_accumulator() -> _TextAccumulator: text: list[str] = [] - def output(): + def output() -> str: s = ''.join(text) text.clear() return s @@ -433,10 +433,10 @@ class FormatCounterFormatter(string.Formatter): the counts dict would now look like {'a': 2, 'b': 1, 'c': 1} """ - def __init__(self): - self.counts = collections.Counter() + def __init__(self) -> None: + self.counts = collections.Counter[str]() - def get_value(self, key, args, kwargs): + def get_value(self, key: str, args, kwargs) -> str: # type: ignore[override] self.counts[key] += 1 return '' @@ -447,18 +447,25 @@ class Language(metaclass=abc.ABCMeta): stop_line = "" checksum_line = "" - def __init__(self, filename): + def __init__(self, filename: str) -> None: pass @abc.abstractmethod - def render(self, clinic, signatures): + def render( + self, + clinic: Clinic | None, + signatures: Iterable[Module | Class | Function] + ) -> str: pass - def parse_line(self, line): + def parse_line(self, line: str) -> None: pass - def validate(self): - def assert_only_one(attr, *additional_fields): + def validate(self) -> None: + def assert_only_one( + attr: str, + *additional_fields: str + ) -> None: """ Ensures that the string found at getattr(self, attr) contains exactly one formatter replacement string for @@ -485,10 +492,10 @@ def assert_only_one(attr, *additional_fields): """ fields = ['dsl_name'] fields.extend(additional_fields) - line = getattr(self, attr) + line: str = getattr(self, attr) fcf = FormatCounterFormatter() fcf.format(line) - def local_fail(should_be_there_but_isnt): + def local_fail(should_be_there_but_isnt: bool) -> None: if should_be_there_but_isnt: fail("{} {} must contain {{{}}} exactly once!".format( self.__class__.__name__, attr, name)) @@ -749,10 +756,10 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" - def __init__(self, filename): + def __init__(self, filename: str) -> None: super().__init__(filename) self.cpp = cpp.Monitor(filename) - self.cpp.fail = fail + self.cpp.fail = fail # type: ignore[method-assign] def parse_line(self, line: str) -> None: self.cpp.writeline(line) @@ -935,6 +942,7 @@ def parser_body( add(field) return linear_format(output(), parser_declarations=declarations) + parsearg: str | None if not parameters: parser_code: list[str] | None if not requires_defining_class: @@ -1880,7 +1888,12 @@ class BlockPrinter: language: Language f: io.StringIO = dc.field(default_factory=io.StringIO) - def print_block(self, block, *, core_includes=False): + def print_block( + self, + block: Block, + *, + core_includes: bool = False + ) -> None: input = block.input output = block.output dsl_name = block.dsl_name @@ -1931,7 +1944,7 @@ def print_block(self, block, *, core_includes=False): write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) write("\n") - def write(self, text): + def write(self, text: str) -> None: self.f.write(text) @@ -2755,7 +2768,7 @@ class CConverter(metaclass=CConverterAutoRegister): # If not None, should be a string representing a pointer to a # PyTypeObject (e.g. "&PyUnicode_Type"). # Only used by the 'O!' format unit (and the "object" converter). - subclass_of = None + subclass_of: str | None = None # Do we want an adjacent '_length' variable for this variable? # Only used by format units ending with '#'. @@ -2948,7 +2961,7 @@ def simple_declaration(self, by_reference=False, *, in_parser=False): prototype.append(name) return "".join(prototype) - def declaration(self, *, in_parser=False): + def declaration(self, *, in_parser=False) -> str: """ The C statement to declare this variable. """ @@ -3006,7 +3019,7 @@ def pre_render(self): """ pass - def parse_arg(self, argname, displayname): + def parse_arg(self, argname: str, displayname: str): if self.format_unit == 'O&': return """ if (!{converter}({argname}, &{paramname})) {{{{ From ad95c7253a70e559e7d3f25d53f4772f28bb8b44 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 17 Jul 2023 17:55:30 +0200 Subject: [PATCH 49/75] gh-106687: _ssl: use uint64_t for SSL options (#106700) SSL_CTX_get_options() uses uint64_t for options: https://www.openssl.org/docs/man3.1/man3/SSL_CTX_get_options.html Fix this compiler warning on Windows with MSC: conversion from 'uint64_t' to 'long', possible loss of data --- Lib/test/test_ssl.py | 24 +++++++++++++ Modules/_ssl.c | 80 ++++++++++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d46ce5e60e2141..6117ca3fdba1b7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -339,6 +339,15 @@ def test_constants(self): ssl.OP_NO_TLSv1_2 self.assertEqual(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv23) + def test_options(self): + # gh-106687: SSL options values are unsigned integer (uint64_t) + for name in dir(ssl): + if not name.startswith('OP_'): + continue + with self.subTest(option=name): + value = getattr(ssl, name) + self.assertGreaterEqual(value, 0, f"ssl.{name}") + def test_ssl_types(self): ssl_types = [ _ssl._SSLContext, @@ -951,6 +960,7 @@ def test_get_ciphers(self): ) def test_options(self): + # Test default SSLContext options ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) @@ -959,16 +969,30 @@ def test_options(self): OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | OP_ENABLE_MIDDLEBOX_COMPAT) self.assertEqual(default, ctx.options) + + # disallow TLSv1 with warnings_helper.check_warnings(): ctx.options |= ssl.OP_NO_TLSv1 self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + + # allow TLSv1 with warnings_helper.check_warnings(): ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) self.assertEqual(default, ctx.options) + + # clear all options ctx.options = 0 # Ubuntu has OP_NO_SSLv3 forced on by default self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + # invalid options + with self.assertRaises(OverflowError): + ctx.options = -1 + with self.assertRaises(OverflowError): + ctx.options = 2 ** 100 + with self.assertRaises(TypeError): + ctx.options = "abc" + def test_verify_mode_protocol(self): with warnings_helper.check_warnings(): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 571de331e92cd9..0cf4d3e9dc8c9b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3025,7 +3025,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) /*[clinic end generated code: output=2cf0d7a0741b6bd1 input=8d58a805b95fc534]*/ { PySSLContext *self; - long options; + uint64_t options; const SSL_METHOD *method = NULL; SSL_CTX *ctx = NULL; X509_VERIFY_PARAM *params; @@ -3618,20 +3618,32 @@ PyDoc_STRVAR(PySSLContext_security_level_doc, "The current security level"); static PyObject * get_options(PySSLContext *self, void *c) { - return PyLong_FromLong(SSL_CTX_get_options(self->ctx)); + uint64_t options = SSL_CTX_get_options(self->ctx); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(options)); + return PyLong_FromUnsignedLongLong(options); } static int set_options(PySSLContext *self, PyObject *arg, void *c) { - long new_opts, opts, set, clear; - long opt_no = ( + PyObject *new_opts_obj; + unsigned long long new_opts_arg; + uint64_t new_opts, opts, clear, set; + uint64_t opt_no = ( SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3 ); - if (!PyArg_Parse(arg, "l", &new_opts)) + if (!PyArg_Parse(arg, "O!", &PyLong_Type, &new_opts_obj)) { return -1; + } + new_opts_arg = PyLong_AsUnsignedLongLong(new_opts_obj); + if (new_opts_arg == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + Py_BUILD_ASSERT(sizeof(new_opts) >= sizeof(new_opts_arg)); + new_opts = (uint64_t)new_opts_arg; + opts = SSL_CTX_get_options(self->ctx); clear = opts & ~new_opts; set = ~opts & new_opts; @@ -3645,8 +3657,9 @@ set_options(PySSLContext *self, PyObject *arg, void *c) if (clear) { SSL_CTX_clear_options(self->ctx, clear); } - if (set) + if (set) { SSL_CTX_set_options(self->ctx, set); + } return 0; } @@ -5754,10 +5767,24 @@ sslmodule_init_socketapi(PyObject *module) return 0; } + static int -sslmodule_init_constants(PyObject *m) +sslmodule_add_option(PyObject *m, const char *name, uint64_t value) { + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(value)); + PyObject *obj = PyLong_FromUnsignedLongLong(value); + if (obj == NULL) { + return -1; + } + int res = PyModule_AddObjectRef(m, name, obj); + Py_DECREF(obj); + return res; +} + +static int +sslmodule_init_constants(PyObject *m) +{ PyModule_AddStringConstant(m, "_DEFAULT_CIPHERS", PY_SSL_DEFAULT_CIPHER_STRING); @@ -5877,46 +5904,47 @@ sslmodule_init_constants(PyObject *m) PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); +#define ADD_OPTION(NAME, VALUE) if (sslmodule_add_option(m, NAME, (VALUE)) < 0) return -1 + /* protocol options */ - PyModule_AddIntConstant(m, "OP_ALL", - SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); - PyModule_AddIntConstant(m, "OP_NO_SSLv2", SSL_OP_NO_SSLv2); - PyModule_AddIntConstant(m, "OP_NO_SSLv3", SSL_OP_NO_SSLv3); - PyModule_AddIntConstant(m, "OP_NO_TLSv1", SSL_OP_NO_TLSv1); - PyModule_AddIntConstant(m, "OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); - PyModule_AddIntConstant(m, "OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); + ADD_OPTION("OP_ALL", SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); + ADD_OPTION("OP_NO_SSLv2", SSL_OP_NO_SSLv2); + ADD_OPTION("OP_NO_SSLv3", SSL_OP_NO_SSLv3); + ADD_OPTION("OP_NO_TLSv1", SSL_OP_NO_TLSv1); + ADD_OPTION("OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); + ADD_OPTION("OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); #ifdef SSL_OP_NO_TLSv1_3 - PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); + ADD_OPTION("OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); #else - PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", 0); + ADD_OPTION("OP_NO_TLSv1_3", 0); #endif - PyModule_AddIntConstant(m, "OP_CIPHER_SERVER_PREFERENCE", + ADD_OPTION("OP_CIPHER_SERVER_PREFERENCE", SSL_OP_CIPHER_SERVER_PREFERENCE); - PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); - PyModule_AddIntConstant(m, "OP_NO_TICKET", SSL_OP_NO_TICKET); - PyModule_AddIntConstant(m, "OP_LEGACY_SERVER_CONNECT", + ADD_OPTION("OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE); + ADD_OPTION("OP_NO_TICKET", SSL_OP_NO_TICKET); + ADD_OPTION("OP_LEGACY_SERVER_CONNECT", SSL_OP_LEGACY_SERVER_CONNECT); #ifdef SSL_OP_SINGLE_ECDH_USE - PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); + ADD_OPTION("OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE); #endif #ifdef SSL_OP_NO_COMPRESSION - PyModule_AddIntConstant(m, "OP_NO_COMPRESSION", + ADD_OPTION("OP_NO_COMPRESSION", SSL_OP_NO_COMPRESSION); #endif #ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT - PyModule_AddIntConstant(m, "OP_ENABLE_MIDDLEBOX_COMPAT", + ADD_OPTION("OP_ENABLE_MIDDLEBOX_COMPAT", SSL_OP_ENABLE_MIDDLEBOX_COMPAT); #endif #ifdef SSL_OP_NO_RENEGOTIATION - PyModule_AddIntConstant(m, "OP_NO_RENEGOTIATION", + ADD_OPTION("OP_NO_RENEGOTIATION", SSL_OP_NO_RENEGOTIATION); #endif #ifdef SSL_OP_IGNORE_UNEXPECTED_EOF - PyModule_AddIntConstant(m, "OP_IGNORE_UNEXPECTED_EOF", + ADD_OPTION("OP_IGNORE_UNEXPECTED_EOF", SSL_OP_IGNORE_UNEXPECTED_EOF); #endif #ifdef SSL_OP_ENABLE_KTLS - PyModule_AddIntConstant(m, "OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); + ADD_OPTION("OP_ENABLE_KTLS", SSL_OP_ENABLE_KTLS); #endif #ifdef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT From b2b261ab2a2d4ff000c6248dbc52247c78cfa5ab Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 17 Jul 2023 10:06:05 -0700 Subject: [PATCH 50/75] gh-106529: Generate uops for POP_JUMP_IF_[NOT_]NONE (#106796) These aren't automatically translated because (ironically) they are macros deferring to POP_JUMP_IF_{TRUE,FALSE}, which are not viable uops (being manually translated). The hack is that we emit IS_NONE and then set opcode and jump to the POP_JUMP_IF_{TRUE,FALSE} translation code. --- Lib/test/test_capi/test_misc.py | 30 ++++++++++++++++++++++++++++++ Python/optimizer.c | 17 +++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6df918997b2b19..c0dcff825758ad 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2532,6 +2532,36 @@ def testfunc(n): uops = {opname for opname, _ in ex} self.assertIn("_POP_JUMP_IF_FALSE", uops) + def test_pop_jump_if_none(self): + def testfunc(a): + for x in a: + if x is None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc([1, 2, 3]) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _ in ex} + self.assertIn("_POP_JUMP_IF_TRUE", uops) + + def test_pop_jump_if_not_none(self): + def testfunc(a): + for x in a: + if x is not None: + x = 0 + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc([1, 2, 3]) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _ in ex} + self.assertIn("_POP_JUMP_IF_FALSE", uops) + def test_pop_jump_if_true(self): def testfunc(n): i = 0 diff --git a/Python/optimizer.c b/Python/optimizer.c index 289b202f806ae1..693ba375971ae7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -464,9 +464,26 @@ translate_bytecode_to_trace( switch (opcode) { + case POP_JUMP_IF_NONE: + { + RESERVE(2, 2); + ADD_TO_TRACE(IS_NONE, 0); + opcode = POP_JUMP_IF_TRUE; + goto pop_jump_if_bool; + } + + case POP_JUMP_IF_NOT_NONE: + { + RESERVE(2, 2); + ADD_TO_TRACE(IS_NONE, 0); + opcode = POP_JUMP_IF_FALSE; + goto pop_jump_if_bool; + } + case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { +pop_jump_if_bool: // Assume jump unlikely (TODO: handle jump likely case) RESERVE(1, 2); _Py_CODEUNIT *target_instr = From 2b94a05a0e45e4aae030a28b716a038ef529f8ef Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 17 Jul 2023 11:02:58 -0700 Subject: [PATCH 51/75] gh-106581: Add 10 new opcodes by allowing `assert(kwnames == NULL)` (#106707) By turning `assert(kwnames == NULL)` into a macro that is not in the "forbidden" list, many instructions that formerly were skipped because they contained such an assert (but no other mention of `kwnames`) are now supported in Tier 2. This covers 10 instructions in total (all specializations of `CALL` that invoke some C code): - `CALL_NO_KW_TYPE_1` - `CALL_NO_KW_STR_1` - `CALL_NO_KW_TUPLE_1` - `CALL_NO_KW_BUILTIN_O` - `CALL_NO_KW_BUILTIN_FAST` - `CALL_NO_KW_LEN` - `CALL_NO_KW_ISINSTANCE` - `CALL_NO_KW_METHOD_DESCRIPTOR_O` - `CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS` - `CALL_NO_KW_METHOD_DESCRIPTOR_FAST` --- Include/internal/pycore_opcode_metadata.h | 10 + Python/bytecodes.c | 30 +- Python/ceval.c | 4 + Python/ceval_macros.h | 2 + Python/executor_cases.c.h | 329 ++++++++++++++++++++++ Python/generated_cases.c.h | 30 +- Tools/cases_generator/generate_cases.py | 14 +- 7 files changed, 385 insertions(+), 34 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3b2eab23e092ff..028736e115b3f4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1309,7 +1309,17 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [CALL_NO_KW_TYPE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TYPE_1, 0, 0 } } }, + [CALL_NO_KW_STR_1] = { .nuops = 1, .uops = { { CALL_NO_KW_STR_1, 0, 0 } } }, + [CALL_NO_KW_TUPLE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TUPLE_1, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, + [CALL_NO_KW_BUILTIN_O] = { .nuops = 1, .uops = { { CALL_NO_KW_BUILTIN_O, 0, 0 } } }, + [CALL_NO_KW_BUILTIN_FAST] = { .nuops = 1, .uops = { { CALL_NO_KW_BUILTIN_FAST, 0, 0 } } }, + [CALL_NO_KW_LEN] = { .nuops = 1, .uops = { { CALL_NO_KW_LEN, 0, 0 } } }, + [CALL_NO_KW_ISINSTANCE] = { .nuops = 1, .uops = { { CALL_NO_KW_ISINSTANCE, 0, 0 } } }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { CALL_NO_KW_METHOD_DESCRIPTOR_O, 0, 0 } } }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { CALL_NO_KW_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, [MAKE_FUNCTION] = { .nuops = 1, .uops = { { MAKE_FUNCTION, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { BUILD_SLICE, 0, 0 } } }, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3c3992c068b063..652372cb23dc5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2776,7 +2776,7 @@ dummy_func( } inst(KW_NAMES, (--)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg < PyTuple_GET_SIZE(FRAME_CO_CONSTS)); kwnames = GETITEM(FRAME_CO_CONSTS, oparg); } @@ -2927,7 +2927,7 @@ dummy_func( } inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; int argcount = oparg; @@ -2955,7 +2955,7 @@ dummy_func( } inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; int argcount = oparg; @@ -2993,7 +2993,7 @@ dummy_func( } inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; @@ -3005,7 +3005,7 @@ dummy_func( } inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -3019,7 +3019,7 @@ dummy_func( } inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); @@ -3038,7 +3038,7 @@ dummy_func( * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); @@ -3122,7 +3122,7 @@ dummy_func( inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) { /* Builtin METH_O functions */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3153,7 +3153,7 @@ dummy_func( inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3222,7 +3222,7 @@ dummy_func( } inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; int total_args = oparg; @@ -3249,7 +3249,7 @@ dummy_func( } inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; int total_args = oparg; @@ -3279,7 +3279,7 @@ dummy_func( // This is secretly a super-instruction inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -3299,7 +3299,7 @@ dummy_func( } inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, method, unused, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3365,7 +3365,7 @@ dummy_func( } inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; int total_args = oparg; @@ -3397,7 +3397,7 @@ dummy_func( } inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, method, unused, args[oparg] -- res)) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { diff --git a/Python/ceval.c b/Python/ceval.c index d6c72fa3ff386c..f13ba9883d9814 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2706,6 +2706,9 @@ void Py_LeaveRecursiveCall(void) ///////////////////// Experimental UOp Interpreter ///////////////////// +#undef ASSERT_KWNAMES_IS_NULL +#define ASSERT_KWNAMES_IS_NULL() (void)0 + #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ if ((COND)) { \ @@ -2746,6 +2749,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject int opcode; uint64_t operand; int oparg; + for (;;) { opcode = self->trace[pc].opcode; operand = self->trace[pc].operand; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 72800aaaaa2ac4..874bd45becf0c9 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -349,3 +349,5 @@ static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { [FVC_REPR] = PyObject_Repr, [FVC_ASCII] = PyObject_ASCII }; + +#define ASSERT_KWNAMES_IS_NULL() assert(kwnames == NULL) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ae21ffad94d801..d85e23b5abb8e6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1904,6 +1904,68 @@ break; } + case CALL_NO_KW_TYPE_1: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_STR_1: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_TUPLE_1: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + case EXIT_INIT_CHECK: { PyObject *should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); @@ -1917,6 +1979,273 @@ break; } + case CALL_NO_KW_BUILTIN_O: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + /* Builtin METH_O functions */ + ASSERT_KWNAMES_IS_NULL(); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_BUILTIN_FAST: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + /* Builtin METH_FASTCALL functions, without keywords */ + ASSERT_KWNAMES_IS_NULL(); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_LEN: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + /* len(o) */ + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + goto error; + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_ISINSTANCE: { + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + /* isinstance(o, o2) */ + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + goto error; + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_O: { + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } + PyMethodDescrObject *callable = + (PyMethodDescrObject *)PEEK(total_args + 1); + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = callable->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + assert(oparg == 0 || oparg == 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = callable->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *res; + ASSERT_KWNAMES_IS_NULL(); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } + PyMethodDescrObject *callable = + (PyMethodDescrObject *)PEEK(total_args + 1); + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = callable->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + case MAKE_FUNCTION: { PyObject *codeobj = stack_pointer[-1]; PyObject *func; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 392914c0521e9d..1fd76715dc3e4a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3462,7 +3462,7 @@ } TARGET(KW_NAMES) { - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg < PyTuple_GET_SIZE(FRAME_CO_CONSTS)); kwnames = GETITEM(FRAME_CO_CONSTS, oparg); DISPATCH(); @@ -3599,7 +3599,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; int argcount = oparg; @@ -3631,7 +3631,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; int argcount = oparg; @@ -3673,7 +3673,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; @@ -3694,7 +3694,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -3717,7 +3717,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); @@ -3744,7 +3744,7 @@ * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); @@ -3844,7 +3844,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; /* Builtin METH_O functions */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3884,7 +3884,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; /* Builtin METH_FASTCALL functions, without keywords */ - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3971,7 +3971,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; int total_args = oparg; @@ -4007,7 +4007,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; int total_args = oparg; @@ -4044,7 +4044,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -4067,7 +4067,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4149,7 +4149,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; int total_args = oparg; @@ -4189,7 +4189,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - assert(kwnames == NULL); + ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; if (is_meth) { diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index a0a8b8cbe4baba..112f29a83e4c10 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -408,27 +408,31 @@ def __init__(self, inst: parser.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" + dprint: typing.Callable[..., None] = lambda *args, **kwargs: None + # if self.name.startswith("CALL"): + # dprint = print + if self.name == "EXIT_TRACE": return True # This has 'return frame' but it's okay if self.always_exits: - # print(f"Skipping {self.name} because it always exits") + dprint(f"Skipping {self.name} because it always exits") return False if self.instr_flags.HAS_ARG_FLAG: # If the instruction uses oparg, it cannot use any caches if self.active_caches: - # print(f"Skipping {self.name} because it uses oparg and caches") + dprint(f"Skipping {self.name} because it uses oparg and caches") return False else: # If it doesn't use oparg, it can have one cache entry if len(self.active_caches) > 1: - # print(f"Skipping {self.name} because it has >1 cache entries") + dprint(f"Skipping {self.name} because it has >1 cache entries") return False res = True for forbidden in FORBIDDEN_NAMES_IN_UOPS: # NOTE: To disallow unspecialized uops, use # if variable_used(self.inst, forbidden): if variable_used_unspecialized(self.inst, forbidden): - # print(f"Skipping {self.name} because it uses {forbidden}") + dprint(f"Skipping {self.name} because it uses {forbidden}") res = False return res @@ -1499,6 +1503,8 @@ def write_executor_instructions(self) -> None: with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) self.out.emit("break;") + # elif instr.kind != "op": + # print(f"NOTE: {thing.name} is not a viable uop") case parser.Macro(): pass case parser.Pseudo(): From 7e96370a946a2ca0f2f25af4ce5b3b59f020721b Mon Sep 17 00:00:00 2001 From: Mario Corchero Date: Mon, 17 Jul 2023 20:57:40 +0200 Subject: [PATCH 52/75] gh-61215: threadingmock: Improve test suite to avoid race conditions (#106822) threadingmock: Improve test suite to avoid race conditions Simplify tests and split them into multiple tests to prevent assertions from triggering race conditions. Additionally, we rely on calling the mocks without delay to validate the functionality of matching calls. --- .../testmock/testthreadingmock.py | 195 ++++++------------ 1 file changed, 58 insertions(+), 137 deletions(-) diff --git a/Lib/test/test_unittest/testmock/testthreadingmock.py b/Lib/test/test_unittest/testmock/testthreadingmock.py index b6e12bcb3cda9c..94e71921d9bc03 100644 --- a/Lib/test/test_unittest/testmock/testthreadingmock.py +++ b/Lib/test/test_unittest/testmock/testthreadingmock.py @@ -8,6 +8,8 @@ threading_helper.requires_working_threading(module=True) +VERY_SHORT_TIMEOUT = 0.1 + class Something: def method_1(self): @@ -93,167 +95,86 @@ def test_no_name_clash(self): waitable_mock.wait_until_called() waitable_mock.wait_until_any_call_with("works") - def test_wait_success(self): + def test_patch(self): waitable_mock = self._make_mock(spec=Something) - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, delay=0.01) - something.method_1.wait_until_called() - something.method_1.wait_until_any_call_with() - something.method_1.assert_called() - - def test_wait_success_with_instance_timeout(self): - waitable_mock = self._make_mock(timeout=1) - - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, delay=0.01) - something.method_1.wait_until_called() - something.method_1.wait_until_any_call_with() - something.method_1.assert_called() - - def test_wait_failed_with_instance_timeout(self): - waitable_mock = self._make_mock(timeout=0.01) - - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, delay=0.5) - self.assertRaises(AssertionError, waitable_mock.method_1.wait_until_called) - self.assertRaises( - AssertionError, waitable_mock.method_1.wait_until_any_call_with - ) - - def test_wait_success_with_timeout_override(self): - waitable_mock = self._make_mock(timeout=0.01) - - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, delay=0.05) - something.method_1.wait_until_called(timeout=1) - - def test_wait_failed_with_timeout_override(self): - waitable_mock = self._make_mock(timeout=1) - - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, delay=0.5) - with self.assertRaises(AssertionError): - something.method_1.wait_until_called(timeout=0.05) - - def test_wait_success_called_before(self): - waitable_mock = self._make_mock() - with patch(f"{__name__}.Something", waitable_mock): something = Something() something.method_1() something.method_1.wait_until_called() - something.method_1.wait_until_any_call_with() - something.method_1.assert_called() - - def test_wait_magic_method(self): - waitable_mock = self._make_mock() - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1.__str__, delay=0.01) - something.method_1.__str__.wait_until_called() - something.method_1.__str__.assert_called() - - def test_wait_until_any_call_with_positional(self): + def test_wait_already_called_success(self): waitable_mock = self._make_mock(spec=Something) + waitable_mock.method_1() + waitable_mock.method_1.wait_until_called() + waitable_mock.method_1.wait_until_any_call_with() + waitable_mock.method_1.assert_called() - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, 1, delay=0.2) - self.assertNotIn(call(1), something.method_1.mock_calls) - self.run_async(something.method_1, 2, delay=0.5) - self.run_async(something.method_1, 3, delay=0.6) - - something.method_1.wait_until_any_call_with(1) - something.method_1.assert_called_with(1) - self.assertNotIn(call(2), something.method_1.mock_calls) - self.assertNotIn(call(3), something.method_1.mock_calls) - - something.method_1.wait_until_any_call_with(3) - self.assertIn(call(2), something.method_1.mock_calls) - something.method_1.wait_until_any_call_with(2) - - def test_wait_until_any_call_with_keywords(self): + def test_wait_until_called_success(self): waitable_mock = self._make_mock(spec=Something) + self.run_async(waitable_mock.method_1, delay=VERY_SHORT_TIMEOUT) + waitable_mock.method_1.wait_until_called() - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - self.run_async(something.method_1, a=1, delay=0.2) - self.assertNotIn(call(a=1), something.method_1.mock_calls) - self.run_async(something.method_1, b=2, delay=0.5) - self.run_async(something.method_1, c=3, delay=0.6) - - something.method_1.wait_until_any_call_with(a=1) - something.method_1.assert_called_with(a=1) - self.assertNotIn(call(b=2), something.method_1.mock_calls) - self.assertNotIn(call(c=3), something.method_1.mock_calls) - - something.method_1.wait_until_any_call_with(c=3) - self.assertIn(call(b=2), something.method_1.mock_calls) - something.method_1.wait_until_any_call_with(b=2) - - def test_wait_until_any_call_with_no_argument_fails_when_called_with_arg(self): - waitable_mock = self._make_mock(timeout=0.01) - - with patch(f"{__name__}.Something", waitable_mock): - something = Something() - something.method_1(1) - - something.method_1.assert_called_with(1) - with self.assertRaises(AssertionError): - something.method_1.wait_until_any_call_with() + def test_wait_until_called_method_timeout(self): + waitable_mock = self._make_mock(spec=Something) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called(timeout=VERY_SHORT_TIMEOUT) - something.method_1() - something.method_1.wait_until_any_call_with() + def test_wait_until_called_instance_timeout(self): + waitable_mock = self._make_mock(spec=Something, timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called() - def test_wait_until_any_call_with_global_default(self): + def test_wait_until_called_global_timeout(self): with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): - ThreadingMock.DEFAULT_TIMEOUT = 0.01 - m = self._make_mock() + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock(spec=Something) with self.assertRaises(AssertionError): - m.wait_until_any_call_with() - with self.assertRaises(AssertionError): - m.wait_until_called() + waitable_mock.method_1.wait_until_called() - m() - m.wait_until_any_call_with() - assert ThreadingMock.DEFAULT_TIMEOUT != 0.01 + def test_wait_until_any_call_with_success(self): + waitable_mock = self._make_mock() + self.run_async(waitable_mock, delay=VERY_SHORT_TIMEOUT) + waitable_mock.wait_until_any_call_with() - def test_wait_until_any_call_with_change_global_and_override(self): - with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): - ThreadingMock.DEFAULT_TIMEOUT = 0.01 + def test_wait_until_any_call_with_instance_timeout(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.wait_until_any_call_with() - m1 = self._make_mock() - self.run_async(m1, delay=0.1) + def test_wait_until_any_call_global_timeout(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock() with self.assertRaises(AssertionError): - m1.wait_until_called() + waitable_mock.wait_until_any_call_with() - m2 = self._make_mock(timeout=1) - self.run_async(m2, delay=0.1) - m2.wait_until_called() - - m3 = self._make_mock() - self.run_async(m3, delay=0.1) - m3.wait_until_called(timeout=1) - - m4 = self._make_mock() - self.run_async(m4, delay=0.1) - m4.wait_until_called(timeout=None) + def test_wait_until_any_call_positional(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(1, 2, 3) + waitable_mock.method_1.wait_until_any_call_with(1, 2, 3) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(2, 3, 1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() - m5 = self._make_mock(timeout=None) - self.run_async(m5, delay=0.1) - m5.wait_until_called() + def test_wait_until_any_call_kw(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(a=1, b=2) + waitable_mock.method_1.wait_until_any_call_with(a=1, b=2) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(a=2, b=1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() - assert ThreadingMock.DEFAULT_TIMEOUT != 0.01 + def test_magic_methods_success(self): + waitable_mock = self._make_mock() + str(waitable_mock) + waitable_mock.__str__.wait_until_called() + waitable_mock.__str__.assert_called() def test_reset_mock_resets_wait(self): - m = self._make_mock(timeout=0.01) + m = self._make_mock(timeout=VERY_SHORT_TIMEOUT) with self.assertRaises(AssertionError): m.wait_until_called() From 8e9a1a032233f06ce0f1acdf5f983d614c8745a5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 17 Jul 2023 12:12:33 -0700 Subject: [PATCH 53/75] gh-106603: Make uop struct a triple (opcode, oparg, operand) (#106794) --- Include/internal/pycore_opcode_metadata.h | 42 +++++++---- Include/internal/pycore_uops.h | 5 +- Lib/test/test_capi/test_misc.py | 24 +++--- Python/bytecodes.c | 16 ++-- Python/ceval.c | 7 +- Python/executor_cases.c.h | 85 +++++++++++++++++++-- Python/generated_cases.c.h | 14 +--- Python/optimizer.c | 91 +++++++++++++---------- Tools/cases_generator/generate_cases.py | 15 +--- 9 files changed, 190 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 028736e115b3f4..c3a0dbb478a7c1 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -38,21 +38,24 @@ #define _SKIP_CACHE 314 #define _GUARD_GLOBALS_VERSION 315 #define _GUARD_BUILTINS_VERSION 316 -#define _GUARD_TYPE_VERSION 317 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 318 -#define IS_NONE 319 -#define _ITER_CHECK_LIST 320 -#define _IS_ITER_EXHAUSTED_LIST 321 -#define _ITER_NEXT_LIST 322 -#define _ITER_CHECK_TUPLE 323 -#define _IS_ITER_EXHAUSTED_TUPLE 324 -#define _ITER_NEXT_TUPLE 325 -#define _ITER_CHECK_RANGE 326 -#define _IS_ITER_EXHAUSTED_RANGE 327 -#define _ITER_NEXT_RANGE 328 -#define _POP_JUMP_IF_FALSE 329 -#define _POP_JUMP_IF_TRUE 330 -#define JUMP_TO_TOP 331 +#define _LOAD_GLOBAL_MODULE 317 +#define _LOAD_GLOBAL_BUILTINS 318 +#define _GUARD_TYPE_VERSION 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _LOAD_ATTR_INSTANCE_VALUE 321 +#define IS_NONE 322 +#define _ITER_CHECK_LIST 323 +#define _IS_ITER_EXHAUSTED_LIST 324 +#define _ITER_NEXT_LIST 325 +#define _ITER_CHECK_TUPLE 326 +#define _IS_ITER_EXHAUSTED_TUPLE 327 +#define _ITER_NEXT_TUPLE 328 +#define _ITER_CHECK_RANGE 329 +#define _IS_ITER_EXHAUSTED_RANGE 330 +#define _ITER_NEXT_RANGE 331 +#define _POP_JUMP_IF_FALSE 332 +#define _POP_JUMP_IF_TRUE 333 +#define JUMP_TO_TOP 334 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -1245,7 +1248,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 1, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 0, 0 } } }, [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, @@ -1264,6 +1267,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { STORE_ATTR, 0, 0 } } }, [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, @@ -1271,6 +1275,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_GLOBALS_VERSION, 1, 1 }, { _SKIP_CACHE, 0, 0 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, @@ -1292,6 +1298,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, @@ -1348,8 +1355,11 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_SKIP_CACHE] = "_SKIP_CACHE", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", + [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", [IS_NONE] = "IS_NONE", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST", diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 5ed275fb857679..edb141cc79f752 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -11,8 +11,9 @@ extern "C" { #define _Py_UOP_MAX_TRACE_LENGTH 32 typedef struct { - int opcode; - uint64_t operand; // Sometimes oparg, sometimes a cache entry + uint32_t opcode; + uint32_t oparg; + uint64_t operand; // A cache entry } _PyUOpInstruction; typedef struct { diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c0dcff825758ad..4e519fa73c50cc 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2448,7 +2448,7 @@ def testfunc(x): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("SAVE_IP", uops) self.assertIn("LOAD_FAST", uops) @@ -2493,7 +2493,7 @@ def many_vars(): ex = get_first_executor(many_vars) self.assertIsNotNone(ex) - self.assertIn(("LOAD_FAST", 259), list(ex)) + self.assertIn(("LOAD_FAST", 259, 0), list(ex)) def test_unspecialized_unpack(self): # An example of an unspecialized opcode @@ -2514,7 +2514,7 @@ def testfunc(x): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("UNPACK_SEQUENCE", uops) def test_pop_jump_if_false(self): @@ -2529,7 +2529,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_POP_JUMP_IF_FALSE", uops) def test_pop_jump_if_none(self): @@ -2544,7 +2544,7 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_POP_JUMP_IF_TRUE", uops) def test_pop_jump_if_not_none(self): @@ -2559,7 +2559,7 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_POP_JUMP_IF_FALSE", uops) def test_pop_jump_if_true(self): @@ -2574,7 +2574,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_POP_JUMP_IF_TRUE", uops) def test_jump_backward(self): @@ -2589,7 +2589,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("JUMP_TO_TOP", uops) def test_jump_forward(self): @@ -2609,7 +2609,7 @@ def testfunc(n): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} # Since there is no JUMP_FORWARD instruction, # look for indirect evidence: the += operator self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -2630,7 +2630,7 @@ def testfunc(n): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_IS_ITER_EXHAUSTED_RANGE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -2652,7 +2652,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_IS_ITER_EXHAUSTED_LIST", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -2674,7 +2674,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _ in ex} + uops = {opname for opname, _, _ in ex} self.assertIn("_IS_ITER_EXHAUSTED_TUPLE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 652372cb23dc5e..19fb138ee64cba 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -645,18 +645,16 @@ dummy_func( STORE_SUBSCR_LIST_INT, }; - inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) { + inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -1198,19 +1196,17 @@ dummy_func( STORE_ATTR_WITH_HINT, }; - inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) { + inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); diff --git a/Python/ceval.c b/Python/ceval.c index f13ba9883d9814..b56ddfb4bd286d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2747,17 +2747,18 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; int pc = 0; int opcode; - uint64_t operand; int oparg; + uint64_t operand; for (;;) { opcode = self->trace[pc].opcode; + oparg = self->trace[pc].oparg; operand = self->trace[pc].operand; - oparg = (int)operand; DPRINTF(3, - "%4d: uop %s, operand %" PRIu64 ", stack_level %d\n", + "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n", pc, opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode], + oparg, operand, (int)(stack_pointer - _PyFrame_Stackbase(frame))); pc++; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d85e23b5abb8e6..f492c1fa9d8e3f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -485,18 +485,15 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; - uint16_t counter = (uint16_t)operand; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -849,6 +846,30 @@ break; } + case STORE_ATTR: { + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner = stack_pointer[-1]; + PyObject *v = stack_pointer[-2]; + #if ENABLE_SPECIALIZATION + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr--; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + #endif /* ENABLE_SPECIALIZATION */ + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + STACK_SHRINK(2); + break; + } + case DELETE_ATTR: { PyObject *owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1010,6 +1031,42 @@ break; } + case _LOAD_GLOBAL_MODULE: { + PyObject *null = NULL; + PyObject *res; + uint16_t index = (uint16_t)operand; + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + PyObject *null = NULL; + PyObject *res; + uint16_t index = (uint16_t)operand; + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } + break; + } + case DELETE_FAST: { PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; @@ -1443,6 +1500,24 @@ break; } + case _LOAD_ATTR_INSTANCE_VALUE: { + PyObject *owner = stack_pointer[-1]; + PyObject *res2 = NULL; + PyObject *res; + uint16_t index = (uint16_t)operand; + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + res = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + res2 = NULL; + Py_DECREF(owner); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + break; + } + case COMPARE_OP: { static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1fd76715dc3e4a..0148078d18bdc3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -773,18 +773,15 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; - uint16_t counter = read_u16(&next_instr[0].cache); #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -1437,19 +1434,16 @@ static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; - uint16_t counter = read_u16(&next_instr[0].cache); #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); - #else - (void)counter; // Unused. #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); diff --git a/Python/optimizer.c b/Python/optimizer.c index 693ba375971ae7..3d385a1506cba3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -344,13 +344,19 @@ uop_item(_PyUOpExecutorObject *self, Py_ssize_t index) if (oname == NULL) { return NULL; } + PyObject *oparg = PyLong_FromUnsignedLong(self->trace[index].oparg); + if (oparg == NULL) { + Py_DECREF(oname); + return NULL; + } PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); if (operand == NULL) { + Py_DECREF(oparg); Py_DECREF(oname); return NULL; } - PyObject *args[2] = { oname, operand }; - return _PyTuple_FromArraySteal(args, 2); + PyObject *args[3] = { oname, oparg, operand }; + return _PyTuple_FromArraySteal(args, 3); } PySequenceMethods uop_as_sequence = { @@ -395,29 +401,33 @@ translate_bytecode_to_trace( #define DPRINTF(level, ...) #endif -#define ADD_TO_TRACE(OPCODE, OPERAND) \ +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND) \ DPRINTF(2, \ - " ADD_TO_TRACE(%s, %" PRIu64 ")\n", \ + " ADD_TO_TRACE(%s, %d, %" PRIu64 ")\n", \ uop_name(OPCODE), \ + (OPARG), \ (uint64_t)(OPERAND)); \ assert(trace_length < max_length); \ assert(reserved > 0); \ reserved--; \ trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].oparg = (OPARG); \ trace[trace_length].operand = (OPERAND); \ trace_length++; #define INSTR_IP(INSTR, CODE) \ - ((long)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) + ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) -#define ADD_TO_STUB(INDEX, OPCODE, OPERAND) \ - DPRINTF(2, " ADD_TO_STUB(%d, %s, %" PRIu64 ")\n", \ +#define ADD_TO_STUB(INDEX, OPCODE, OPARG, OPERAND) \ + DPRINTF(2, " ADD_TO_STUB(%d, %s, %d, %" PRIu64 ")\n", \ (INDEX), \ uop_name(OPCODE), \ + (OPARG), \ (uint64_t)(OPERAND)); \ assert(reserved > 0); \ reserved--; \ trace[(INDEX)].opcode = (OPCODE); \ + trace[(INDEX)].oparg = (OPARG); \ trace[(INDEX)].operand = (OPERAND); // Reserve space for n uops @@ -433,7 +443,7 @@ translate_bytecode_to_trace( #define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 2, uop_name(opcode)) DPRINTF(4, - "Optimizing %s (%s:%d) at byte offset %ld\n", + "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, @@ -441,11 +451,11 @@ translate_bytecode_to_trace( for (;;) { RESERVE_RAW(2, "epilogue"); // Always need space for SAVE_IP and EXIT_TRACE - ADD_TO_TRACE(SAVE_IP, INSTR_IP(instr, code)); + ADD_TO_TRACE(SAVE_IP, INSTR_IP(instr, code), 0); - int opcode = instr->op.code; - int oparg = instr->op.arg; - int extras = 0; + uint32_t opcode = instr->op.code; + uint32_t oparg = instr->op.arg; + uint32_t extras = 0; while (opcode == EXTENDED_ARG) { instr++; @@ -467,7 +477,7 @@ translate_bytecode_to_trace( case POP_JUMP_IF_NONE: { RESERVE(2, 2); - ADD_TO_TRACE(IS_NONE, 0); + ADD_TO_TRACE(IS_NONE, 0, 0); opcode = POP_JUMP_IF_TRUE; goto pop_jump_if_bool; } @@ -475,7 +485,7 @@ translate_bytecode_to_trace( case POP_JUMP_IF_NOT_NONE: { RESERVE(2, 2); - ADD_TO_TRACE(IS_NONE, 0); + ADD_TO_TRACE(IS_NONE, 0, 0); opcode = POP_JUMP_IF_FALSE; goto pop_jump_if_bool; } @@ -489,11 +499,11 @@ translate_bytecode_to_trace( _Py_CODEUNIT *target_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg; max_length -= 2; // Really the start of the stubs - int uopcode = opcode == POP_JUMP_IF_TRUE ? + uint32_t uopcode = opcode == POP_JUMP_IF_TRUE ? _POP_JUMP_IF_TRUE : _POP_JUMP_IF_FALSE; - ADD_TO_TRACE(uopcode, max_length); - ADD_TO_STUB(max_length, SAVE_IP, INSTR_IP(target_instr, code)); - ADD_TO_STUB(max_length + 1, EXIT_TRACE, 0); + ADD_TO_TRACE(uopcode, max_length, 0); + ADD_TO_STUB(max_length, SAVE_IP, INSTR_IP(target_instr, code), 0); + ADD_TO_STUB(max_length + 1, EXIT_TRACE, 0, 0); break; } @@ -501,7 +511,7 @@ translate_bytecode_to_trace( { if (instr + 2 - oparg == initial_instr) { RESERVE(1, 0); - ADD_TO_TRACE(JUMP_TO_TOP, 0); + ADD_TO_TRACE(JUMP_TO_TOP, 0, 0); } else { DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); @@ -546,14 +556,14 @@ translate_bytecode_to_trace( _Py_CODEUNIT *target_instr = // +1 at the end skips over END_FOR instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg + 1; max_length -= 3; // Really the start of the stubs - ADD_TO_TRACE(check_op, 0); - ADD_TO_TRACE(exhausted_op, 0); - ADD_TO_TRACE(_POP_JUMP_IF_TRUE, max_length); - ADD_TO_TRACE(next_op, 0); - - ADD_TO_STUB(max_length + 0, POP_TOP, 0); - ADD_TO_STUB(max_length + 1, SAVE_IP, INSTR_IP(target_instr, code)); - ADD_TO_STUB(max_length + 2, EXIT_TRACE, 0); + ADD_TO_TRACE(check_op, 0, 0); + ADD_TO_TRACE(exhausted_op, 0, 0); + ADD_TO_TRACE(_POP_JUMP_IF_TRUE, max_length, 0); + ADD_TO_TRACE(next_op, 0, 0); + + ADD_TO_STUB(max_length + 0, POP_TOP, 0, 0); + ADD_TO_STUB(max_length + 1, SAVE_IP, INSTR_IP(target_instr, code), 0); + ADD_TO_STUB(max_length + 2, EXIT_TRACE, 0, 0); break; } @@ -564,19 +574,20 @@ translate_bytecode_to_trace( // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE) int nuops = expansion->nuops; RESERVE(nuops, 0); + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { - uint64_t operand; + oparg = orig_oparg; + uint64_t operand = 0; int offset = expansion->uops[i].offset; switch (expansion->uops[i].size) { case OPARG_FULL: - operand = oparg; if (extras && OPCODE_HAS_JUMP(opcode)) { if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { - operand -= extras; + oparg -= extras; } else { assert(opcode != JUMP_BACKWARD); - operand += extras; + oparg += extras; } } break; @@ -590,10 +601,10 @@ translate_bytecode_to_trace( operand = read_u64(&instr[offset].cache); break; case OPARG_TOP: // First half of super-instr - operand = oparg >> 4; + oparg = orig_oparg >> 4; break; case OPARG_BOTTOM: // Second half of super-instr - operand = oparg & 0xF; + oparg = orig_oparg & 0xF; break; default: fprintf(stderr, @@ -603,7 +614,7 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(expansion->uops[i].uop, operand); + ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand); } break; } @@ -621,9 +632,9 @@ translate_bytecode_to_trace( done: // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE if (trace_length > 3) { - ADD_TO_TRACE(EXIT_TRACE, 0); + ADD_TO_TRACE(EXIT_TRACE, 0, 0); DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %ld -- length %d\n", + "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, @@ -644,10 +655,10 @@ translate_bytecode_to_trace( if (trace[i].opcode == _POP_JUMP_IF_FALSE || trace[i].opcode == _POP_JUMP_IF_TRUE) { - uint64_t target = trace[i].operand; - if (target >= (uint64_t)max_length) { + int target = trace[i].oparg; + if (target >= max_length) { target += trace_length - max_length; - trace[i].operand = target; + trace[i].oparg = target; } } } @@ -657,7 +668,7 @@ translate_bytecode_to_trace( } else { DPRINTF(4, - "No trace for %s (%s:%d) at byte offset %ld\n", + "No trace for %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 112f29a83e4c10..037bee107cb13a 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -417,16 +417,9 @@ def is_viable_uop(self) -> bool: if self.always_exits: dprint(f"Skipping {self.name} because it always exits") return False - if self.instr_flags.HAS_ARG_FLAG: - # If the instruction uses oparg, it cannot use any caches - if self.active_caches: - dprint(f"Skipping {self.name} because it uses oparg and caches") - return False - else: - # If it doesn't use oparg, it can have one cache entry - if len(self.active_caches) > 1: - dprint(f"Skipping {self.name} because it has >1 cache entries") - return False + if len(self.active_caches) > 1: + # print(f"Skipping {self.name} because it has >1 cache entries") + return False res = True for forbidden in FORBIDDEN_NAMES_IN_UOPS: # NOTE: To disallow unspecialized uops, use @@ -1374,7 +1367,7 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: if not part.instr.is_viable_uop(): print(f"NOTE: Part {part.instr.name} of {name} is not a viable uop") return - if part.instr.instr_flags.HAS_ARG_FLAG or not part.active_caches: + if not part.active_caches: size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: # If this assert triggers, is_viable_uops() lied From ebf2c56b33553a448da8f60fcd89a622f071b5f4 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 17 Jul 2023 22:55:40 +0300 Subject: [PATCH 54/75] gh-106831: Fix NULL check of d2i_SSL_SESSION() result in _ssl.c (#106832) --- .../Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst | 2 ++ Modules/_ssl.c | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst diff --git a/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst b/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst new file mode 100644 index 00000000000000..d3b98626845392 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst @@ -0,0 +1,2 @@ +Fix potential missing ``NULL`` check of ``d2i_SSL_SESSION`` result in +``_ssl.c``. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 0cf4d3e9dc8c9b..8612b3dd53924c 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2808,7 +2808,7 @@ _ssl_session_dup(SSL_SESSION *session) { /* get length */ slen = i2d_SSL_SESSION(session, NULL); if (slen == 0 || slen > 0xFF00) { - PyErr_SetString(PyExc_ValueError, "i2d() failed."); + PyErr_SetString(PyExc_ValueError, "i2d() failed"); goto error; } if ((senc = PyMem_Malloc(slen)) == NULL) { @@ -2817,12 +2817,13 @@ _ssl_session_dup(SSL_SESSION *session) { } p = senc; if (!i2d_SSL_SESSION(session, &p)) { - PyErr_SetString(PyExc_ValueError, "i2d() failed."); + PyErr_SetString(PyExc_ValueError, "i2d() failed"); goto error; } const_p = senc; newsession = d2i_SSL_SESSION(NULL, &const_p, slen); - if (session == NULL) { + if (newsession == NULL) { + PyErr_SetString(PyExc_ValueError, "d2i() failed"); goto error; } PyMem_Free(senc); From 22379c60ab8f8b49e75da9bd032a8722af50b409 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 17 Jul 2023 22:55:10 +0200 Subject: [PATCH 55/75] gh-106368: Increase Argument Clinic test coverage for cpp.Monitor (#106833) --- Lib/test/clinic.test.c | 132 +++++++++++++++++++++++++++++++++++++++- Lib/test/test_clinic.py | 49 +++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 2fd8760415dc72..da99e58c77f021 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -3726,6 +3726,47 @@ test_preprocessor_guarded_else_impl(PyObject *module) /*[clinic end generated code: output=13af7670aac51b12 input=6657ab31d74c29fc]*/ #endif +#ifndef CONDITION_C +/*[clinic input] +test_preprocessor_guarded_ifndef_condition_c +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_ifndef_condition_c_impl(PyObject *module) +/*[clinic end generated code: output=ed422e8c895bb0a5 input=e9b50491cea2b668]*/ +#else +/*[clinic input] +test_preprocessor_guarded_ifndef_not_condition_c +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_ifndef_not_condition_c_impl(PyObject *module) +/*[clinic end generated code: output=de6f4c6a67f8c536 input=da74e30e01c6f2c5]*/ +#endif + +#if \ +CONDITION_D +/*[clinic input] +test_preprocessor_guarded_if_with_continuation +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_if_with_continuation_impl(PyObject *module) +/*[clinic end generated code: output=3d0712ca9e2d15b9 input=4a956fd91be30284]*/ +#endif + +#if CONDITION_E || CONDITION_F +#warning "different type of CPP directive" +/*[clinic input] +test_preprocessor_guarded_if_e_or_f +Makes sure cpp.Monitor handles other directives than preprocessor conditionals. +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_if_e_or_f_impl(PyObject *module) +/*[clinic end generated code: output=e49d24ff64ad88bc input=57b9c37f938bc4f1]*/ +#endif + /*[clinic input] dump buffer output pop @@ -3785,6 +3826,79 @@ test_preprocessor_guarded_else(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* !defined(CONDITION_A) && !(CONDITION_B) */ +#if !defined(CONDITION_C) + +PyDoc_STRVAR(test_preprocessor_guarded_ifndef_condition_c__doc__, +"test_preprocessor_guarded_ifndef_condition_c($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_IFNDEF_CONDITION_C_METHODDEF \ + {"test_preprocessor_guarded_ifndef_condition_c", (PyCFunction)test_preprocessor_guarded_ifndef_condition_c, METH_NOARGS, test_preprocessor_guarded_ifndef_condition_c__doc__}, + +static PyObject * +test_preprocessor_guarded_ifndef_condition_c(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_ifndef_condition_c_impl(module); +} + +#endif /* !defined(CONDITION_C) */ + +#if defined(CONDITION_C) + +PyDoc_STRVAR(test_preprocessor_guarded_ifndef_not_condition_c__doc__, +"test_preprocessor_guarded_ifndef_not_condition_c($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_IFNDEF_NOT_CONDITION_C_METHODDEF \ + {"test_preprocessor_guarded_ifndef_not_condition_c", (PyCFunction)test_preprocessor_guarded_ifndef_not_condition_c, METH_NOARGS, test_preprocessor_guarded_ifndef_not_condition_c__doc__}, + +static PyObject * +test_preprocessor_guarded_ifndef_not_condition_c(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_ifndef_not_condition_c_impl(module); +} + +#endif /* defined(CONDITION_C) */ + +#if (CONDITION_D) + +PyDoc_STRVAR(test_preprocessor_guarded_if_with_continuation__doc__, +"test_preprocessor_guarded_if_with_continuation($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_IF_WITH_CONTINUATION_METHODDEF \ + {"test_preprocessor_guarded_if_with_continuation", (PyCFunction)test_preprocessor_guarded_if_with_continuation, METH_NOARGS, test_preprocessor_guarded_if_with_continuation__doc__}, + +static PyObject * +test_preprocessor_guarded_if_with_continuation(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_if_with_continuation_impl(module); +} + +#endif /* (CONDITION_D) */ + +#if (CONDITION_E || CONDITION_F) + +PyDoc_STRVAR(test_preprocessor_guarded_if_e_or_f__doc__, +"test_preprocessor_guarded_if_e_or_f($module, /)\n" +"--\n" +"\n" +"Makes sure cpp.Monitor handles other directives than preprocessor conditionals."); + +#define TEST_PREPROCESSOR_GUARDED_IF_E_OR_F_METHODDEF \ + {"test_preprocessor_guarded_if_e_or_f", (PyCFunction)test_preprocessor_guarded_if_e_or_f, METH_NOARGS, test_preprocessor_guarded_if_e_or_f__doc__}, + +static PyObject * +test_preprocessor_guarded_if_e_or_f(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_if_e_or_f_impl(module); +} + +#endif /* (CONDITION_E || CONDITION_F) */ + #ifndef TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF #define TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF #endif /* !defined(TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF) */ @@ -3796,7 +3910,23 @@ test_preprocessor_guarded_else(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF #define TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF #endif /* !defined(TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF) */ -/*[clinic end generated code: output=3804bb18d454038c input=3fc80c9989d2f2e1]*/ + +#ifndef TEST_PREPROCESSOR_GUARDED_IFNDEF_CONDITION_C_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_IFNDEF_CONDITION_C_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_IFNDEF_CONDITION_C_METHODDEF) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_IFNDEF_NOT_CONDITION_C_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_IFNDEF_NOT_CONDITION_C_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_IFNDEF_NOT_CONDITION_C_METHODDEF) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_IF_WITH_CONTINUATION_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_IF_WITH_CONTINUATION_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_IF_WITH_CONTINUATION_METHODDEF) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_IF_E_OR_F_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_IF_E_OR_F_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_IF_E_OR_F_METHODDEF) */ +/*[clinic end generated code: output=fcfae7cac7a99e62 input=3fc80c9989d2f2e1]*/ /*[clinic input] test_vararg_and_posonly diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index b5744f7013d6ad..e925ecca1b9c5d 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -200,6 +200,55 @@ def test_parse_with_body_prefix(self): """).lstrip() # Note, lstrip() because of the newline self.assertEqual(out, expected) + def test_cpp_monitor_fail_nested_block_comment(self): + raw = """ + /* start + /* nested + */ + */ + """ + msg = ( + 'Error in file "test.c" on line 2:\n' + 'Nested block comment!\n' + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_cpp_monitor_fail_invalid_format_noarg(self): + raw = """ + #if + a() + #endif + """ + msg = ( + 'Error in file "test.c" on line 1:\n' + 'Invalid format for #if line: no argument!\n' + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_cpp_monitor_fail_invalid_format_toomanyargs(self): + raw = """ + #ifdef A B + a() + #endif + """ + msg = ( + 'Error in file "test.c" on line 1:\n' + 'Invalid format for #ifdef line: should be exactly one argument!\n' + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + + def test_cpp_monitor_fail_no_matching_if(self): + raw = '#else' + msg = ( + 'Error in file "test.c" on line 1:\n' + '#else without matching #if / #ifdef / #ifndef!\n' + ) + out = self.expect_failure(raw) + self.assertEqual(out, msg) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): From 1654916c4864a741507617020453147acf20c9f3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jul 2023 00:10:03 +0200 Subject: [PATCH 56/75] Add Erlend as CODEOWNER for Argument Clinic docs (#106840) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 234a954cc7662f..882ba9e9c9ebea 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -177,3 +177,4 @@ Doc/c-api/stable.rst @encukou # Argument Clinic /Tools/clinic/** @erlend-aasland @AlexWaygood /Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +Doc/howto/clinic.rst @erlend-aasland From 00e52acebd2beb2663202bfc4be0ce79ba77361e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jul 2023 00:37:11 +0200 Subject: [PATCH 57/75] gh-104683: Argument Clinic: Modernise parse_special_symbol() (#106837) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 136 ++++++++++++++++++++++++----------------- 1 file changed, 80 insertions(+), 56 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 311f0a1a56a038..bff8935df13bc6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4864,11 +4864,21 @@ def state_parameter(self, line: str | None) -> None: self.parameter_continuation = line[:-1] return - line = line.lstrip() - - if line in ('*', '/', '[', ']'): - self.parse_special_symbol(line) - return + func = self.function + match line.lstrip(): + case '*': + self.parse_star(func) + case '[': + self.parse_opening_square_bracket(func) + case ']': + self.parse_closing_square_bracket(func) + case '/': + self.parse_slash(func) + case param: + self.parse_parameter(param) + + def parse_parameter(self, line: str) -> None: + assert self.function is not None match self.parameter_state: case ParamState.START | ParamState.REQUIRED: @@ -5146,57 +5156,71 @@ def parse_converter( "Annotations must be either a name, a function call, or a string." ) - def parse_special_symbol(self, symbol): - if symbol == '*': - if self.keyword_only: - fail("Function " + self.function.name + " uses '*' more than once.") - self.keyword_only = True - elif symbol == '[': - match self.parameter_state: - case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: - self.parameter_state = ParamState.LEFT_SQUARE_BEFORE - case ParamState.REQUIRED | ParamState.GROUP_AFTER: - self.parameter_state = ParamState.GROUP_AFTER - case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.b)") - self.group += 1 - self.function.docstring_only = True - elif symbol == ']': - if not self.group: - fail("Function " + self.function.name + " has a ] without a matching [.") - if not any(p.group == self.group for p in self.function.parameters.values()): - fail("Function " + self.function.name + " has an empty group.\nAll groups must contain at least one parameter.") - self.group -= 1 - match self.parameter_state: - case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: - self.parameter_state = ParamState.GROUP_BEFORE - case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: - self.parameter_state = ParamState.RIGHT_SQUARE_AFTER - case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.c)") - elif symbol == '/': - if self.positional_only: - fail("Function " + self.function.name + " uses '/' more than once.") - self.positional_only = True - # REQUIRED and OPTIONAL are allowed here, that allows positional-only without option groups - # to work (and have default values!) - allowed = { - ParamState.REQUIRED, - ParamState.OPTIONAL, - ParamState.RIGHT_SQUARE_AFTER, - ParamState.GROUP_BEFORE, - } - if (self.parameter_state not in allowed) or self.group: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {self.parameter_state}.d)") - if self.keyword_only: - fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") - # fixup preceding parameters - for p in self.function.parameters.values(): - if p.is_vararg(): - continue - if (p.kind != inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter)): - fail("Function " + self.function.name + " mixes keyword-only and positional-only parameters, which is unsupported.") - p.kind = inspect.Parameter.POSITIONAL_ONLY + def parse_star(self, function: Function) -> None: + """Parse keyword-only parameter marker '*'.""" + if self.keyword_only: + fail(f"Function {function.name} uses '*' more than once.") + self.keyword_only = True + + def parse_opening_square_bracket(self, function: Function) -> None: + """Parse opening parameter group symbol '['.""" + match self.parameter_state: + case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: + self.parameter_state = ParamState.LEFT_SQUARE_BEFORE + case ParamState.REQUIRED | ParamState.GROUP_AFTER: + self.parameter_state = ParamState.GROUP_AFTER + case st: + fail(f"Function {function.name} has an unsupported group configuration. " + f"(Unexpected state {st}.b)") + self.group += 1 + function.docstring_only = True + + def parse_closing_square_bracket(self, function: Function) -> None: + """Parse closing parameter group symbol ']'.""" + if not self.group: + fail(f"Function {function.name} has a ] without a matching [.") + if not any(p.group == self.group for p in function.parameters.values()): + fail(f"Function {function.name} has an empty group.\n" + "All groups must contain at least one parameter.") + self.group -= 1 + match self.parameter_state: + case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: + self.parameter_state = ParamState.GROUP_BEFORE + case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: + self.parameter_state = ParamState.RIGHT_SQUARE_AFTER + case st: + fail(f"Function {function.name} has an unsupported group configuration. " + f"(Unexpected state {st}.c)") + + def parse_slash(self, function: Function) -> None: + """Parse positional-only parameter marker '/'.""" + if self.positional_only: + fail(f"Function {function.name} uses '/' more than once.") + self.positional_only = True + # REQUIRED and OPTIONAL are allowed here, that allows positional-only + # without option groups to work (and have default values!) + allowed = { + ParamState.REQUIRED, + ParamState.OPTIONAL, + ParamState.RIGHT_SQUARE_AFTER, + ParamState.GROUP_BEFORE, + } + if (self.parameter_state not in allowed) or self.group: + fail(f"Function {function.name} has an unsupported group configuration. " + f"(Unexpected state {self.parameter_state}.d)") + if self.keyword_only: + fail(f"Function {function.name} mixes keyword-only and " + "positional-only parameters, which is unsupported.") + # fixup preceding parameters + for p in function.parameters.values(): + if p.is_vararg(): + continue + if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and + not isinstance(p.converter, self_converter) + ): + fail(f"Function {function.name} mixes keyword-only and " + "positional-only parameters, which is unsupported.") + p.kind = inspect.Parameter.POSITIONAL_ONLY def state_parameter_docstring_start(self, line: str | None) -> None: self.parameter_docstring_indent = len(self.indent.margin) From 1e36ca63f9f5e0399efe13a80499cef290314c2a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 17 Jul 2023 18:30:41 -0700 Subject: [PATCH 58/75] Small fixes to code generator (#106845) These repair nits I found in PR gh-106798 (issue gh-106797) and in PR gh-106716 (issue gh-106706). --- Include/internal/pycore_opcode_metadata.h | 10 +++++----- Python/generated_cases.c.h | 8 +++----- Tools/cases_generator/generate_cases.py | 10 +++++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c3a0dbb478a7c1..a5844b3135d398 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -842,15 +842,15 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case PUSH_EXC_INFO: return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return 1 + 1; + return 2; case LOAD_ATTR_METHOD_NO_DICT: - return 1 + 1; + return 2; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 0 + 1; + return 1; case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 0 + 1; + return 1; case LOAD_ATTR_METHOD_LAZY_DICT: - return 1 + 1; + return 2; case KW_NAMES: return 0; case INSTRUMENTED_CALL: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0148078d18bdc3..0a8e4da46b8be3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3356,7 +3356,7 @@ res = self; STACK_GROW(1); stack_pointer[-1] = res; - stack_pointer[-(1 + 1)] = res2; + stack_pointer[-2] = res2; next_instr += 9; DISPATCH(); } @@ -3378,7 +3378,7 @@ res = self; STACK_GROW(1); stack_pointer[-1] = res; - stack_pointer[-(1 + 1)] = res2; + stack_pointer[-2] = res2; next_instr += 9; DISPATCH(); } @@ -3403,7 +3403,6 @@ assert(descr != NULL); Py_DECREF(self); res = Py_NewRef(descr); - STACK_GROW(0); stack_pointer[-1] = res; next_instr += 9; DISPATCH(); @@ -3423,7 +3422,6 @@ assert(descr != NULL); Py_DECREF(self); res = Py_NewRef(descr); - STACK_GROW(0); stack_pointer[-1] = res; next_instr += 9; DISPATCH(); @@ -3450,7 +3448,7 @@ res = self; STACK_GROW(1); stack_pointer[-1] = res; - stack_pointer[-(1 + 1)] = res2; + stack_pointer[-2] = res2; next_instr += 9; DISPATCH(); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 037bee107cb13a..2713fc6774e845 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -99,7 +99,7 @@ def effect_size(effect: StackEffect) -> tuple[int, str]: return 0, effect.size elif effect.cond: if effect.cond in ("0", "1"): - return 0, effect.cond + return int(effect.cond), "" return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" else: return 1, "" @@ -841,9 +841,9 @@ def map_families(self) -> None: def check_families(self) -> None: """Check each family: - - Must have at least 2 members - - All members must be known instructions - - All members must have the same cache, input and output effects + - Must have at least 2 members (including head) + - Head and all members must be known instructions + - Head and all members must have the same cache, input and output effects """ for family in self.families.values(): if family.name not in self.macro_instrs and family.name not in self.instrs: @@ -868,7 +868,7 @@ def check_families(self) -> None: self.error( f"Family {family.name!r} has inconsistent " f"(cache, input, output) effects:\n" - f" {family.members[0]} = {expected_effects}; " + f" {family.name} = {expected_effects}; " f"{member} = {member_effects}", family, ) From ece3b9d12a2f47da8b144f185dfba9b2b725fc82 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 18 Jul 2023 12:44:16 +0900 Subject: [PATCH 59/75] gh-106843: fix memleak in _PyCompile_CleanDoc (#106846) --- Python/compile.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index b80f7c01bcd90e..2a735382c0cfda 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2267,6 +2267,7 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f } } if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { + Py_XDECREF(docstring); compiler_exit_scope(c); return ERROR; } @@ -8060,7 +8061,9 @@ _PyCompile_CleanDoc(PyObject *doc) } Py_DECREF(doc); - return PyUnicode_FromStringAndSize(buff, w - buff); + PyObject *res = PyUnicode_FromStringAndSize(buff, w - buff); + PyMem_Free(buff); + return res; } From e1c295e3da9ff5a3eb6b009a1f821d80e564ac87 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Jul 2023 08:56:58 +0300 Subject: [PATCH 60/75] gh-106719: Fix __annotations__ getter and setter in the type and module types (GH-106720) No longer suppress arbitrary errors. Simplify the code. --- ...-07-13-15-59-07.gh-issue-106719.jmVrsv.rst | 2 + Objects/moduleobject.c | 48 ++++++++----------- Objects/typeobject.c | 32 +++++-------- 3 files changed, 35 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst new file mode 100644 index 00000000000000..dc4bef193a3220 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst @@ -0,0 +1,2 @@ +No longer suppress arbitrary errors in the ``__annotations__`` getter and +setter in the type and module types. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 4071b5a3f1a62c..ba20534c3bdd8d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -937,26 +937,20 @@ static PyObject * module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); - - if ((dict == NULL) || !PyDict_Check(dict)) { + if (dict == NULL) { + return NULL; + } + if (!PyDict_Check(dict)) { PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - Py_XDECREF(dict); + Py_DECREF(dict); return NULL; } - PyObject *annotations; - /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ - if (PyDict_Contains(dict, &_Py_ID(__annotations__))) { - annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); - /* - ** _PyDict_GetItemIdWithError could still fail, - ** for instance with a well-timed Ctrl-C or a MemoryError. - ** so let's be totally safe. - */ - if (annotations) { - Py_INCREF(annotations); - } - } else { + PyObject *annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); + if (annotations) { + Py_INCREF(annotations); + } + else if (!PyErr_Occurred()) { annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( @@ -975,8 +969,10 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor { int ret = -1; PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); - - if ((dict == NULL) || !PyDict_Check(dict)) { + if (dict == NULL) { + return -1; + } + if (!PyDict_Check(dict)) { PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); goto exit; } @@ -984,19 +980,17 @@ module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignor if (value != NULL) { /* set */ ret = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); - goto exit; } - - /* delete */ - if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) { - PyErr_Format(PyExc_AttributeError, "__annotations__"); - goto exit; + else { + /* delete */ + ret = PyDict_DelItem(dict, &_Py_ID(__annotations__)); + if (ret < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + } } - ret = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - exit: - Py_XDECREF(dict); + Py_DECREF(dict); return ret; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b1f9f1280fd04d..7e5282cabd1bfb 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1451,24 +1451,17 @@ type_get_annotations(PyTypeObject *type, void *context) } PyObject *annotations; - /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ PyObject *dict = lookup_tp_dict(type); - if (PyDict_Contains(dict, &_Py_ID(__annotations__))) { - annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); - /* - ** PyDict_GetItemWithError could still fail, - ** for instance with a well-timed Ctrl-C or a MemoryError. - ** so let's be totally safe. - */ - if (annotations) { - if (Py_TYPE(annotations)->tp_descr_get) { - annotations = Py_TYPE(annotations)->tp_descr_get( - annotations, NULL, (PyObject *)type); - } else { - Py_INCREF(annotations); - } + annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); + if (annotations) { + if (Py_TYPE(annotations)->tp_descr_get) { + annotations = Py_TYPE(annotations)->tp_descr_get( + annotations, NULL, (PyObject *)type); + } else { + Py_INCREF(annotations); } - } else { + } + else if (!PyErr_Occurred()) { annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( @@ -1500,11 +1493,10 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) { - PyErr_Format(PyExc_AttributeError, "__annotations__"); - return -1; - } result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); + if (result < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + } } if (result == 0) { From 745492355b94d109e47827e5865846f25ae42d26 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Jul 2023 09:00:22 +0300 Subject: [PATCH 61/75] gh-86493: Fix possible leaks in modules initialization: _curses_panel, _decimal, posix, xxsubtype (GH-106767) --- Modules/_curses_panel.c | 3 +- Modules/_decimal/_decimal.c | 24 +++++++-------- Modules/posixmodule.c | 59 ++++++++++++++----------------------- Modules/xxsubtype.c | 6 ++-- 4 files changed, 37 insertions(+), 55 deletions(-) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index a3124ff80551e0..292b57c083d0c8 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -662,8 +662,7 @@ _curses_panel_exec(PyObject *mod) state->PyCursesError = PyErr_NewException( "_curses_panel.error", NULL, NULL); - if (PyModule_AddObject(mod, "error", Py_NewRef(state->PyCursesError)) < 0) { - Py_DECREF(state->PyCursesError); + if (PyModule_AddObjectRef(mod, "error", state->PyCursesError) < 0) { return -1; } diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e3dc304066b45b..f9dc6e875fa5fc 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5957,7 +5957,7 @@ PyInit__decimal(void) Py_DECREF(base); /* add to module */ - CHECK_INT(PyModule_AddObject(m, cm->name, Py_NewRef(cm->ex))); + CHECK_INT(PyModule_AddObjectRef(m, cm->name, cm->ex)); /* add to signal tuple */ PyTuple_SET_ITEM(state->SignalTuple, i, Py_NewRef(cm->ex)); @@ -5986,39 +5986,39 @@ PyInit__decimal(void) ASSIGN_PTR(cm->ex, PyErr_NewException(cm->fqname, base, NULL)); Py_DECREF(base); - CHECK_INT(PyModule_AddObject(m, cm->name, Py_NewRef(cm->ex))); + CHECK_INT(PyModule_AddObjectRef(m, cm->name, cm->ex)); } /* Init default context template first */ ASSIGN_PTR(state->default_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); - CHECK_INT(PyModule_AddObject(m, "DefaultContext", - Py_NewRef(state->default_context_template))); + CHECK_INT(PyModule_AddObjectRef(m, "DefaultContext", + state->default_context_template)); #ifndef WITH_DECIMAL_CONTEXTVAR ASSIGN_PTR(state->tls_context_key, PyUnicode_FromString("___DECIMAL_CTX__")); - CHECK_INT(PyModule_AddObject(m, "HAVE_CONTEXTVAR", Py_NewRef(Py_False))); + CHECK_INT(PyModule_AddObjectRef(m, "HAVE_CONTEXTVAR", Py_False)); #else ASSIGN_PTR(state->current_context_var, PyContextVar_New("decimal_context", NULL)); - CHECK_INT(PyModule_AddObject(m, "HAVE_CONTEXTVAR", Py_NewRef(Py_True))); + CHECK_INT(PyModule_AddObjectRef(m, "HAVE_CONTEXTVAR", Py_True)); #endif - CHECK_INT(PyModule_AddObject(m, "HAVE_THREADS", Py_NewRef(Py_True))); + CHECK_INT(PyModule_AddObjectRef(m, "HAVE_THREADS", Py_True)); /* Init basic context template */ ASSIGN_PTR(state->basic_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); init_basic_context(state->basic_context_template); - CHECK_INT(PyModule_AddObject(m, "BasicContext", - Py_NewRef(state->basic_context_template))); + CHECK_INT(PyModule_AddObjectRef(m, "BasicContext", + state->basic_context_template)); /* Init extended context template */ ASSIGN_PTR(state->extended_context_template, PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL)); init_extended_context(state->extended_context_template); - CHECK_INT(PyModule_AddObject(m, "ExtendedContext", - Py_NewRef(state->extended_context_template))); + CHECK_INT(PyModule_AddObjectRef(m, "ExtendedContext", + state->extended_context_template)); /* Init mpd_ssize_t constants */ @@ -6037,7 +6037,7 @@ PyInit__decimal(void) /* Init string constants */ for (i = 0; i < _PY_DEC_ROUND_GUARD; i++) { ASSIGN_PTR(state->round_map[i], PyUnicode_InternFromString(mpd_round_string[i])); - CHECK_INT(PyModule_AddObject(m, mpd_round_string[i], Py_NewRef(state->round_map[i]))); + CHECK_INT(PyModule_AddObjectRef(m, mpd_round_string[i], state->round_map[i])); } /* Add specification version number */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index aef802c232c6ce..fd5e491f4611ee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16793,57 +16793,49 @@ posixmodule_exec(PyObject *m) if (setup_confname_tables(m)) return -1; - PyModule_AddObject(m, "error", Py_NewRef(PyExc_OSError)); + if (PyModule_AddObjectRef(m, "error", PyExc_OSError) < 0) { + return -1; + } #if defined(HAVE_WAITID) && !defined(__APPLE__) waitid_result_desc.name = MODNAME ".waitid_result"; - PyObject *WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); - if (WaitidResultType == NULL) { + state->WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); + if (PyModule_AddObjectRef(m, "waitid_result", state->WaitidResultType) < 0) { return -1; } - PyModule_AddObject(m, "waitid_result", Py_NewRef(WaitidResultType)); - state->WaitidResultType = WaitidResultType; #endif stat_result_desc.name = "os.stat_result"; /* see issue #19209 */ stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; stat_result_desc.fields[9].name = PyStructSequence_UnnamedField; - PyObject *StatResultType = (PyObject *)PyStructSequence_NewType(&stat_result_desc); - if (StatResultType == NULL) { + state->StatResultType = (PyObject *)PyStructSequence_NewType(&stat_result_desc); + if (PyModule_AddObjectRef(m, "stat_result", state->StatResultType) < 0) { return -1; } - PyModule_AddObject(m, "stat_result", Py_NewRef(StatResultType)); - state->StatResultType = StatResultType; - state->statresult_new_orig = ((PyTypeObject *)StatResultType)->tp_new; - ((PyTypeObject *)StatResultType)->tp_new = statresult_new; + state->statresult_new_orig = ((PyTypeObject *)state->StatResultType)->tp_new; + ((PyTypeObject *)state->StatResultType)->tp_new = statresult_new; statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ - PyObject *StatVFSResultType = (PyObject *)PyStructSequence_NewType(&statvfs_result_desc); - if (StatVFSResultType == NULL) { + state->StatVFSResultType = (PyObject *)PyStructSequence_NewType(&statvfs_result_desc); + if (PyModule_AddObjectRef(m, "statvfs_result", state->StatVFSResultType) < 0) { return -1; } - PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); - state->StatVFSResultType = StatVFSResultType; #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) sched_param_desc.name = MODNAME ".sched_param"; - PyObject *SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); - if (SchedParamType == NULL) { + state->SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); + if (PyModule_AddObjectRef(m, "sched_param", state->SchedParamType) < 0) { return -1; } - PyModule_AddObject(m, "sched_param", Py_NewRef(SchedParamType)); - state->SchedParamType = SchedParamType; - ((PyTypeObject *)SchedParamType)->tp_new = os_sched_param; + ((PyTypeObject *)state->SchedParamType)->tp_new = os_sched_param; #endif /* initialize TerminalSize_info */ - PyObject *TerminalSizeType = (PyObject *)PyStructSequence_NewType(&TerminalSize_desc); - if (TerminalSizeType == NULL) { + state->TerminalSizeType = (PyObject *)PyStructSequence_NewType(&TerminalSize_desc); + if (PyModule_AddObjectRef(m, "terminal_size", state->TerminalSizeType) < 0) { return -1; } - PyModule_AddObject(m, "terminal_size", Py_NewRef(TerminalSizeType)); - state->TerminalSizeType = TerminalSizeType; /* initialize scandir types */ PyObject *ScandirIteratorType = PyType_FromModuleAndSpec(m, &ScandirIteratorType_spec, NULL); @@ -16852,28 +16844,21 @@ posixmodule_exec(PyObject *m) } state->ScandirIteratorType = ScandirIteratorType; - PyObject *DirEntryType = PyType_FromModuleAndSpec(m, &DirEntryType_spec, NULL); - if (DirEntryType == NULL) { + state->DirEntryType = PyType_FromModuleAndSpec(m, &DirEntryType_spec, NULL); + if (PyModule_AddObjectRef(m, "DirEntry", state->DirEntryType) < 0) { return -1; } - PyModule_AddObject(m, "DirEntry", Py_NewRef(DirEntryType)); - state->DirEntryType = DirEntryType; times_result_desc.name = MODNAME ".times_result"; - PyObject *TimesResultType = (PyObject *)PyStructSequence_NewType(×_result_desc); - if (TimesResultType == NULL) { + state->TimesResultType = (PyObject *)PyStructSequence_NewType(×_result_desc); + if (PyModule_AddObjectRef(m, "times_result", state->TimesResultType) < 0) { return -1; } - PyModule_AddObject(m, "times_result", Py_NewRef(TimesResultType)); - state->TimesResultType = TimesResultType; - PyTypeObject *UnameResultType = PyStructSequence_NewType(&uname_result_desc); - if (UnameResultType == NULL) { + state->UnameResultType = (PyObject *)PyStructSequence_NewType(&uname_result_desc); + if (PyModule_AddObjectRef(m, "uname_result", state->UnameResultType) < 0) { return -1; } - ; - PyModule_AddObject(m, "uname_result", Py_NewRef(UnameResultType)); - state->UnameResultType = (PyObject *)UnameResultType; if ((state->billion = PyLong_FromLong(1000000000)) == NULL) return -1; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 744ba7bf5d28b6..9e4a3d66ef41bd 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -274,12 +274,10 @@ xxsubtype_exec(PyObject* m) if (PyType_Ready(&spamdict_type) < 0) return -1; - if (PyModule_AddObject(m, "spamlist", - Py_NewRef(&spamlist_type)) < 0) + if (PyModule_AddObjectRef(m, "spamlist", (PyObject *)&spamlist_type) < 0) return -1; - if (PyModule_AddObject(m, "spamdict", - Py_NewRef(&spamdict_type)) < 0) + if (PyModule_AddObjectRef(m, "spamdict", (PyObject *)&spamdict_type) < 0) return -1; return 0; } From 83ac1284909433f3f77c0a4f459996b1ba3f1a4d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Jul 2023 09:42:05 +0300 Subject: [PATCH 62/75] bpo-42327: C API: Add PyModule_Add() function (GH-23443) It is a fixed implementation of PyModule_AddObject() which consistently steals reference both on success and on failure. --- Doc/c-api/module.rst | 65 +++++++++---------- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.13.rst | 5 ++ Include/modsupport.h | 12 +++- Lib/test/test_stable_abi_ctypes.py | 1 + .../2020-11-11-22-36-29.bpo-42327.ODSZBM.rst | 1 + Misc/stable_abi.toml | 2 + PC/python3dll.c | 1 + Python/modsupport.c | 29 +++------ 9 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-11-22-36-29.bpo-42327.ODSZBM.rst diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 230b471d473be7..d35b302fce6aa6 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -486,12 +486,29 @@ state: .. versionadded:: 3.10 +.. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value) + + Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference + to *value*. + It can be called with a result of function that returns a new reference + without bothering to check its result or even saving it to a variable. + + Example usage:: + + if (PyModule_Add(module, "spam", PyBytes_FromString(value)) < 0) { + goto error; + } + + .. versionadded:: 3.13 + + .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to *value* on success (if it returns ``0``). - The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is + The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` + functions are recommended, since it is easy to introduce reference leaks by misusing the :c:func:`PyModule_AddObject` function. @@ -501,44 +518,24 @@ state: only decrements the reference count of *value* **on success**. This means that its return value must be checked, and calling code must - :c:func:`Py_DECREF` *value* manually on error. + :c:func:`Py_XDECREF` *value* manually on error. Example usage:: - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (obj == NULL) { - return -1; - } - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_DECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } - - The example can also be written without checking explicitly if *obj* is - ``NULL``:: + PyObject *obj = PyBytes_FromString(value); + if (PyModule_AddObject(module, "spam", obj) < 0) { + // If 'obj' is not NULL and PyModule_AddObject() failed, + // 'obj' strong reference must be deleted with Py_XDECREF(). + // If 'obj' is NULL, Py_XDECREF() does nothing. + Py_XDECREF(obj); + goto error; + } + // PyModule_AddObject() stole a reference to obj: + // Py_XDECREF(obj) is not needed here. - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } + .. deprecated:: 3.13 - Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in - this case, since *obj* can be ``NULL``. + :c:func:`PyModule_AddObject` is :term:`soft deprecated`. .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index e3dd3dab27a035..aa1edf54637058 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -399,6 +399,7 @@ type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi function,PyModuleDef_Init,3.5,, var,PyModuleDef_Type,3.5,, +function,PyModule_Add,3.13,, function,PyModule_AddFunctions,3.7,, function,PyModule_AddIntConstant,3.2,, function,PyModule_AddObject,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 161d5fb1c59a30..0181e16f7d9b9d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -774,6 +774,11 @@ New Features If the assertion fails, make sure that the size is set before. (Contributed by Victor Stinner in :gh:`106168`.) +* Add :c:func:`PyModule_Add` function: similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but + always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/modsupport.h b/Include/modsupport.h index 7d4cfe853aaa7e..51061c5bc8090a 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -23,12 +23,18 @@ PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); PyAPI_FUNC(PyObject *) Py_VaBuildValue(const char *, va_list); // Add an attribute with name 'name' and value 'obj' to the module 'mod. -// On success, return 0 on success. +// On success, return 0. // On error, raise an exception and return -1. PyAPI_FUNC(int) PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value); -// Similar to PyModule_AddObjectRef() but steal a reference to 'obj' -// (Py_DECREF(obj)) on success (if it returns 0). +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +// Similar to PyModule_AddObjectRef() but steal a reference to 'value'. +PyAPI_FUNC(int) PyModule_Add(PyObject *mod, const char *name, PyObject *value); +#endif /* Py_LIMITED_API */ + +// Similar to PyModule_AddObjectRef() and PyModule_Add() but steal +// a reference to 'value' on success and only on success. +// Errorprone. Should not be used in new code. PyAPI_FUNC(int) PyModule_AddObject(PyObject *mod, const char *, PyObject *value); PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long); diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e92e986b293377..4e74bb374c93bf 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -425,6 +425,7 @@ def test_windows_feature_macros(self): "PyMethodDescr_Type", "PyModuleDef_Init", "PyModuleDef_Type", + "PyModule_Add", "PyModule_AddFunctions", "PyModule_AddIntConstant", "PyModule_AddObject", diff --git a/Misc/NEWS.d/next/C API/2020-11-11-22-36-29.bpo-42327.ODSZBM.rst b/Misc/NEWS.d/next/C API/2020-11-11-22-36-29.bpo-42327.ODSZBM.rst new file mode 100644 index 00000000000000..3d935aceb57a79 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-11-22-36-29.bpo-42327.ODSZBM.rst @@ -0,0 +1 @@ +Add :func:`PyModule_Add` function: similar to :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject`, but always steals a reference to the value. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8ea8fde68b833a..dd2c9910b83ccb 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2444,3 +2444,5 @@ added = '3.13' [function.PyMapping_GetOptionalItemString] added = '3.13' +[function.PyModule_Add] + added = '3.13' diff --git a/PC/python3dll.c b/PC/python3dll.c index 8f2df6950cfc05..0b54c5a707231c 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -374,6 +374,7 @@ EXPORT_FUNC(PyMemoryView_FromBuffer) EXPORT_FUNC(PyMemoryView_FromMemory) EXPORT_FUNC(PyMemoryView_FromObject) EXPORT_FUNC(PyMemoryView_GetContiguous) +EXPORT_FUNC(PyModule_Add) EXPORT_FUNC(PyModule_AddFunctions) EXPORT_FUNC(PyModule_AddIntConstant) EXPORT_FUNC(PyModule_AddObject) diff --git a/Python/modsupport.c b/Python/modsupport.c index 3db95f1f07284b..18b3322ae81d11 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -606,13 +606,16 @@ PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value) PyModule_GetName(mod)); return -1; } - - if (PyDict_SetItemString(dict, name, value)) { - return -1; - } - return 0; + return PyDict_SetItemString(dict, name, value); } +int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} int PyModule_AddObject(PyObject *mod, const char *name, PyObject *value) @@ -627,25 +630,13 @@ PyModule_AddObject(PyObject *mod, const char *name, PyObject *value) int PyModule_AddIntConstant(PyObject *m, const char *name, long value) { - PyObject *obj = PyLong_FromLong(value); - if (!obj) { - return -1; - } - int res = PyModule_AddObjectRef(m, name, obj); - Py_DECREF(obj); - return res; + return PyModule_Add(m, name, PyLong_FromLong(value)); } int PyModule_AddStringConstant(PyObject *m, const char *name, const char *value) { - PyObject *obj = PyUnicode_FromString(value); - if (!obj) { - return -1; - } - int res = PyModule_AddObjectRef(m, name, obj); - Py_DECREF(obj); - return res; + return PyModule_Add(m, name, PyUnicode_FromString(value)); } int From 3e65baee72131b49f4ce8ca2da568a6f2001ce93 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Jul 2023 10:50:47 +0300 Subject: [PATCH 63/75] gh-86493: Fix possible leaks in some modules initialization (GH-106768) Fix _ssl, _stat, _testinternalcapi, _threadmodule, cmath, math, posix, time. --- Modules/_ssl.c | 8 ++++---- Modules/_stat.c | 18 +++++++++--------- Modules/_testinternalcapi.c | 8 ++++---- Modules/_threadmodule.c | 4 ++-- Modules/cmathmodule.c | 15 +++++++-------- Modules/mathmodule.c | 10 +++++----- Modules/posixmodule.c | 12 ++++-------- Modules/timemodule.c | 7 ++----- 8 files changed, 37 insertions(+), 45 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8612b3dd53924c..7c8f4225d178aa 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6118,22 +6118,22 @@ sslmodule_init_versioninfo(PyObject *m) */ libver = OpenSSL_version_num(); r = PyLong_FromUnsignedLong(libver); - if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION_NUMBER", r)) + if (PyModule_Add(m, "OPENSSL_VERSION_NUMBER", r) < 0) return -1; parse_openssl_version(libver, &major, &minor, &fix, &patch, &status); r = Py_BuildValue("IIIII", major, minor, fix, patch, status); - if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION_INFO", r)) + if (PyModule_Add(m, "OPENSSL_VERSION_INFO", r) < 0) return -1; r = PyUnicode_FromString(OpenSSL_version(OPENSSL_VERSION)); - if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION", r)) + if (PyModule_Add(m, "OPENSSL_VERSION", r) < 0) return -1; libver = OPENSSL_VERSION_NUMBER; parse_openssl_version(libver, &major, &minor, &fix, &patch, &status); r = Py_BuildValue("IIIII", major, minor, fix, patch, status); - if (r == NULL || PyModule_AddObject(m, "_OPENSSL_API_VERSION", r)) + if (PyModule_Add(m, "_OPENSSL_API_VERSION", r) < 0) return -1; return 0; diff --git a/Modules/_stat.c b/Modules/_stat.c index 9747d848cbacb8..6cea26175dee5e 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -591,17 +591,17 @@ stat_exec(PyObject *module) ADD_INT_MACRO(module, FILE_ATTRIBUTE_TEMPORARY); ADD_INT_MACRO(module, FILE_ATTRIBUTE_VIRTUAL); - if (PyModule_AddObject(module, "IO_REPARSE_TAG_SYMLINK", - PyLong_FromUnsignedLong(IO_REPARSE_TAG_SYMLINK)) < 0) { - return -1; + if (PyModule_Add(module, "IO_REPARSE_TAG_SYMLINK", + PyLong_FromUnsignedLong(IO_REPARSE_TAG_SYMLINK)) < 0) { + return -1; } - if (PyModule_AddObject(module, "IO_REPARSE_TAG_MOUNT_POINT", - PyLong_FromUnsignedLong(IO_REPARSE_TAG_MOUNT_POINT)) < 0) { - return -1; + if (PyModule_Add(module, "IO_REPARSE_TAG_MOUNT_POINT", + PyLong_FromUnsignedLong(IO_REPARSE_TAG_MOUNT_POINT)) < 0) { + return -1; } - if (PyModule_AddObject(module, "IO_REPARSE_TAG_APPEXECLINK", - PyLong_FromUnsignedLong(IO_REPARSE_TAG_APPEXECLINK)) < 0) { - return -1; + if (PyModule_Add(module, "IO_REPARSE_TAG_APPEXECLINK", + PyLong_FromUnsignedLong(IO_REPARSE_TAG_APPEXECLINK)) < 0) { + return -1; } #endif diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 271ad6cfcaee32..74c932fa921cd0 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1511,13 +1511,13 @@ static PyMethodDef module_functions[] = { static int module_exec(PyObject *module) { - if (PyModule_AddObject(module, "SIZEOF_PYGC_HEAD", - PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { + if (PyModule_Add(module, "SIZEOF_PYGC_HEAD", + PyLong_FromSsize_t(sizeof(PyGC_Head))) < 0) { return 1; } - if (PyModule_AddObject(module, "SIZEOF_TIME_T", - PyLong_FromSsize_t(sizeof(time_t))) < 0) { + if (PyModule_Add(module, "SIZEOF_TIME_T", + PyLong_FromSsize_t(sizeof(time_t))) < 0) { return 1; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 82393309b67039..d8a797f34dbc4b 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1671,8 +1671,8 @@ thread_module_exec(PyObject *module) // Round towards minus infinity timeout_max = floor(timeout_max); - if (PyModule_AddObject(module, "TIMEOUT_MAX", - PyFloat_FromDouble(timeout_max)) < 0) { + if (PyModule_Add(module, "TIMEOUT_MAX", + PyFloat_FromDouble(timeout_max)) < 0) { return -1; } diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index db8b321e72e8ce..57bc55632be485 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1217,30 +1217,29 @@ static PyMethodDef cmath_methods[] = { static int cmath_exec(PyObject *mod) { - if (PyModule_AddObject(mod, "pi", PyFloat_FromDouble(Py_MATH_PI)) < 0) { + if (PyModule_Add(mod, "pi", PyFloat_FromDouble(Py_MATH_PI)) < 0) { return -1; } - if (PyModule_AddObject(mod, "e", PyFloat_FromDouble(Py_MATH_E)) < 0) { + if (PyModule_Add(mod, "e", PyFloat_FromDouble(Py_MATH_E)) < 0) { return -1; } // 2pi - if (PyModule_AddObject(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { + if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_AddObject(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { return -1; } Py_complex infj = {0.0, Py_INFINITY}; - if (PyModule_AddObject(mod, "infj", - PyComplex_FromCComplex(infj)) < 0) { + if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } - if (PyModule_AddObject(mod, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { + if (PyModule_Add(mod, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { return -1; } Py_complex nanj = {0.0, fabs(Py_NAN)}; - if (PyModule_AddObject(mod, "nanj", PyComplex_FromCComplex(nanj)) < 0) { + if (PyModule_Add(mod, "nanj", PyComplex_FromCComplex(nanj)) < 0) { return -1; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index f5679fe3a6f362..0d238c086ac78f 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -4037,20 +4037,20 @@ math_exec(PyObject *module) if (state->str___trunc__ == NULL) { return -1; } - if (PyModule_AddObject(module, "pi", PyFloat_FromDouble(Py_MATH_PI)) < 0) { + if (PyModule_Add(module, "pi", PyFloat_FromDouble(Py_MATH_PI)) < 0) { return -1; } - if (PyModule_AddObject(module, "e", PyFloat_FromDouble(Py_MATH_E)) < 0) { + if (PyModule_Add(module, "e", PyFloat_FromDouble(Py_MATH_E)) < 0) { return -1; } // 2pi - if (PyModule_AddObject(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { + if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_AddObject(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { return -1; } - if (PyModule_AddObject(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { + if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { return -1; } return 0; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fd5e491f4611ee..23bf978d0cdbf1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13466,7 +13466,7 @@ setup_confname_table(struct constdef *table, size_t tablesize, } Py_DECREF(o); } - return PyModule_AddObject(module, tablename, d); + return PyModule_Add(module, tablename, d); } /* Return -1 on failure, 0 on success. */ @@ -16781,11 +16781,9 @@ posixmodule_exec(PyObject *m) #endif /* Initialize environ dictionary */ - PyObject *v = convertenviron(); - Py_XINCREF(v); - if (v == NULL || PyModule_AddObject(m, "environ", v) != 0) + if (PyModule_Add(m, "environ", convertenviron()) != 0) { return -1; - Py_DECREF(v); + } if (all_ins(m)) return -1; @@ -16900,9 +16898,7 @@ posixmodule_exec(PyObject *m) Py_DECREF(unicode); } - PyModule_AddObject(m, "_have_functions", list); - - return 0; + return PyModule_Add(m, "_have_functions", list); } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3607855dbd8f27..912710219bd014 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1790,11 +1790,9 @@ init_timezone(PyObject *m) return -1; } #endif // MS_WINDOWS - PyObject *tzname_obj = Py_BuildValue("(NN)", otz0, otz1); - if (tzname_obj == NULL) { + if (PyModule_Add(m, "tzname", Py_BuildValue("(NN)", otz0, otz1)) < 0) { return -1; } - PyModule_AddObject(m, "tzname", tzname_obj); #else // !HAVE_DECL_TZNAME static const time_t YEAR = (365 * 24 + 6) * 3600; time_t t; @@ -1837,10 +1835,9 @@ init_timezone(PyObject *m) PyModule_AddIntConstant(m, "daylight", janzone != julyzone); tzname_obj = Py_BuildValue("(zz)", janname, julyname); } - if (tzname_obj == NULL) { + if (PyModule_Add(m, "tzname", tzname_obj) < 0) { return -1; } - PyModule_AddObject(m, "tzname", tzname_obj); #endif // !HAVE_DECL_TZNAME if (PyErr_Occurred()) { From 4cb0b9c0a9f6a4154238c98013d2679229b1f794 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jul 2023 10:50:17 +0200 Subject: [PATCH 64/75] Docs: Normalise Argument Clinic advanced topics headings (#106842) Co-authored-by: Ezio Melotti --- Doc/howto/clinic.rst | 95 +++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 0f99cb64994ab2..12d7a77d43209a 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -560,15 +560,12 @@ Let's dive in! Congratulations, you've ported your first function to work with Argument Clinic! -Advanced topics -=============== +How-to guides +============= -Now that you've had some experience working with Argument Clinic, it's time -for some advanced topics. - -Symbolic default values ------------------------ +How to use symbolic default values +---------------------------------- The default value you provide for a parameter can't be any arbitrary expression. Currently the following are explicitly supported: @@ -583,8 +580,8 @@ expression. Currently the following are explicitly supported: to allow full expressions like ``CONSTANT - 1``.) -Renaming the C functions and variables generated by Argument Clinic -------------------------------------------------------------------- +How to to rename C functions and variables generated by Argument Clinic +----------------------------------------------------------------------- Argument Clinic automatically names the functions it generates for you. Occasionally this may cause a problem, if the generated name collides with @@ -626,8 +623,8 @@ array) would be ``file``, but the C variable would be named ``file_obj``. You can use this to rename the ``self`` parameter too! -Converting functions using PyArg_UnpackTuple --------------------------------------------- +How to convert functions using ``PyArg_UnpackTuple`` +---------------------------------------------------- To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, simply write out all the arguments, specifying each as an ``object``. You @@ -639,8 +636,8 @@ Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this will change soon. -Optional groups ---------------- +How to use optional groups +-------------------------- Some legacy functions have a tricky approach to parsing their arguments: they count the number of positional arguments, then use a ``switch`` statement @@ -732,8 +729,8 @@ Notes: use optional groups for new code. -Using real Argument Clinic converters, instead of "legacy converters" ---------------------------------------------------------------------- +How to use real Argument Clinic converters, instead of "legacy converters" +-------------------------------------------------------------------------- To save time, and to minimize how much you need to learn to achieve your first port to Argument Clinic, the walkthrough above tells @@ -903,8 +900,8 @@ it accepts, along with the default value for each parameter. Just run ``Tools/clinic/clinic.py --converters`` to see the full list. -Py_buffer ---------- +How to use the ``Py_buffer`` converter +-------------------------------------- When using the ``Py_buffer`` converter (or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters), @@ -912,8 +909,8 @@ you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. Argument Clinic generates code that does it for you (in the parsing function). -Advanced converters -------------------- +How to use advanced converters +------------------------------ Remember those format units you skipped for your first time because they were advanced? Here's how to handle those too. @@ -944,8 +941,8 @@ hard-coded encoding strings for parameters whose format units start with ``e``. .. _default_values: -Parameter default values ------------------------- +How to assign default values to parameter +----------------------------------------- Default values for parameters can be any of a number of values. At their simplest, they can be string, int, or float literals: @@ -968,8 +965,8 @@ There's also special support for a default value of ``NULL``, and for simple expressions, documented in the following sections. -The ``NULL`` default value --------------------------- +How to use the ``NULL`` default value +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For string and object parameters, you can set them to ``None`` to indicate that there's no default. However, that means the C variable will be @@ -979,8 +976,8 @@ behaves like a default value of ``None``, but the C variable is initialized with ``NULL``. -Expressions specified as default values ---------------------------------------- +How to use expressions as default values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The default value for a parameter can be more than just a literal value. It can be an entire expression, using math operators and looking up attributes @@ -1036,8 +1033,8 @@ you're not permitted to use: * Tuple/list/set/dict literals. -Using a return converter ------------------------- +How to use return converters +---------------------------- By default, the impl function Argument Clinic generates for you returns :c:type:`PyObject * `. @@ -1106,8 +1103,8 @@ their parameters (if any), just run ``Tools/clinic/clinic.py --converters`` for the full list. -Cloning existing functions --------------------------- +How to clone existing functions +------------------------------- If you have a number of functions that look similar, you may be able to use Clinic's "clone" feature. When you clone an existing function, @@ -1150,8 +1147,8 @@ Also, the function you are cloning from must have been previously defined in the current file. -Calling Python code -------------------- +How to call Python code +----------------------- The rest of the advanced topics require you to write Python code which lives inside your C file and modifies Argument Clinic's @@ -1178,8 +1175,8 @@ variable to the C code:: /*[python checksum:...]*/ -Using a "self converter" ------------------------- +How to use the "self converter" +------------------------------- Argument Clinic automatically adds a "self" parameter for you using a default converter. It automatically sets the ``type`` @@ -1230,8 +1227,8 @@ type for ``self``, it's best to create your own converter, subclassing [clinic start generated code]*/ -Using a "defining class" converter ----------------------------------- +How to use the "defining class" converter +----------------------------------------- Argument Clinic facilitates gaining access to the defining class of a method. This is useful for :ref:`heap type ` methods that need to fetch @@ -1293,8 +1290,8 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. -Writing a custom converter --------------------------- +How to write a custom converter +------------------------------- As we hinted at in the previous section... you can write your own converters! A converter is simply a Python class that inherits from ``CConverter``. @@ -1385,8 +1382,8 @@ You can see more examples of custom converters in the CPython source tree; grep the C files for the string ``CConverter``. -Writing a custom return converter ---------------------------------- +How to write a custom return converter +-------------------------------------- Writing a custom return converter is much like writing a custom converter. Except it's somewhat simpler, because return @@ -1400,8 +1397,8 @@ specifically the implementation of ``CReturnConverter`` and all its subclasses. -METH_O and METH_NOARGS ----------------------- +How to convert ``METH_O`` and ``METH_NOARGS`` functions +------------------------------------------------------- To convert a function using ``METH_O``, make sure the function's single argument is using the ``object`` converter, and mark the @@ -1422,8 +1419,8 @@ You can still use a self converter, a return converter, and specify a ``type`` argument to the object converter for ``METH_O``. -tp_new and tp_init functions ----------------------------- +How to convert ``tp_new`` and ``tp_init`` functions +--------------------------------------------------- You can convert ``tp_new`` and ``tp_init`` functions. Just name them ``__new__`` or ``__init__`` as appropriate. Notes: @@ -1445,8 +1442,8 @@ them ``__new__`` or ``__init__`` as appropriate. Notes: generated will throw an exception if it receives any.) -Changing and redirecting Clinic's output ----------------------------------------- +How to change and redirect Clinic's output +------------------------------------------ It can be inconvenient to have Clinic's output interspersed with your conventional hand-edited C code. Luckily, Clinic is configurable: @@ -1728,8 +1725,8 @@ it in a Clinic block lets Clinic use its existing checksum functionality to ensu the file was not modified by hand before it gets overwritten. -The #ifdef trick ----------------- +How to use the ``#ifdef`` trick +------------------------------- If you're converting a function that isn't available on all platforms, there's a trick you can use to make life a little easier. The existing @@ -1809,8 +1806,8 @@ Argument Clinic added to your file (it'll be at the very bottom), then move it above the ``PyMethodDef`` structure where that macro is used. -Using Argument Clinic in Python files -------------------------------------- +How to use Argument Clinic in Python files +------------------------------------------ It's actually possible to use Argument Clinic to preprocess Python files. There's no point to using Argument Clinic blocks, of course, as the output From aecf6aca515a203a823a87c711f15cbb82097c8b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 18 Jul 2023 00:16:32 -1000 Subject: [PATCH 65/75] gh-106751: selectors: optimize EpollSelector.select() (#106754) Co-authored-by: Pieter Eendebak --- Lib/selectors.py | 17 +++++++++-------- ...23-07-14-20-31-09.gh-issue-106751.52F6yQ.rst | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-14-20-31-09.gh-issue-106751.52F6yQ.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index 6d82935445d4b1..a42d1563406417 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -430,6 +430,9 @@ class PollSelector(_PollLikeSelector): if hasattr(select, 'epoll'): + _NOT_EPOLLIN = ~select.EPOLLIN + _NOT_EPOLLOUT = ~select.EPOLLOUT + class EpollSelector(_PollLikeSelector): """Epoll-based selector.""" _selector_cls = select.epoll @@ -452,22 +455,20 @@ def select(self, timeout=None): # epoll_wait() expects `maxevents` to be greater than zero; # we want to make sure that `select()` can be called when no # FD is registered. - max_ev = max(len(self._fd_to_key), 1) + max_ev = len(self._fd_to_key) or 1 ready = [] try: fd_event_list = self._selector.poll(timeout, max_ev) except InterruptedError: return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.EPOLLIN: - events |= EVENT_WRITE - if event & ~select.EPOLLOUT: - events |= EVENT_READ - key = self._fd_to_key.get(fd) + fd_to_key = self._fd_to_key + for fd, event in fd_event_list: + key = fd_to_key.get(fd) if key: + events = ((event & _NOT_EPOLLIN and EVENT_WRITE) + | (event & _NOT_EPOLLOUT and EVENT_READ)) ready.append((key, events & key.events)) return ready diff --git a/Misc/NEWS.d/next/Library/2023-07-14-20-31-09.gh-issue-106751.52F6yQ.rst b/Misc/NEWS.d/next/Library/2023-07-14-20-31-09.gh-issue-106751.52F6yQ.rst new file mode 100644 index 00000000000000..486b1f9bbd0a97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-14-20-31-09.gh-issue-106751.52F6yQ.rst @@ -0,0 +1 @@ +:mod:`selectors`: Optimize ``EpollSelector.select()`` code by moving some code outside of the loop. From 3535ef1eec2563bbd7bff7c830465441fbbf759e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 18 Jul 2023 17:13:51 +0200 Subject: [PATCH 66/75] gh-106535: Document soft deprecations in What's New In Python 3.13 (#106859) --- Doc/whatsnew/3.13.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0181e16f7d9b9d..479d08b24b112a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -184,6 +184,13 @@ Deprecated Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) +* The :mod:`getopt` and :mod:`optparse` modules are now + :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. + Previously, the :mod:`optparse` module was already deprecated, its removal + was not scheduled, and no warnings was emitted: so there is no change in + practice. + (Contributed by Victor Stinner in :gh:`106535`.) + Pending Removal in Python 3.14 ------------------------------ @@ -946,6 +953,11 @@ Removed :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. (Contributed by Victor Stinner in :gh:`106320`.) +* The :c:func:`PyModule_AddObject` function is now :term:`soft deprecated`: + :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions should + be used instead. + (Contributed by Serhiy Storchaka in :gh:`86493`.) + Pending Removal in Python 3.14 ------------------------------ From 40f3f11a773b854c6d94746aa3b1881c8ac71b0f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:42:44 +0100 Subject: [PATCH 67/75] gh-105481: Generate the opcode lists in dis from data extracted from bytecodes.c (#106758) --- .gitattributes | 1 - Doc/library/dis.rst | 28 ++- Include/cpython/compile.h | 4 + Include/internal/pycore_opcode_metadata.h | 36 ++-- Lib/opcode.py | 129 +++++------- Lib/test/test__opcode.py | 63 ++---- ...-07-17-16-46-00.gh-issue-105481.fek_Nn.rst | 1 + Modules/_opcode.c | 60 ++++++ Modules/clinic/_opcode.c.h | 196 +++++++++++++++++- Python/bytecodes.c | 2 +- Python/ceval_macros.h | 1 + Python/compile.c | 18 ++ Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Tools/build/generate_opcode_h.py | 6 - Tools/cases_generator/generate_cases.py | 13 +- 16 files changed, 403 insertions(+), 159 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst diff --git a/.gitattributes b/.gitattributes index 2616da74b48c0f..5d5558da711b17 100644 --- a/.gitattributes +++ b/.gitattributes @@ -87,7 +87,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Include/internal/pycore_opcode_metadata.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 099b6410f165ed..6beaad3825aba8 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1803,15 +1803,12 @@ instructions: Sequence of bytecodes that access an attribute by name. -.. data:: hasjrel - - Sequence of bytecodes that have a relative jump target. +.. data:: hasjump + Sequence of bytecodes that have a jump target. All jumps + are relative. -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. - + .. versionadded:: 3.13 .. data:: haslocal @@ -1827,3 +1824,20 @@ instructions: Sequence of bytecodes that set an exception handler. .. versionadded:: 3.12 + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + .. deprecated:: 3.13 + All jumps are now relative. Use :data:`hasjump`. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + .. deprecated:: 3.13 + All jumps are now relative. This list is empty. + diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index cd7fd7bd377663..fd52697840203a 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -73,3 +73,7 @@ PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); + diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index a5844b3135d398..d525913f8a7aba 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -955,10 +955,14 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) #define HAS_JUMP_FLAG (8) +#define HAS_FREE_FLAG (16) +#define HAS_LOCAL_FLAG (32) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) +#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) struct opcode_metadata { bool valid_entry; @@ -995,16 +999,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [END_FOR] = { true, INSTR_FMT_IB, 0 }, @@ -1028,7 +1032,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, 0 }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, @@ -1080,12 +1084,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Lib/opcode.py b/Lib/opcode.py index 1b36300785aaea..08dfd2674dca78 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,10 +4,12 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] +# Note that __all__ is further extended below +__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", + "HAVE_ARGUMENT", "EXTENDED_ARG"] + +import _opcode from _opcode import stack_effect import sys @@ -17,55 +19,24 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - ENABLE_SPECIALIZATION = True def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - opmap = {} -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. +# pseudo opcodes (used in the compiler) mapped to the values +# they can become in the actual code. _pseudo_ops = {} def def_op(name, op): opmap[name] = op -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - def pseudo_op(name, op, real_ops): def_op(name, op) _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) # Instruction opcodes for compiled code @@ -137,74 +108,61 @@ def pseudo_op(name, op, real_ops): HAVE_ARGUMENT = 90 # real opcodes from here have an argument: -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" +def_op('STORE_NAME', 90) # Index in name list +def_op('DELETE_NAME', 91) # "" def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) +def_op('FOR_ITER', 93) def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" +def_op('STORE_ATTR', 95) # Index in name list +def_op('DELETE_ATTR', 96) # "" +def_op('STORE_GLOBAL', 97) # "" +def_op('DELETE_GLOBAL', 98) # "" def_op('SWAP', 99) def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list +def_op('LOAD_NAME', 101) # Index in name list def_op('BUILD_TUPLE', 102) # Number of tuple items def_op('BUILD_LIST', 103) # Number of list items def_op('BUILD_SET', 104) # Number of set items def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list +def_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip - -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list +def_op('IMPORT_NAME', 108) # Index in name list +def_op('IMPORT_FROM', 109) # Index in name list +def_op('JUMP_FORWARD', 110) # Number of words to skip + +def_op('POP_JUMP_IF_FALSE', 114) +def_op('POP_JUMP_IF_TRUE', 115) +def_op('LOAD_GLOBAL', 116) # Index in name list def_op('IS_OP', 117) def_op('CONTAINS_OP', 118) def_op('RERAISE', 119) def_op('COPY', 120) def_op('RETURN_CONST', 121) -hasconst.append(121) def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of words to skip +def_op('SEND', 123) # Number of words to skip def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) +def_op('POP_JUMP_IF_NOT_NONE', 128) +def_op('POP_JUMP_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('GET_AWAITABLE', 131) def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) +def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) def_op('MAKE_CELL', 135) -hasfree.append(135) def_op('LOAD_DEREF', 137) -hasfree.append(137) def_op('STORE_DEREF', 138) -hasfree.append(138) def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -name_op('LOAD_SUPER_ATTR', 141) +def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) +def_op('LOAD_SUPER_ATTR', 141) def_op('CALL_FUNCTION_EX', 142) # Flags def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -haslocal.append(143) def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 +EXTENDED_ARG = opmap['EXTENDED_ARG'] def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) -hasfree.append(148) def_op('COPY_FREE_VARS', 149) def_op('YIELD_VALUE', 150) def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py @@ -224,12 +182,10 @@ def pseudo_op(name, op, real_ops): def_op('STORE_FAST_STORE_FAST', 170) def_op('CALL', 171) def_op('KW_NAMES', 172) -hasconst.append(172) def_op('CALL_INTRINSIC_1', 173) def_op('CALL_INTRINSIC_2', 174) -name_op('LOAD_FROM_DICT_OR_GLOBALS', 175) +def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) def_op('LOAD_FROM_DICT_OR_DEREF', 176) -hasfree.append(176) def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute # Optimizer hook @@ -258,16 +214,12 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_LINE', 254) # 255 is reserved -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) pseudo_op('POP_BLOCK', 259, ['NOP']) pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) @@ -283,12 +235,29 @@ def pseudo_op(name, op, real_ops): MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 -del def_op, name_op, jrel_op, jabs_op, pseudo_op +del def_op, pseudo_op opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] for op, i in opmap.items(): opname[i] = op +# The build uses older versions of Python which do not have _opcode.has_* functions +if sys.version_info[:2] >= (3, 13): + # These lists are documented as part of the dis module's API + hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] + hasconst = [op for op in opmap.values() if _opcode.has_const(op)] + hasname = [op for op in opmap.values() if _opcode.has_name(op)] + hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjrel = hasjump # for backward compatibility + hasjabs = [] + hasfree = [op for op in opmap.values() if _opcode.has_free(op)] + haslocal = [op for op in opmap.values() if _opcode.has_local(op)] + hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] + + __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "hasfree", "haslocal", "hasexc"]) + +hascompare = [opmap["COMPARE_OP"]] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 7d9553d9e383a3..b3a9bcbe160453 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -7,16 +7,7 @@ from _opcode import stack_effect -class OpcodeTests(unittest.TestCase): - - def check_bool_function_result(self, func, ops, expected): - for op in ops: - if isinstance(op, str): - op = dis.opmap[op] - with self.subTest(opcode=op, func=func): - self.assertIsInstance(func(op), bool) - self.assertEqual(func(op), expected) - +class OpListTests(unittest.TestCase): def test_invalid_opcodes(self): invalid = [-100, -1, 255, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) @@ -24,6 +15,9 @@ def test_invalid_opcodes(self): self.check_bool_function_result(_opcode.has_const, invalid, False) self.check_bool_function_result(_opcode.has_name, invalid, False) self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) def test_is_valid(self): names = [ @@ -36,43 +30,24 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) - def test_has_arg(self): - has_arg = ['SWAP', 'LOAD_FAST', 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'JUMP'] - no_arg = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_arg, has_arg, True) - self.check_bool_function_result(_opcode.has_arg, no_arg, False) - - def test_has_const(self): - has_const = ['LOAD_CONST', 'RETURN_CONST', 'KW_NAMES'] - no_const = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_const, has_const, True) - self.check_bool_function_result(_opcode.has_const, no_const, False) - - def test_has_name(self): - has_name = ['STORE_NAME', 'DELETE_ATTR', 'STORE_GLOBAL', 'IMPORT_FROM', - 'LOAD_FROM_DICT_OR_GLOBALS'] - no_name = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_name, has_name, True) - self.check_bool_function_result(_opcode.has_name, no_name, False) + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) - def test_has_jump(self): - has_jump = ['FOR_ITER', 'JUMP_FORWARD', 'JUMP', 'POP_JUMP_IF_TRUE', 'SEND'] - no_jump = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_jump, has_jump, True) - self.check_bool_function_result(_opcode.has_jump, no_jump, False) + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) - # the following test is part of the refactor, it will be removed soon - def test_against_legacy_bool_values(self): - # limiting to ops up to ENTER_EXECUTOR, because everything after that - # is not currently categorized correctly in opcode.py. - for op in range(0, opcode.opmap['ENTER_EXECUTOR']): - with self.subTest(op=op): - if opcode.opname[op] != f'<{op}>': - self.assertEqual(op in dis.hasarg, _opcode.has_arg(op)) - self.assertEqual(op in dis.hasconst, _opcode.has_const(op)) - self.assertEqual(op in dis.hasname, _opcode.has_name(op)) - self.assertEqual(op in dis.hasjrel, _opcode.has_jump(op)) +class OpListTests(unittest.TestCase): def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) diff --git a/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst new file mode 100644 index 00000000000000..d82eb987c83e96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst @@ -0,0 +1 @@ +The various opcode lists in the :mod:`dis` module are now generated from bytecodes.c instead of explicitly constructed in opcode.py. diff --git a/Modules/_opcode.c b/Modules/_opcode.c index b3b9873d21a5be..daabdce1655777 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -147,6 +147,63 @@ _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic input] +_opcode.has_free -> bool + + opcode: int + +Return True if the opcode accesses a free variable, False otherwise. + +Note that 'free' in this context refers to names in the current scope +that are referenced by inner scopes or names in outer scopes that are +referenced from this scope. It does not include references to global +or builtin scopes. +[clinic start generated code]*/ + +static int +_opcode_has_free_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasFree(opcode); + +} + +/*[clinic input] + +_opcode.has_local -> bool + + opcode: int + +Return True if the opcode accesses a local variable, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_local_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasLocal(opcode); +} + +/*[clinic input] + +_opcode.has_exc -> bool + + opcode: int + +Return True if the opcode sets an exception handler, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_exc_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasExc(opcode); +} + +/*[clinic input] + _opcode.get_specialization_stats Return the specialization stats @@ -171,6 +228,9 @@ opcode_functions[] = { _OPCODE_HAS_CONST_METHODDEF _OPCODE_HAS_NAME_METHODDEF _OPCODE_HAS_JUMP_METHODDEF + _OPCODE_HAS_FREE_METHODDEF + _OPCODE_HAS_LOCAL_METHODDEF + _OPCODE_HAS_EXC_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 3eb050e470c343..e6381fa73a5506 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -401,6 +401,200 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_opcode_has_free__doc__, +"has_free($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a free variable, False otherwise.\n" +"\n" +"Note that \'free\' in this context refers to names in the current scope\n" +"that are referenced by inner scopes or names in outer scopes that are\n" +"referenced from this scope. It does not include references to global\n" +"or builtin scopes."); + +#define _OPCODE_HAS_FREE_METHODDEF \ + {"has_free", _PyCFunction_CAST(_opcode_has_free), METH_FASTCALL|METH_KEYWORDS, _opcode_has_free__doc__}, + +static int +_opcode_has_free_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_free", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_free_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_local__doc__, +"has_local($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a local variable, False otherwise."); + +#define _OPCODE_HAS_LOCAL_METHODDEF \ + {"has_local", _PyCFunction_CAST(_opcode_has_local), METH_FASTCALL|METH_KEYWORDS, _opcode_has_local__doc__}, + +static int +_opcode_has_local_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_local", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_local_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(_opcode_has_exc__doc__, +"has_exc($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode sets an exception handler, False otherwise."); + +#define _OPCODE_HAS_EXC_METHODDEF \ + {"has_exc", _PyCFunction_CAST(_opcode_has_exc), METH_FASTCALL|METH_KEYWORDS, _opcode_has_exc__doc__}, + +static int +_opcode_has_exc_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_exc", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_exc_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_get_specialization_stats__doc__, "get_specialization_stats($module, /)\n" "--\n" @@ -418,4 +612,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=ae2b2ef56d582180 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e507bf14fb2796f8 input=a9049054013a1b77]*/ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 19fb138ee64cba..ea136a3fca2e02 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3604,7 +3604,7 @@ dummy_func( _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 874bd45becf0c9..c2c323317d10f9 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -234,6 +234,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Local variable macros */ +#define LOCALS_ARRAY (frame->localsplus) #define GETLOCAL(i) (frame->localsplus[i]) /* The SETLOCAL() macro must not DECREF the local variable in-place and diff --git a/Python/compile.c b/Python/compile.c index 2a735382c0cfda..d5405b46561820 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -896,6 +896,24 @@ PyUnstable_OpcodeHasJump(int opcode) return OPCODE_HAS_JUMP(opcode); } +int +PyUnstable_OpcodeHasFree(int opcode) +{ + return OPCODE_HAS_FREE(opcode); +} + +int +PyUnstable_OpcodeHasLocal(int opcode) +{ + return OPCODE_HAS_LOCAL(opcode); +} + +int +PyUnstable_OpcodeHasExc(int opcode) +{ + return IS_BLOCK_PUSH_OPCODE(opcode); +} + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f492c1fa9d8e3f..e1f8b9f208c76d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2449,7 +2449,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0a8e4da46b8be3..b2b0aa6ece4816 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4451,7 +4451,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 2e841e6097aa25..5b0560e6b21a99 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -84,13 +84,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - hasarg = opcode['hasarg'] - hasconst = opcode['hasconst'] - hasjrel = opcode['hasjrel'] - hasjabs = opcode['hasjabs'] is_pseudo = opcode['is_pseudo'] - _pseudo_ops = opcode['_pseudo_ops'] - ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2713fc6774e845..33eff548a18809 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -261,6 +261,8 @@ class InstructionFlags: HAS_CONST_FLAG: bool HAS_NAME_FLAG: bool HAS_JUMP_FLAG: bool + HAS_FREE_FLAG: bool + HAS_LOCAL_FLAG: bool def __post_init__(self): self.bitmask = { @@ -269,16 +271,25 @@ def __post_init__(self): @staticmethod def fromInstruction(instr: "AnyInstruction"): + + has_free = (variable_used(instr, "PyCell_New") or + variable_used(instr, "PyCell_GET") or + variable_used(instr, "PyCell_SET")) + return InstructionFlags( HAS_ARG_FLAG=variable_used(instr, "oparg"), HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), + HAS_FREE_FLAG=has_free, + HAS_LOCAL_FLAG=(variable_used(instr, "GETLOCAL") or + variable_used(instr, "SETLOCAL")) and + not has_free, ) @staticmethod def newEmpty(): - return InstructionFlags(False, False, False, False) + return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items(): From a293fa5915c21b21f5cb8ed9649fbdb37b4c1421 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 18 Jul 2023 23:59:53 +0300 Subject: [PATCH 68/75] gh-86493: Use PyModule_Add() instead of PyModule_AddObjectRef() (GH-106860) --- Modules/_ctypes/_ctypes.c | 8 +------- Modules/_hashopenssl.c | 14 ++------------ Modules/_json.c | 14 ++------------ Modules/_sre/sre.c | 7 +------ Modules/_ssl.c | 8 +------- Modules/_testcapi/mem.c | 8 +++----- Modules/_testcapi/watchers.c | 8 +------- Modules/_testsinglephase.c | 8 +------- Modules/pyexpat.c | 13 +++++-------- Modules/socketmodule.c | 12 ++---------- Modules/unicodedata.c | 8 +------- PC/msvcrtmodule.c | 17 +++-------------- PC/winreg.c | 10 ++-------- Python/import.c | 7 +------ 14 files changed, 26 insertions(+), 116 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 7624c15ac522da..200fd36748c403 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5740,15 +5740,9 @@ _ctypes_add_objects(PyObject *mod) { #define MOD_ADD(name, expr) \ do { \ - PyObject *obj = (expr); \ - if (obj == NULL) { \ + if (PyModule_Add(mod, name, (expr)) < 0) { \ return -1; \ } \ - if (PyModule_AddObjectRef(mod, name, obj) < 0) { \ - Py_DECREF(obj); \ - return -1; \ - } \ - Py_DECREF(obj); \ } while (0) MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 011cb765ed82e6..246eea74098820 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2189,7 +2189,6 @@ hashlib_init_constructors(PyObject *module) */ PyModuleDef *mdef; PyMethodDef *fdef; - PyObject *proxy; PyObject *func, *name_obj; _hashlibstate *state = get_hashlib_state(module); @@ -2224,17 +2223,8 @@ hashlib_init_constructors(PyObject *module) } } - proxy = PyDictProxy_New(state->constructs); - if (proxy == NULL) { - return -1; - } - - int rc = PyModule_AddObjectRef(module, "_constructors", proxy); - Py_DECREF(proxy); - if (rc < 0) { - return -1; - } - return 0; + return PyModule_Add(module, "_constructors", + PyDictProxy_New(state->constructs)); } static int diff --git a/Modules/_json.c b/Modules/_json.c index 360fb453cd111c..2d0e30d70932bd 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1756,22 +1756,12 @@ static int _json_exec(PyObject *module) { PyObject *PyScannerType = PyType_FromSpec(&PyScannerType_spec); - if (PyScannerType == NULL) { - return -1; - } - int rc = PyModule_AddObjectRef(module, "make_scanner", PyScannerType); - Py_DECREF(PyScannerType); - if (rc < 0) { + if (PyModule_Add(module, "make_scanner", PyScannerType) < 0) { return -1; } PyObject *PyEncoderType = PyType_FromSpec(&PyEncoderType_spec); - if (PyEncoderType == NULL) { - return -1; - } - rc = PyModule_AddObjectRef(module, "make_encoder", PyEncoderType); - Py_DECREF(PyEncoderType); - if (rc < 0) { + if (PyModule_Add(module, "make_encoder", PyEncoderType) < 0) { return -1; } diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index f34a353432dec0..ddbdc9f478aab3 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3195,12 +3195,7 @@ do { \ #define ADD_ULONG_CONSTANT(module, name, value) \ do { \ - PyObject *o = PyLong_FromUnsignedLong(value); \ - if (!o) \ - goto error; \ - int res = PyModule_AddObjectRef(module, name, o); \ - Py_DECREF(o); \ - if (res < 0) { \ + if (PyModule_Add(module, name, PyLong_FromUnsignedLong(value)) < 0) { \ goto error; \ } \ } while (0) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 7c8f4225d178aa..ed720b4295f8ec 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5773,13 +5773,7 @@ static int sslmodule_add_option(PyObject *m, const char *name, uint64_t value) { Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(value)); - PyObject *obj = PyLong_FromUnsignedLongLong(value); - if (obj == NULL) { - return -1; - } - int res = PyModule_AddObjectRef(m, name, obj); - Py_DECREF(obj); - return res; + return PyModule_Add(m, name, PyLong_FromUnsignedLongLong(value)); } diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 16bda66af554af..979b3a4b2b1af6 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -692,13 +692,11 @@ _PyTestCapi_Init_Mem(PyObject *mod) PyObject *v; #ifdef WITH_PYMALLOC - v = Py_NewRef(Py_True); + v = Py_True; #else - v = Py_NewRef(Py_False); + v = Py_False; #endif - int rc = PyModule_AddObjectRef(mod, "WITH_PYMALLOC", v); - Py_DECREF(v); - if (rc < 0) { + if (PyModule_AddObjectRef(mod, "WITH_PYMALLOC", v) < 0) { return -1; } diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 7167943fffab39..4cf567b3314980 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -516,13 +516,7 @@ static PyFunction_WatchCallback func_watcher_callbacks[NUM_TEST_FUNC_WATCHERS] = static int add_func_event(PyObject *module, const char *name, PyFunction_WatchEvent event) { - PyObject *value = PyLong_FromLong(event); - if (value == NULL) { - return -1; - } - int ok = PyModule_AddObjectRef(module, name, value); - Py_DECREF(value); - return ok; + return PyModule_Add(module, name, PyLong_FromLong(event)); } static PyObject * diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 922b41005e8419..c42a15a0eff494 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -137,13 +137,7 @@ init_module(PyObject *module, module_state *state) } double d = _PyTime_AsSecondsDouble(state->initialized); - PyObject *initialized = PyFloat_FromDouble(d); - if (initialized == NULL) { - return -1; - } - int rc = PyModule_AddObjectRef(module, "_module_initialized", initialized); - Py_DECREF(initialized); - if (rc < 0) { + if (PyModule_Add(module, "_module_initialized", PyFloat_FromDouble(d)) < 0) { return -1; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 5721ed4412be57..bd8a98a46579a3 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1812,16 +1812,13 @@ add_errors_module(PyObject *mod) goto error; } - int rc = PyModule_AddObjectRef(errors_module, "codes", codes_dict); - Py_CLEAR(codes_dict); - if (rc < 0) { - goto error; + if (PyModule_Add(errors_module, "codes", codes_dict) < 0) { + Py_DECREF(rev_codes_dict); + return -1; } - rc = PyModule_AddObjectRef(errors_module, "messages", rev_codes_dict); - Py_CLEAR(rev_codes_dict); - if (rc < 0) { - goto error; + if (PyModule_Add(errors_module, "messages", rev_codes_dict) < 0) { + return -1; } return 0; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 1d3f34b857a1d2..39bbc911712376 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7425,9 +7425,7 @@ socket_exec(PyObject *m) sock_free_api(capi); goto error; } - int rc = PyModule_AddObjectRef(m, PySocket_CAPI_NAME, capsule); - Py_DECREF(capsule); - if (rc < 0) { + if (PyModule_Add(m, PySocket_CAPI_NAME, capsule) < 0) { goto error; } @@ -8818,13 +8816,7 @@ socket_exec(PyObject *m) }; int i; for (i = 0; i < Py_ARRAY_LENGTH(codes); ++i) { - PyObject *tmp = PyLong_FromUnsignedLong(codes[i]); - if (tmp == NULL) { - goto error; - } - int rc = PyModule_AddObjectRef(m, names[i], tmp); - Py_DECREF(tmp); - if (rc < 0) { + if (PyModule_Add(m, names[i], PyLong_FromUnsignedLong(codes[i])) < 0) { goto error; } } diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index b6e50528a23c86..966123f4624c08 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1496,13 +1496,7 @@ unicodedata_exec(PyObject *module) } /* Export C API */ - PyObject *capsule = unicodedata_create_capi(); - if (capsule == NULL) { - return -1; - } - int rc = PyModule_AddObjectRef(module, "_ucnhash_CAPI", capsule); - Py_DECREF(capsule); - if (rc < 0) { + if (PyModule_Add(module, "_ucnhash_CAPI", unicodedata_create_capi()) < 0) { return -1; } return 0; diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 53ef26b732f615..afc810adcf7499 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -565,15 +565,9 @@ static struct PyMethodDef msvcrt_functions[] = { }; static int -insertptr(PyObject *mod, char *name, void *value) +insertptr(PyObject *mod, const char *name, void *value) { - PyObject *v = PyLong_FromVoidPtr(value); - if (v == NULL) { - return -1; - } - int rc = PyModule_AddObjectRef(mod, name, v); - Py_DECREF(v); - return rc; + return PyModule_Add(mod, name, PyLong_FromVoidPtr(value)); } #define INSERTINT(MOD, NAME, VAL) do { \ @@ -646,12 +640,7 @@ exec_module(PyObject* m) _VC_CRT_MINOR_VERSION, _VC_CRT_BUILD_VERSION, _VC_CRT_RBUILD_VERSION); - if (version == NULL) { - return -1; - } - int st = PyModule_AddObjectRef(m, "CRT_ASSEMBLY_VERSION", version); - Py_DECREF(version); - if (st < 0) { + if (PyModule_Add(m, "CRT_ASSEMBLY_VERSION", version) < 0) { return -1; } #endif diff --git a/PC/winreg.c b/PC/winreg.c index aa2055c7e619d2..5252f78a9bdf72 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2079,15 +2079,9 @@ static struct PyMethodDef winreg_methods[] = { } while (0) static int -inskey(PyObject *mod, char *name, HKEY key) +inskey(PyObject *mod, const char *name, HKEY key) { - PyObject *v = PyLong_FromVoidPtr(key); - if (v == NULL) { - return -1; - } - int rc = PyModule_AddObjectRef(mod, name, v); - Py_DECREF(v); - return rc; + return PyModule_Add(mod, name, PyLong_FromVoidPtr(key)); } #define ADD_KEY(VAL) do { \ diff --git a/Python/import.c b/Python/import.c index 3e52a4e4eb1450..cf993cbd62a2ef 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3844,14 +3844,9 @@ imp_module_exec(PyObject *module) { const wchar_t *mode = _Py_GetConfig()->check_hash_pycs_mode; PyObject *pyc_mode = PyUnicode_FromWideChar(mode, -1); - if (pyc_mode == NULL) { + if (PyModule_Add(module, "check_hash_based_pycs", pyc_mode) < 0) { return -1; } - if (PyModule_AddObjectRef(module, "check_hash_based_pycs", pyc_mode) < 0) { - Py_DECREF(pyc_mode); - return -1; - } - Py_DECREF(pyc_mode); return 0; } From 505eede38d141d43e40e246319b157e3c77211d3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jul 2023 00:46:50 +0200 Subject: [PATCH 69/75] Docs: Argument Clinic: Group guides about default values (#106872) Previous ToC layout (excerpt): - How to use symbolic default values ... - How to assign default values to parameter - How to use the ``NULL`` default value - How to use expressions as default values New layout: - How to assign default values to parameter - The ``NULL`` default value - Symbolic default values - Expressions as default values --- Doc/howto/clinic.rst | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 12d7a77d43209a..efeb22c618b512 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -564,22 +564,6 @@ How-to guides ============= -How to use symbolic default values ----------------------------------- - -The default value you provide for a parameter can't be any arbitrary -expression. Currently the following are explicitly supported: - -* Numeric constants (integer and float) -* String constants -* ``True``, ``False``, and ``None`` -* Simple symbolic constants like ``sys.maxsize``, which must - start with the name of the module - -(In the future, this may need to get even more elaborate, -to allow full expressions like ``CONSTANT - 1``.) - - How to to rename C functions and variables generated by Argument Clinic ----------------------------------------------------------------------- @@ -965,8 +949,8 @@ There's also special support for a default value of ``NULL``, and for simple expressions, documented in the following sections. -How to use the ``NULL`` default value -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The ``NULL`` default value +^^^^^^^^^^^^^^^^^^^^^^^^^^ For string and object parameters, you can set them to ``None`` to indicate that there's no default. However, that means the C variable will be @@ -976,8 +960,24 @@ behaves like a default value of ``None``, but the C variable is initialized with ``NULL``. -How to use expressions as default values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Symbolic default values +^^^^^^^^^^^^^^^^^^^^^^^ + +The default value you provide for a parameter can't be any arbitrary +expression. Currently the following are explicitly supported: + +* Numeric constants (integer and float) +* String constants +* ``True``, ``False``, and ``None`` +* Simple symbolic constants like ``sys.maxsize``, which must + start with the name of the module + +(In the future, this may need to get even more elaborate, +to allow full expressions like ``CONSTANT - 1``.) + + +Expressions as default values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The default value for a parameter can be more than just a literal value. It can be an entire expression, using math operators and looking up attributes From 663854d73b35feeb004ae0970e45b53ca27774a1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 18 Jul 2023 15:20:31 -0800 Subject: [PATCH 70/75] gh-106727: Make `inspect.getsource` smarter for class for same name definitions (#106815) --- Lib/inspect.py | 57 +++++++++++++++---- Lib/test/inspect_fodder2.py | 20 +++++++ Lib/test/test_inspect.py | 5 +- ...-07-16-23-59-33.gh-issue-106727.bk3uCu.rst | 1 + 4 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-16-23-59-33.gh-issue-106727.bk3uCu.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 15f94a194856ac..675714dc8b3f70 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1034,9 +1034,13 @@ class ClassFoundException(Exception): class _ClassFinder(ast.NodeVisitor): - def __init__(self, qualname): + def __init__(self, cls, tree, lines, qualname): self.stack = [] + self.cls = cls + self.tree = tree + self.lines = lines self.qualname = qualname + self.lineno_found = [] def visit_FunctionDef(self, node): self.stack.append(node.name) @@ -1057,11 +1061,48 @@ def visit_ClassDef(self, node): line_number = node.lineno # decrement by one since lines starts with indexing by zero - line_number -= 1 - raise ClassFoundException(line_number) + self.lineno_found.append((line_number - 1, node.end_lineno)) self.generic_visit(node) self.stack.pop() + def get_lineno(self): + self.visit(self.tree) + lineno_found_number = len(self.lineno_found) + if lineno_found_number == 0: + raise OSError('could not find class definition') + elif lineno_found_number == 1: + return self.lineno_found[0][0] + else: + # We have multiple candidates for the class definition. + # Now we have to guess. + + # First, let's see if there are any method definitions + for member in self.cls.__dict__.values(): + if isinstance(member, types.FunctionType): + for lineno, end_lineno in self.lineno_found: + if lineno <= member.__code__.co_firstlineno <= end_lineno: + return lineno + + class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) + for lineno, end_lineno in self.lineno_found] + + # Maybe the class has a docstring and it's unique? + if self.cls.__doc__: + ret = None + for candidate, lineno in class_strings: + if self.cls.__doc__.strip() in candidate: + if ret is None: + ret = lineno + else: + break + else: + if ret is not None: + return ret + + # We are out of ideas, just return the last one found, which is + # slightly better than previous ones + return self.lineno_found[-1][0] + def findsource(object): """Return the entire source file and starting line number for an object. @@ -1098,14 +1139,8 @@ def findsource(object): qualname = object.__qualname__ source = ''.join(lines) tree = ast.parse(source) - class_finder = _ClassFinder(qualname) - try: - class_finder.visit(tree) - except ClassFoundException as e: - line_number = e.args[0] - return lines, line_number - else: - raise OSError('could not find class definition') + class_finder = _ClassFinder(object, tree, lines, qualname) + return lines, class_finder.get_lineno() if ismethod(object): object = object.__func__ diff --git a/Lib/test/inspect_fodder2.py b/Lib/test/inspect_fodder2.py index 03464613694605..8639cf2e72cd7a 100644 --- a/Lib/test/inspect_fodder2.py +++ b/Lib/test/inspect_fodder2.py @@ -290,3 +290,23 @@ def complex_decorated(foo=0, bar=lambda: 0): nested_lambda = ( lambda right: [].map( lambda length: ())) + +# line 294 +if True: + class cls296: + def f(): + pass +else: + class cls296: + def g(): + pass + +# line 304 +if False: + class cls310: + def f(): + pass +else: + class cls310: + def g(): + pass diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 64afeec351b353..33a593f3591d68 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -949,7 +949,6 @@ def test_class_decorator(self): self.assertSourceEqual(mod2.cls196.cls200, 198, 201) def test_class_inside_conditional(self): - self.assertSourceEqual(mod2.cls238, 238, 240) self.assertSourceEqual(mod2.cls238.cls239, 239, 240) def test_multiple_children_classes(self): @@ -975,6 +974,10 @@ def test_nested_class_definition_inside_async_function(self): self.assertSourceEqual(mod2.cls226, 231, 235) self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + def test_class_definition_same_name_diff_methods(self): + self.assertSourceEqual(mod2.cls296, 296, 298) + self.assertSourceEqual(mod2.cls310, 310, 312) + class TestNoEOL(GetSourceBase): def setUp(self): self.tempdir = TESTFN + '_dir' diff --git a/Misc/NEWS.d/next/Library/2023-07-16-23-59-33.gh-issue-106727.bk3uCu.rst b/Misc/NEWS.d/next/Library/2023-07-16-23-59-33.gh-issue-106727.bk3uCu.rst new file mode 100644 index 00000000000000..e4ea0ce1890d2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-16-23-59-33.gh-issue-106727.bk3uCu.rst @@ -0,0 +1 @@ +Make :func:`inspect.getsource` smarter for class for same name definitions From 7513e2e7e48f6c004ed9bce55f2dcc6b388e02cd Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 19 Jul 2023 10:18:23 +0900 Subject: [PATCH 71/75] gh-106751: Optimize KqueueSelector.select() for many iteration case (gh-106864) --- Lib/selectors.py | 14 ++++++-------- .../2023-07-18-23-05-12.gh-issue-106751.tVvzN_.rst | 2 ++ 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-18-23-05-12.gh-issue-106751.tVvzN_.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index a42d1563406417..d13405963f219d 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -547,23 +547,21 @@ def select(self, timeout=None): # If max_ev is 0, kqueue will ignore the timeout. For consistent # behavior with the other selector classes, we prevent that here # (using max). See https://bugs.python.org/issue29255 - max_ev = max(len(self._fd_to_key), 1) + max_ev = len(self._fd_to_key) or 1 ready = [] try: kev_list = self._selector.control(None, max_ev, timeout) except InterruptedError: return ready + + fd_to_key_get = self._fd_to_key.get for kev in kev_list: fd = kev.ident flag = kev.filter - events = 0 - if flag == select.KQ_FILTER_READ: - events |= EVENT_READ - if flag == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._fd_to_key.get(fd) + key = fd_to_key_get(fd) if key: + events = ((flag == select.KQ_FILTER_READ and EVENT_READ) + | (flag == select.KQ_FILTER_WRITE and EVENT_WRITE)) ready.append((key, events & key.events)) return ready diff --git a/Misc/NEWS.d/next/Library/2023-07-18-23-05-12.gh-issue-106751.tVvzN_.rst b/Misc/NEWS.d/next/Library/2023-07-18-23-05-12.gh-issue-106751.tVvzN_.rst new file mode 100644 index 00000000000000..1cb8424b6221ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-18-23-05-12.gh-issue-106751.tVvzN_.rst @@ -0,0 +1,2 @@ +Optimize :meth:`KqueueSelector.select` for many iteration case. Patch By +Dong-hee Na. From e6f96cf9c62e38514e8f5465a1c43f85d861adb2 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 19 Jul 2023 15:12:38 +0900 Subject: [PATCH 72/75] gh-106751: Optimize SelectSelector.select() for many iteration case (gh-106879) --- Lib/selectors.py | 18 ++++++++---------- ...3-07-19-10-45-24.gh-issue-106751.3HJ1of.rst | 2 ++ 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-19-10-45-24.gh-issue-106751.3HJ1of.rst diff --git a/Lib/selectors.py b/Lib/selectors.py index d13405963f219d..13497a24097232 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -314,17 +314,15 @@ def select(self, timeout=None): r, w, _ = self._select(self._readers, self._writers, [], timeout) except InterruptedError: return ready - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._fd_to_key.get(fd) + r = frozenset(r) + w = frozenset(w) + rw = r | w + fd_to_key_get = self._fd_to_key.get + for fd in rw: + key = fd_to_key_get(fd) if key: + events = ((fd in r and EVENT_READ) + | (fd in w and EVENT_WRITE)) ready.append((key, events & key.events)) return ready diff --git a/Misc/NEWS.d/next/Library/2023-07-19-10-45-24.gh-issue-106751.3HJ1of.rst b/Misc/NEWS.d/next/Library/2023-07-19-10-45-24.gh-issue-106751.3HJ1of.rst new file mode 100644 index 00000000000000..2696b560371d13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-19-10-45-24.gh-issue-106751.3HJ1of.rst @@ -0,0 +1,2 @@ +Optimize :meth:`SelectSelector.select` for many iteration case. Patch By +Dong-hee Na. From 70b961ed93f67e34d0624e178f6029c886afaeee Mon Sep 17 00:00:00 2001 From: Yonatan Bitton Date: Wed, 19 Jul 2023 14:03:47 +0300 Subject: [PATCH 73/75] gh-104090: Fix unittest collectedDurations resources leak (#106795) --- Lib/unittest/result.py | 3 ++- .../next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 7757dba9670b43..3ace0a5b7bf2ef 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -166,7 +166,8 @@ def addDuration(self, test, elapsed): """ # support for a TextTestRunner using an old TestResult class if hasattr(self, "collectedDurations"): - self.collectedDurations.append((test, elapsed)) + # Pass test repr and not the test object itself to avoid resources leak + self.collectedDurations.append((str(test), elapsed)) def wasSuccessful(self): """Tells whether or not this result was a success.""" diff --git a/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst b/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst new file mode 100644 index 00000000000000..5cc6c5bbe15446 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst @@ -0,0 +1 @@ +Avoid creating a reference to the test object in :meth:`~unittest.TestResult.collectedDurations`. From a1a3193990cd6658c1fe859b88a2bc03971a16df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 19 Jul 2023 13:07:40 +0200 Subject: [PATCH 74/75] Export _PyEval_SetProfile() as a function, not data (#106887) --- Include/cpython/ceval.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index a9616bd6a4f518..5255d715142b97 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -4,7 +4,7 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *); -PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *); PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); From c6c5665ee0c0a5ddc96da255c9a62daa332c32b3 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 19 Jul 2023 18:59:55 +0100 Subject: [PATCH 75/75] GH-100502: Add `pathlib.PurePath.pathmod` attribute (GH-106533) This instance attribute stores the implementation of `os.path` used for low-level path operations: either `posixpath` or `ntpath`. --- Doc/library/pathlib.rst | 7 ++ Lib/pathlib.py | 84 +++++++++---------- Lib/test/test_pathlib.py | 32 +++---- ...-07-07-21-15-17.gh-issue-100502.Iici1B.rst | 3 + 4 files changed, 68 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-07-21-15-17.gh-issue-100502.Iici1B.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index af81df217eea92..01dabe286969bb 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -303,6 +303,13 @@ Methods and properties Pure paths provide the following methods and properties: +.. attribute:: PurePath.pathmod + + The implementation of the :mod:`os.path` module used for low-level path + operations: either ``posixpath`` or ``ntpath``. + + .. versionadded:: 3.13 + .. attribute:: PurePath.drive A string representing the drive letter or name, if any:: diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f3813e04109904..8ff4d4ea19168f 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -56,8 +56,8 @@ def _ignore_error(exception): @functools.cache -def _is_case_sensitive(flavour): - return flavour.normcase('Aa') == 'Aa' +def _is_case_sensitive(pathmod): + return pathmod.normcase('Aa') == 'Aa' # # Globbing helpers @@ -293,7 +293,7 @@ class PurePath: # path. It's set when `__hash__()` is called for the first time. '_hash', ) - _flavour = os.path + pathmod = os.path def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing @@ -314,7 +314,7 @@ def __init__(self, *args): paths = [] for arg in args: if isinstance(arg, PurePath): - if arg._flavour is ntpath and self._flavour is posixpath: + if arg.pathmod is ntpath and self.pathmod is posixpath: # GH-103631: Convert separators for backwards compatibility. paths.extend(path.replace('\\', '/') for path in arg._raw_paths) else: @@ -343,11 +343,11 @@ def with_segments(self, *pathsegments): def _parse_path(cls, path): if not path: return '', '', [] - sep = cls._flavour.sep - altsep = cls._flavour.altsep + sep = cls.pathmod.sep + altsep = cls.pathmod.altsep if altsep: path = path.replace(altsep, sep) - drv, root, rel = cls._flavour.splitroot(path) + drv, root, rel = cls.pathmod.splitroot(path) if not root and drv.startswith(sep) and not drv.endswith(sep): drv_parts = drv.split(sep) if len(drv_parts) == 4 and drv_parts[2] not in '?.': @@ -366,7 +366,7 @@ def _load_parts(self): elif len(paths) == 1: path = paths[0] else: - path = self._flavour.join(*paths) + path = self.pathmod.join(*paths) drv, root, tail = self._parse_path(path) self._drv = drv self._root = root @@ -384,10 +384,10 @@ def _from_parsed_parts(self, drv, root, tail): @classmethod def _format_parsed_parts(cls, drv, root, tail): if drv or root: - return drv + root + cls._flavour.sep.join(tail) - elif tail and cls._flavour.splitdrive(tail[0])[0]: + return drv + root + cls.pathmod.sep.join(tail) + elif tail and cls.pathmod.splitdrive(tail[0])[0]: tail = ['.'] + tail - return cls._flavour.sep.join(tail) + return cls.pathmod.sep.join(tail) def __str__(self): """Return the string representation of the path, suitable for @@ -405,8 +405,7 @@ def __fspath__(self): def as_posix(self): """Return the string representation of the path with forward (/) slashes.""" - f = self._flavour - return str(self).replace(f.sep, '/') + return str(self).replace(self.pathmod.sep, '/') def __bytes__(self): """Return the bytes representation of the path. This is only @@ -442,7 +441,7 @@ def _str_normcase(self): try: return self._str_normcase_cached except AttributeError: - if _is_case_sensitive(self._flavour): + if _is_case_sensitive(self.pathmod): self._str_normcase_cached = str(self) else: self._str_normcase_cached = str(self).lower() @@ -454,7 +453,7 @@ def _parts_normcase(self): try: return self._parts_normcase_cached except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self._flavour.sep) + self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) return self._parts_normcase_cached @property @@ -467,14 +466,14 @@ def _lines(self): if path_str == '.': self._lines_cached = '' else: - trans = _SWAP_SEP_AND_NEWLINE[self._flavour.sep] + trans = _SWAP_SEP_AND_NEWLINE[self.pathmod.sep] self._lines_cached = path_str.translate(trans) return self._lines_cached def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented - return self._str_normcase == other._str_normcase and self._flavour is other._flavour + return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod def __hash__(self): try: @@ -484,22 +483,22 @@ def __hash__(self): return self._hash def __lt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: return NotImplemented return self._parts_normcase < other._parts_normcase def __le__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: return NotImplemented return self._parts_normcase <= other._parts_normcase def __gt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: return NotImplemented return self._parts_normcase > other._parts_normcase def __ge__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: return NotImplemented return self._parts_normcase >= other._parts_normcase @@ -584,9 +583,9 @@ def with_name(self, name): """Return a new path with the file name changed.""" if not self.name: raise ValueError("%r has an empty name" % (self,)) - f = self._flavour - drv, root, tail = f.splitroot(name) - if drv or root or not tail or f.sep in tail or (f.altsep and f.altsep in tail): + m = self.pathmod + drv, root, tail = m.splitroot(name) + if drv or root or not tail or m.sep in tail or (m.altsep and m.altsep in tail): raise ValueError("Invalid name %r" % (name)) return self._from_parsed_parts(self.drive, self.root, self._tail[:-1] + [name]) @@ -600,8 +599,8 @@ def with_suffix(self, suffix): has no suffix, add given suffix. If the given suffix is an empty string, remove the suffix from the path. """ - f = self._flavour - if f.sep in suffix or f.altsep and f.altsep in suffix: + m = self.pathmod + if m.sep in suffix or m.altsep and m.altsep in suffix: raise ValueError("Invalid suffix %r" % (suffix,)) if suffix and not suffix.startswith('.') or suffix == '.': raise ValueError("Invalid suffix %r" % (suffix)) @@ -702,22 +701,22 @@ def parents(self): def is_absolute(self): """True if the path is absolute (has both a root and, if applicable, a drive).""" - if self._flavour is ntpath: + if self.pathmod is ntpath: # ntpath.isabs() is defective - see GH-44626. return bool(self.drive and self.root) - elif self._flavour is posixpath: + elif self.pathmod is posixpath: # Optimization: work with raw paths on POSIX. for path in self._raw_paths: if path.startswith('/'): return True return False else: - return self._flavour.isabs(str(self)) + return self.pathmod.isabs(str(self)) def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" - if self._flavour is posixpath or not self._tail: + if self.pathmod is posixpath or not self._tail: return False # NOTE: the rules for reserved names seem somewhat complicated @@ -737,7 +736,7 @@ def match(self, path_pattern, *, case_sensitive=None): if not isinstance(path_pattern, PurePath): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self._flavour) + case_sensitive = _is_case_sensitive(self.pathmod) pattern = _compile_pattern_lines(path_pattern._lines, case_sensitive) if path_pattern.drive or path_pattern.root: return pattern.match(self._lines) is not None @@ -758,7 +757,7 @@ class PurePosixPath(PurePath): On a POSIX system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = posixpath + pathmod = posixpath __slots__ = () @@ -768,7 +767,7 @@ class PureWindowsPath(PurePath): On a Windows system, instantiating a PurePath should return this object. However, you can also instantiate it directly on any system. """ - _flavour = ntpath + pathmod = ntpath __slots__ = () @@ -858,7 +857,7 @@ def is_mount(self): """ Check if this path is a mount point """ - return self._flavour.ismount(self) + return os.path.ismount(self) def is_symlink(self): """ @@ -879,7 +878,7 @@ def is_junction(self): """ Whether this path is a junction. """ - return self._flavour.isjunction(self) + return os.path.isjunction(self) def is_block_device(self): """ @@ -954,7 +953,8 @@ def samefile(self, other_path): other_st = other_path.stat() except AttributeError: other_st = self.with_segments(other_path).stat() - return self._flavour.samestat(st, other_st) + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): @@ -1017,7 +1017,7 @@ def _scandir(self): return os.scandir(self) def _make_child_relpath(self, name): - sep = self._flavour.sep + sep = self.pathmod.sep lines_name = name.replace('\n', sep) lines_str = self._lines path_str = str(self) @@ -1062,7 +1062,7 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): raise ValueError("Unacceptable pattern: {!r}".format(pattern)) pattern_parts = list(path_pattern._tail) - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): + if pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): # GH-65238: pathlib doesn't preserve trailing slash. Add it back. pattern_parts.append('') if pattern_parts[-1] == '**': @@ -1071,7 +1071,7 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): if case_sensitive is None: # TODO: evaluate case-sensitivity of each directory in _select_children(). - case_sensitive = _is_case_sensitive(self._flavour) + case_sensitive = _is_case_sensitive(self.pathmod) # If symlinks are handled consistently, and the pattern does not # contain '..' components, then we can use a 'walk-and-match' strategy @@ -1204,7 +1204,7 @@ def absolute(self): return self elif self.drive: # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) + cwd = os.path.abspath(self.drive) else: cwd = os.getcwd() # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). @@ -1230,7 +1230,7 @@ def check_eloop(e): raise RuntimeError("Symlink loop from %r" % e.filename) try: - s = self._flavour.realpath(self, strict=strict) + s = os.path.realpath(self, strict=strict) except OSError as e: check_eloop(e) raise @@ -1394,7 +1394,7 @@ def expanduser(self): """ if (not (self.drive or self.root) and self._tail and self._tail[0][:1] == '~'): - homedir = self._flavour.expanduser(self._tail[0]) + homedir = os.path.expanduser(self._tail[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") drv, root, tail = self._parse_path(homedir) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index eb2b0cfb26e85f..78948e3b720320 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -66,9 +66,9 @@ class PurePathTest(unittest.TestCase): def setUp(self): p = self.cls('a') - self.flavour = p._flavour - self.sep = self.flavour.sep - self.altsep = self.flavour.altsep + self.pathmod = p.pathmod + self.sep = self.pathmod.sep + self.altsep = self.pathmod.altsep def test_constructor_common(self): P = self.cls @@ -93,17 +93,17 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_different_flavours_unequal(self): + def test_different_pathmods_unequal(self): p = self.cls('a') - if p._flavour is posixpath: + if p.pathmod is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') self.assertNotEqual(p, q) - def test_different_flavours_unordered(self): + def test_different_pathmods_unordered(self): p = self.cls('a') - if p._flavour is posixpath: + if p.pathmod is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') @@ -188,16 +188,16 @@ def _get_drive_root_parts(self, parts): return path.drive, path.root, path.parts def _check_drive_root_parts(self, arg, *expected): - sep = self.flavour.sep + sep = self.pathmod.sep actual = self._get_drive_root_parts([x.replace('/', sep) for x in arg]) self.assertEqual(actual, expected) - if altsep := self.flavour.altsep: + if altsep := self.pathmod.altsep: actual = self._get_drive_root_parts([x.replace('/', altsep) for x in arg]) self.assertEqual(actual, expected) def test_drive_root_parts_common(self): check = self._check_drive_root_parts - sep = self.flavour.sep + sep = self.pathmod.sep # Unanchored parts. check((), '', '', ()) check(('a',), '', '', ('a',)) @@ -657,7 +657,7 @@ def test_with_suffix_common(self): self.assertRaises(ValueError, P('a/b').with_suffix, './.d') self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') self.assertRaises(ValueError, P('a/b').with_suffix, - (self.flavour.sep, 'd')) + (self.pathmod.sep, 'd')) def test_relative_to_common(self): P = self.cls @@ -2392,8 +2392,8 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_unsupported_flavour(self): - if self.cls._flavour is os.path: + def test_unsupported_pathmod(self): + if self.cls.pathmod is os.path: self.skipTest("path flavour is supported") else: self.assertRaises(pathlib.UnsupportedOperation, self.cls) @@ -2848,9 +2848,9 @@ def test_symlink_to_unsupported(self): def test_is_junction(self): P = self.cls(BASE) - with mock.patch.object(P._flavour, 'isjunction'): - self.assertEqual(P.is_junction(), P._flavour.isjunction.return_value) - P._flavour.isjunction.assert_called_once_with(P) + with mock.patch.object(P.pathmod, 'isjunction'): + self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) + P.pathmod.isjunction.assert_called_once_with(P) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", diff --git a/Misc/NEWS.d/next/Library/2023-07-07-21-15-17.gh-issue-100502.Iici1B.rst b/Misc/NEWS.d/next/Library/2023-07-07-21-15-17.gh-issue-100502.Iici1B.rst new file mode 100644 index 00000000000000..eea9564118df9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-21-15-17.gh-issue-100502.Iici1B.rst @@ -0,0 +1,3 @@ +Add :attr:`pathlib.PurePath.pathmod` class attribute that stores the +implementation of :mod:`os.path` used for low-level path operations: either +``posixpath`` or ``ntpath``.