From 0387c34f7c91428681ca8a4ba4e3d22b9acffde4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Sep 2024 10:53:17 +0200 Subject: [PATCH 01/66] gh-124402: Speed up test_free_threading and test_super (#124491) * Reduce the number of iterations and the number of threads so a whole test file takes less than a minute. * Refactor test_racing_iter_extend() to remove two levels of indentation. * test_monitoring() uses a sleep of 100 ms instead of 1 second. --- Lib/test/test_free_threading/test_list.py | 67 +++++++++---------- .../test_free_threading/test_monitoring.py | 14 ++-- Lib/test/test_free_threading/test_type.py | 6 +- Lib/test/test_super.py | 7 +- 4 files changed, 41 insertions(+), 53 deletions(-) diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index c6b58fcd86f449..a705161369e8dd 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -3,10 +3,13 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper +NTHREAD = 10 +OBJECT_COUNT = 5_000 + + class C: def __init__(self, v): self.v = v @@ -14,11 +17,8 @@ def __init__(self, v): @threading_helper.requires_working_threading() class TestList(TestCase): - @support.requires_resource('cpu') def test_racing_iter_append(self): - l = [] - OBJECT_COUNT = 10000 def writer_func(): for i in range(OBJECT_COUNT): @@ -34,7 +34,7 @@ def reader_func(): writer = Thread(target=writer_func) readers = [] - for x in range(30): + for x in range(NTHREAD): reader = Thread(target=reader_func) readers.append(reader) reader.start() @@ -44,39 +44,32 @@ def reader_func(): for reader in readers: reader.join() - @support.requires_resource('cpu') def test_racing_iter_extend(self): - iters = [ - lambda x: [x], - ] - for iter_case in iters: - with self.subTest(iter=iter_case): - l = [] - OBJECT_COUNT = 10000 - - def writer_func(): - for i in range(OBJECT_COUNT): - l.extend(iter_case(C(i + OBJECT_COUNT))) - - def reader_func(): - while True: - count = len(l) - for i, x in enumerate(l): - self.assertEqual(x.v, i + OBJECT_COUNT) - if count == OBJECT_COUNT: - break - - writer = Thread(target=writer_func) - readers = [] - for x in range(30): - reader = Thread(target=reader_func) - readers.append(reader) - reader.start() - - writer.start() - writer.join() - for reader in readers: - reader.join() + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend([C(i + OBJECT_COUNT)]) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() if __name__ == "__main__": diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index be582455d118ac..8fec01715531cb 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,7 +7,6 @@ import weakref from sys import monitoring -from test import support from test.support import threading_helper from threading import Thread, _PyRLock from unittest import TestCase @@ -15,7 +14,7 @@ class InstrumentationMultiThreadedMixin: thread_count = 10 - func_count = 200 + func_count = 50 fib = 12 def after_threads(self): @@ -37,14 +36,13 @@ def work(self, n, funcs): def start_work(self, n, funcs): # With the GIL builds we need to make sure that the hooks have # a chance to run as it's possible to run w/o releasing the GIL. - time.sleep(1) + time.sleep(0.1) self.work(n, funcs) def after_test(self): """Runs once after the test is done""" pass - @support.requires_resource('cpu') def test_instrumentation(self): # Setup a bunch of functions which will need instrumentation... funcs = [] @@ -220,29 +218,31 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) - @support.requires_resource('cpu') def test_set_local_trace_opcodes(self): def trace(frame, event, arg): frame.f_trace_opcodes = True return trace + loops = 1_000 + sys.settrace(trace) try: l = _PyRLock() def f(): - for i in range(3000): + for i in range(loops): with l: pass t = Thread(target=f) t.start() - for i in range(3000): + for i in range(loops): with l: pass t.join() finally: sys.settrace(None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 977bfd2c7fd2f7..51463b6bb8c1b4 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -5,7 +5,6 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper @@ -97,8 +96,9 @@ def reader_func(): self.run_one(writer_func, reader_func) - @support.requires_resource('cpu') def test___class___modification(self): + loops = 200 + class Foo: pass @@ -108,7 +108,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(5000): + for _ in range(loops): foo.__class__ = Bar type(foo) foo.__class__ = Foo diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index b0d1f12513d404..1222ec6a3c4109 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -4,7 +4,6 @@ import threading import unittest from unittest.mock import patch -from test import support from test.support import import_helper, threading_helper @@ -515,10 +514,6 @@ def test___class___modification_multithreaded(self): an audit hook. """ - if support.Py_GIL_DISABLED: - # gh-124402: On a Free Threaded build, the test takes a few minutes - support.requires('cpu') - class Foo: pass @@ -528,7 +523,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(5000): + for _ in range(200): foo.__class__ = Bar type(foo) foo.__class__ = Foo From f923605658a29ff9af5a62edc1fc10191977627b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 26 Sep 2024 06:29:43 -0400 Subject: [PATCH 02/66] gh-124538: Fix crash when using `gc.get_referents` on an untracked capsule object (#124559) --- Lib/test/test_gc.py | 18 ++++++++++++++++++ ...4-09-25-18-34-48.gh-issue-124538.nXZk4R.rst | 1 + Objects/capsule.c | 10 +++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 906f9884d6792f..bb7df1f5cfa7f7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1048,6 +1048,24 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + class IncrementalGCTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst new file mode 100644 index 00000000000000..33ae037ae56b0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst @@ -0,0 +1 @@ +Fixed crash when using :func:`gc.get_referents` on a capsule object. diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..28965e0f21b7a0 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -317,10 +317,14 @@ static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + + if (capsule->traverse_func != NULL) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return 0; } From 19fed6cf6eb51044fd0c02c6338259e2dd7fd462 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 26 Sep 2024 15:06:52 +0300 Subject: [PATCH 03/66] gh-124234: Improve docs for `Mock.reset_mock` (#124237) --- Doc/library/unittest.mock.rst | 43 +++++++++++++++++++++++++++-------- Lib/unittest/mock.py | 6 +++-- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d603a163c2e82d..cc2b1b4299553c 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -401,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -409,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: + + .. doctest:: + + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + Note that :meth:`reset_mock` *doesn't* clear the :attr:`return_value`, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - :attr:`return_value` or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + set using normal assignment by default. - .. note:: *return_value*, and *side_effect* are keyword-only - arguments. + Child mocks are reset as well. + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index bb34c7436047ad..df3901f9660ac1 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -628,7 +628,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None, *, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -2218,7 +2220,7 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() - def reset_mock(self, /, *args, return_value=False, **kwargs): + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): if ( return_value and self._mock_name From 274d9ab619b8150a613275835234ea9ef935f21f Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 26 Sep 2024 15:40:18 +0300 Subject: [PATCH 04/66] gh-123560: Correct docs for "empty" format type for floats (#123561) --- Doc/library/string.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1f316307965c11..57a1f920523035 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -574,11 +574,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | From 09aebb1fbc0c1d771d4942844d5e2077fcdf56c9 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Thu, 26 Sep 2024 15:38:25 +0200 Subject: [PATCH 05/66] gh-118181: Fix parameter markup in AST docs (#124473) --- Doc/library/ast.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a8a18ad31fb773..a9518859b83478 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -178,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -199,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -214,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -243,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -1771,9 +1771,9 @@ aliases. .. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. *name* is the name of the type variable. - *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. *default_value* + A :class:`typing.TypeVar`. ``name`` is the name of the type variable. + ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. ``default_value`` is the default value; if the :class:`!TypeVar` has no default, this attribute will be set to ``None``. @@ -1801,8 +1801,8 @@ aliases. .. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. - *default_value* is the default value; if the :class:`!ParamSpec` has no default, + A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, this attribute will be set to ``None``. .. doctest:: @@ -1836,8 +1836,8 @@ aliases. .. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. - *default_value* is the default value; if the :class:`!TypeVarTuple` has no + A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no default, this attribute will be set to ``None``. .. doctest:: From cf2418076d7cf69a3bd4bf6be0e0001635a7ad4d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 26 Sep 2024 07:12:56 -0700 Subject: [PATCH 06/66] gh-119127: functools: Improve docs for partial and Placeholder (#124575) --- Doc/library/functools.rst | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 774b3262117723..c2c25ca67f338a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -354,7 +354,7 @@ The :mod:`functools` module defines the following functions: newfunc.keywords = keywords return newfunc - The :func:`partial` function is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument @@ -368,10 +368,11 @@ The :mod:`functools` module defines the following functions: 18 If :data:`Placeholder` sentinels are present in *args*, they will be filled first - when :func:`partial` is called. This allows custom selection of positional arguments - to be pre-filled when constructing a :ref:`partial object `. + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, only the + first positional argument can be pre-filled. - If :data:`!Placeholder` sentinels are present, all of them must be filled at call time: + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: .. doctest:: @@ -379,14 +380,15 @@ The :mod:`functools` module defines the following functions: >>> say_to_world('Hello', 'dear') Hello dear world! - Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because - only one positional argument is provided, while there are two placeholders - in :ref:`partial object `. + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. - Successive :func:`partial` applications fill :data:`!Placeholder` sentinels - of the input :func:`partial` objects with new positional arguments. - A place for positional argument can be retained by inserting new - :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: .. doctest:: @@ -402,8 +404,8 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - Note, :data:`!Placeholder` has no special treatment when used for keyword - argument of :data:`!Placeholder`. + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. @@ -791,7 +793,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes are not created automatically. From abe5f799e6ce1d177f79554f1b84d348b6141045 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 26 Sep 2024 17:15:38 +0300 Subject: [PATCH 07/66] gh-124498: Fix `TypeAliasType` not to be generic, when `type_params=()` (#124499) --- Lib/test/test_type_aliases.py | 13 +++++++++++++ .../2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst | 2 ++ Objects/typevarobject.c | 11 ++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 49d6aa810304fb..ebb65d8c6cf81b 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -211,6 +211,19 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] def test_keywords(self): TA = TypeAliasType(name="TA", value=int) diff --git a/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst new file mode 100644 index 00000000000000..4dbf4eb709733d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst @@ -0,0 +1,2 @@ +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 09e9ab39364742..552c4745590194 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1915,7 +1915,16 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + ta->type_params = NULL; + } + else { + ta->type_params = Py_NewRef(type_params); + } ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); From 257a20a81764fcc17bcde9c0cec57fbc53a4adc7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Sep 2024 16:50:38 +0200 Subject: [PATCH 08/66] gh-119127: Fix _functools.Placeholder singleton (#124601) * The module state now stores a strong reference to the Placeholder singleton. * Use a regular dealloc function. * Add Py_TPFLAGS_HAVE_GC flag and a traverse function to help the GC to collect the type when a _functools extension is unloaded. --- Modules/_functoolsmodule.c | 48 +++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 2b3bd7c3de1176..31cf7bcc09782c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -26,7 +26,7 @@ typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; PyTypeObject *placeholder_type; - PyObject *placeholder; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -76,13 +76,12 @@ static PyMethodDef placeholder_methods[] = { }; static void -placeholder_dealloc(PyObject* placeholder) +placeholder_dealloc(PyObject* self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref Placeholder out of existence. Instead, - * since Placeholder is an immortal object, re-set the reference count. - */ - _Py_SetImmortal(placeholder); + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); } static PyObject * @@ -93,10 +92,26 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + if (state->placeholder == NULL) { - state->placeholder = PyType_GenericNew(type, NULL, NULL); + state->placeholder = Py_NewRef(placeholder); } - return state->placeholder; + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } static PyType_Slot placeholder_type_slots[] = { @@ -105,13 +120,14 @@ static PyType_Slot placeholder_type_slots[] = { {Py_tp_doc, (void *)placeholder_doc}, {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, {0, 0} }; static PyType_Spec placeholder_type_spec = { .name = "functools._PlaceholderType", .basicsize = sizeof(placeholderobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, .slots = placeholder_type_slots }; @@ -1717,13 +1733,17 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->placeholder_type) < 0) { return -1; } - state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); - if (state->placeholder == NULL) { + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { return -1; } - if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) { + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); return -1; } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1769,6 +1789,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1781,6 +1802,7 @@ _functools_clear(PyObject *module) _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); From 9c98fdab7d1167211c9d162c418e2b443a02867a Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 26 Sep 2024 18:30:21 +0300 Subject: [PATCH 09/66] gh-124606: Fix reference leak in error path in `datetime_fromisoformat` in `_datetimemodule.c` (#124607) Previously `tzdata` and `dtstr_clean` were not decrefed in the `invalid_iso_midnight` error path of the `datetime_isoformat` function. --- Modules/_datetimemodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0d91bef2ff9bda..90527d2a3e0350 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5959,6 +5959,8 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) invalid_iso_midnight: PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); return NULL; invalid_string_error: From 2c472d36b776636fb00881a717f69e43672588b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:44:25 +0100 Subject: [PATCH 10/66] Doc: Use the short version for daily downloads (#124602) --- Doc/tools/templates/download.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index c978e61b16a49e..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -13,7 +13,7 @@ {% endif %} {% block body %} -

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} From d7248cdbc32ffb401a27499223b7f0c2f80614f8 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 27 Sep 2024 01:21:11 +0900 Subject: [PATCH 11/66] gh-124153: Remove `_PyType_GetModuleByDef2` private function (GH-124261) Thank you! --- Include/internal/pycore_typeobject.h | 1 - Modules/_collectionsmodule.c | 20 +++++++---------- Modules/_decimal/_decimal.c | 18 ++++++++++----- Objects/typeobject.c | 33 +++++----------------------- 4 files changed, 26 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index ca5a1e2adb4787..118bc98b35d5e3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -209,7 +209,6 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // Export for _testinternalcapi extension. PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbfed59995c21e..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2179,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2358,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2454,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e99a96ab93392e..68d1da9faab867 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -122,6 +122,7 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -134,10 +135,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), - &_decimal_module); - assert(mod != NULL); - return get_module_state(mod); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); + } + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -745,7 +752,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -5041,6 +5048,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 68e481f8e5163b..3368c1ef577d14 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5207,8 +5207,8 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -static inline PyObject * -get_module_by_def(PyTypeObject *type, PyModuleDef *def) +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); @@ -5241,7 +5241,7 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -5254,37 +5254,14 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) } } END_TYPE_LOCK(); - return res; -} -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *module = get_module_by_def(type, def); - if (module == NULL) { + if (res == NULL) { PyErr_Format( PyExc_TypeError, "PyType_GetModuleByDef: No superclass of '%s' has the given module", type->tp_name); } - return module; -} - -PyObject * -_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, - PyModuleDef *def) -{ - PyObject *module = get_module_by_def(left, def); - if (module == NULL) { - module = get_module_by_def(right, def); - if (module == NULL) { - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " - "the given module", left->tp_name, right->tp_name); - } - } - return module; + return res; } From 986a4e1b6fcae7fe7a1d0a26aea446107dd58dd2 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 26 Sep 2024 09:35:13 -0700 Subject: [PATCH 12/66] Cleanup unnecessary curframe_locals usage (#124369) --- Lib/pdb.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 443160eaaae887..28270167565564 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -403,13 +403,6 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary used to be updated from the actual frame - # locals whenever the .f_locals accessor was called, so it was - # cached here to ensure that modifications were not overwritten. While - # the caching is no longer required now that f_locals is a direct proxy - # on optimized frames, it's also harmless, so the code structure has - # been left unchanged. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) if self._chained_exceptions: @@ -732,7 +725,7 @@ def _exec_in_closure(self, source, globals, locals): def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: buffer = line @@ -960,7 +953,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -991,7 +984,7 @@ def completedefault(self, text, line, begidx, endidx): # Use rlcompleter to do the completion state = 0 matches = [] - completer = Completer(self.curframe.f_globals | self.curframe_locals) + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) while (match := completer.complete(text, state)) is not None: matches.append(match) state += 1 @@ -1153,7 +1146,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1458,7 +1451,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1704,7 +1696,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1749,7 +1741,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1769,15 +1761,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1785,7 +1777,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -2029,7 +2021,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") From 43979fad904bcc343f90cb526faa526c45fcbfa4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 26 Sep 2024 11:50:31 -0700 Subject: [PATCH 13/66] Programming FAQ: Mention object.__setattr__ as a technique for delegation (#124617) This is used for example by threading.local in the stdlib. --- Doc/faq/programming.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4a6f1ca57d89e3..fa7b22bde1dc6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:attr:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? From 7d3497f617edf77cb6ead6f5e62bce98d77b9ab8 Mon Sep 17 00:00:00 2001 From: Matthew Rahtz Date: Thu, 26 Sep 2024 20:35:17 +0100 Subject: [PATCH 14/66] gh-115528: Update language reference for PEP 646 (#121181) To recap: the objective is to make starred expressions valid in `subscription`, which is used for generics: `Generic[...]`, `list[...]`, etc. What _is_ gramatically valid in such contexts? Seemingly any of the following. (At least, none of the following throw `SyntaxError` in a 3.12.3 REPL.) Generic[x] Generic[*x] Generic[*x, y] Generic[y, *x] Generic[x := 1] Generic[x := 1, y := 2] So introducting flexible_expression: expression | assignment_expression | starred_item end then switching `subscription` to use `flexible_expression` sorts that. But then we need to field `yield` - for which any of the following are apparently valid: yield x yield x, yield x, y yield *x, yield *x, *y Introducing a separate `yield_list` is the simplest way I've been figure out to do this - separating out the special case of `starred_item ,`. Co-authored-by: Jelle Zijlstra --- Doc/reference/compound_stmts.rst | 10 ++++++-- Doc/reference/expressions.rst | 39 ++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 097a39cea31c67..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1217,9 +1217,10 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1326,11 +1327,16 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the semantics of a function. See :ref:`annotations` for more information on annotations. +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. + .. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b5f5523d368964..f734221a2cdec5 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -284,7 +284,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -309,7 +309,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -454,7 +454,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -485,9 +485,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -576,7 +576,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -695,7 +695,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -759,7 +759,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -892,7 +892,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -904,9 +904,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1905,10 +1909,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1929,6 +1935,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, From a4d1fdfb152c46e3e05aa6e91a44a9fd0323b632 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 26 Sep 2024 12:58:15 -0700 Subject: [PATCH 15/66] gh-124612: Good bye dockerfile and use GHCR package (gh-124626) --- .devcontainer/Dockerfile | 24 ------------------------ .devcontainer/devcontainer.json | 4 +--- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ada5fb0fe64dc2..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:40 - -ENV CC=clang - -ENV WASI_SDK_VERSION=24 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=22.0.0 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", From 2c108328877984f2d92604764c66ef15bd82b31e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 26 Sep 2024 13:49:48 -0700 Subject: [PATCH 16/66] gh-119180: Rename SOURCE format to STRING (#124620) --- Doc/library/annotationlib.rst | 22 ++-- Doc/library/typing.rst | 2 +- Doc/whatsnew/3.14.rst | 4 +- Lib/_collections_abc.py | 6 +- Lib/annotationlib.py | 52 +++++---- Lib/test/test_annotationlib.py | 185 +++++++++++++----------------- Lib/test/test_type_annotations.py | 2 +- Lib/test/test_type_params.py | 4 +- Lib/test/test_typing.py | 22 ++-- Lib/typing.py | 22 ++-- Objects/typevarobject.c | 2 +- 11 files changed, 154 insertions(+), 169 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 2219e37f6b0677..37490456d13312 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -32,7 +32,7 @@ This module supports retrieving annotations in three main formats for annotations that cannot be resolved, allowing you to inspect the annotations without evaluating them. This is useful when you need to work with annotations that may contain unresolved forward references. -* :attr:`~Format.SOURCE` returns the annotations as a string, similar +* :attr:`~Format.STRING` returns the annotations as a string, similar to how it would appear in the source file. This is useful for documentation generators that want to display annotations in a readable way. @@ -135,7 +135,7 @@ Classes values. Real objects may contain references to, :class:`ForwardRef` proxy objects. - .. attribute:: SOURCE + .. attribute:: STRING :value: 3 Values are the text string of the annotation as it appears in the @@ -197,23 +197,23 @@ Classes Functions --------- -.. function:: annotations_to_source(annotations) +.. function:: annotations_to_string(annotations) Convert an annotations dict containing runtime values to a dict containing only strings. If the values are not already strings, - they are converted using :func:`value_to_source`. + they are converted using :func:`value_to_string`. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. - For example, this is used to implement the :attr:`~Format.SOURCE` for + For example, this is used to implement the :attr:`~Format.STRING` for :class:`typing.TypedDict` classes created through the functional syntax: .. doctest:: >>> from typing import TypedDict >>> Movie = TypedDict("movie", {"name": str, "year": int}) - >>> get_annotations(Movie, format=Format.SOURCE) + >>> get_annotations(Movie, format=Format.STRING) {'name': 'str', 'year': 'int'} .. versionadded:: 3.14 @@ -282,7 +282,7 @@ Functions NameError: name 'undefined' is not defined >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) ForwardRef('undefined') - >>> call_evaluate_function(Alias.evaluate_value, Format.SOURCE) + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) 'undefined' .. versionadded:: 3.14 @@ -369,14 +369,14 @@ Functions .. versionadded:: 3.14 -.. function:: value_to_source(value) +.. function:: value_to_string(value) Convert an arbitrary Python value to a format suitable for use by the - :attr:`~Format.SOURCE` format. This calls :func:`repr` for most + :attr:`~Format.STRING` format. This calls :func:`repr` for most objects, but has special handling for some objects, such as type objects. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. It can also be used to provide a user-friendly string representation for other objects that contain values that are commonly encountered in annotations. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c08b10d67bb031..640bc2c9d503bc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3427,7 +3427,7 @@ Introspection helpers * Replaces type hints that evaluate to :const:`!None` with :class:`types.NoneType`. * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.SOURCE` formats. + :attr:`~annotationlib.Format.STRING` formats. *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. *owner*, if given, should be the object that holds the annotations that diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3d6084e6ecc19b..5b9b01860045fd 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -91,7 +91,7 @@ annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VA format (which evaluates annotations to runtime values, similar to the behavior in earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format (which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.SOURCE` format (which returns annotations as strings). +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). This example shows how these formats behave: @@ -106,7 +106,7 @@ This example shows how these formats behave: NameError: name 'Undefined' is not defined >>> get_annotations(func, format=Format.FORWARDREF) {'arg': ForwardRef('Undefined')} - >>> get_annotations(func, format=Format.SOURCE) + >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} Implications for annotated code diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 4139cbadf93e13..c2edf6c8856c21 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,10 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() - from annotationlib import value_to_source + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([value_to_source(a) for a in self.__args__[:-1]])}], ' - f'{value_to_source(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a027f4de3dfed6..a11188722487b2 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -15,15 +15,15 @@ "call_evaluate_function", "get_annotate_function", "get_annotations", - "annotations_to_source", - "value_to_source", + "annotations_to_string", + "value_to_string", ] class Format(enum.IntEnum): VALUE = 1 FORWARDREF = 2 - SOURCE = 3 + STRING = 3 _Union = None @@ -291,9 +291,21 @@ def __convert_to_ast(self, other): return other.__ast_node__ elif isinstance(other, slice): return ast.Slice( - lower=self.__convert_to_ast(other.start) if other.start is not None else None, - upper=self.__convert_to_ast(other.stop) if other.stop is not None else None, - step=self.__convert_to_ast(other.step) if other.step is not None else None, + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), ) else: return ast.Constant(value=other) @@ -469,7 +481,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): can be called with any of the format arguments in the Format enum, but compiler-generated __annotate__ functions only support the VALUE format. This function provides additional functionality to call __annotate__ - functions with the FORWARDREF and SOURCE formats. + functions with the FORWARDREF and STRING formats. *annotate* must be an __annotate__ function, which takes a single argument and returns a dict of annotations. @@ -487,8 +499,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): return annotate(format) except NotImplementedError: pass - if format == Format.SOURCE: - # SOURCE is implemented by calling the annotate function in a special + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special # environment where every name lookup results in an instance of _Stringifier. # _Stringifier supports every dunder operation and returns a new _Stringifier. # At the end, we get a dictionary that mostly contains _Stringifier objects (or @@ -524,9 +536,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): for key, val in annos.items() } elif format == Format.FORWARDREF: - # FORWARDREF is implemented similarly to SOURCE, but there are two changes, + # FORWARDREF is implemented similarly to STRING, but there are two changes, # at the beginning and the end of the process. - # First, while SOURCE uses an empty dictionary as the namespace, so that all + # First, while STRING uses an empty dictionary as the namespace, so that all # name lookups result in _Stringifier objects, FORWARDREF uses the globals # and builtins, so that defined names map to their real values. # Second, instead of returning strings, we want to return either real values @@ -688,14 +700,14 @@ def get_annotations( # __annotations__ threw NameError and there is no __annotate__. In that case, # we fall back to trying __annotations__ again. return dict(_get_dunder_annotations(obj)) - case Format.SOURCE: - # For SOURCE, we try to call __annotate__ + case Format.STRING: + # For STRING, we try to call __annotate__ ann = _get_and_call_annotate(obj, format) if ann is not None: return ann # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) - return annotations_to_source(ann) + return annotations_to_string(ann) case _: raise ValueError(f"Unsupported format {format!r}") @@ -764,10 +776,10 @@ def get_annotations( return return_value -def value_to_source(value): - """Convert a Python value to a format suitable for use with the SOURCE format. +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. - This is inteded as a helper for tools that support the SOURCE format but do + This is inteded as a helper for tools that support the STRING format but do not have access to the code that originally produced the annotations. It uses repr() for most objects. @@ -783,10 +795,10 @@ def value_to_source(value): return repr(value) -def annotations_to_source(annotations): - """Convert an annotation dict containing values to approximately the SOURCE format.""" +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" return { - n: t if isinstance(t, str) else value_to_source(t) + n: t if isinstance(t, str) else value_to_string(t) for n, t in annotations.items() } diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dc1106aee1e2f1..eedf2506a14912 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -12,8 +12,8 @@ ForwardRef, get_annotations, get_annotate_function, - annotations_to_source, - value_to_source, + annotations_to_string, + value_to_string, ) from typing import Unpack @@ -39,14 +39,14 @@ def __repr__(self): class TestFormat(unittest.TestCase): def test_enum(self): - self.assertEqual(annotationlib.Format.VALUE.value, 1) - self.assertEqual(annotationlib.Format.VALUE, 1) + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) - self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) - self.assertEqual(annotationlib.Format.FORWARDREF, 2) + self.assertEqual(Format.FORWARDREF.value, 2) + self.assertEqual(Format.FORWARDREF, 2) - self.assertEqual(annotationlib.Format.SOURCE.value, 3) - self.assertEqual(annotationlib.Format.SOURCE, 3) + self.assertEqual(Format.STRING.value, 3) + self.assertEqual(Format.STRING, 3) class TestForwardRefFormat(unittest.TestCase): @@ -54,9 +54,7 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) fwdref = anno["arg"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) self.assertEqual(fwdref.__forward_arg__, "x") @@ -66,16 +64,14 @@ def inner(arg: x): x = 1 self.assertEqual(fwdref.evaluate(), x) - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) self.assertIs(anno["x"], int) fwdref = anno["y"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) @@ -92,14 +88,14 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations(inner, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(inner, format=Format.STRING) self.assertEqual(anno, {"arg": "x"}) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) def test_expressions(self): @@ -133,7 +129,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -184,7 +180,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -218,7 +214,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -241,13 +237,13 @@ def f(fstring: f"{a}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) def f(fstring_format: f"{a:02d}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) class TestForwardRefClass(unittest.TestCase): @@ -276,7 +272,7 @@ class Gen[T]: with self.assertRaises(NameError): ForwardRef("T").evaluate(owner=int) - T, = Gen.__type_params__ + (T,) = Gen.__type_params__ self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) @@ -294,8 +290,7 @@ class Gen[T]: def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( - ForwardRef("Counter", module="collections").evaluate(), - collections.Counter + ForwardRef("Counter", module="collections").evaluate(), collections.Counter ) self.assertEqual( ForwardRef("Counter[int]", module="collections").evaluate(), @@ -383,22 +378,20 @@ class C1(metaclass=NoDict): self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(C1, format=Format.FORWARDREF), {"a": int}, ) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(C1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) self.assertEqual( - annotationlib.get_annotations( - NoDict, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), {"b": str}, ) self.assertEqual( - annotationlib.get_annotations(NoDict, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(NoDict, format=Format.STRING), {"b": "str"}, ) @@ -410,20 +403,20 @@ def f2(a: undefined): pass self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.VALUE), + annotationlib.get_annotations(f1, format=Format.VALUE), {"a": int}, ) self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) fwd = annotationlib.ForwardRef("undefined") self.assertEqual( - annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f2, format=Format.FORWARDREF), {"a": fwd}, ) self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(f1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) @@ -446,30 +439,26 @@ def foo(): pass with self.assertRaises(ValueError): - annotationlib.get_annotations( - foo, format=annotationlib.Format.FORWARDREF, eval_str=True - ) - annotationlib.get_annotations( - foo, format=annotationlib.Format.SOURCE, eval_str=True - ) + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) def test_stock_annotations(self): def foo(a: int, b: str): pass - for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + for format in (Format.VALUE, Format.FORWARDREF): with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), {"a": int, "b": str}, ) self.assertEqual( - annotationlib.get_annotations(foo, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(foo, format=Format.STRING), {"a": "int", "b": "str"}, ) foo.__annotations__ = {"a": "foo", "b": "str"} - for format in annotationlib.Format: + for format in Format: with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -491,10 +480,10 @@ def test_stock_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -529,7 +518,7 @@ def test_stock_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -563,48 +552,36 @@ def test_stock_annotations_in_module(self): ) self.assertEqual( - annotationlib.get_annotations(isa, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(isa, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.MyClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function2, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function2, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function3, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function3, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - annotationlib, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(annotationlib, format=Format.STRING), {}, ) self.assertEqual( - annotationlib.get_annotations( - isa.UnannotatedClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), {}, ) self.assertEqual( annotationlib.get_annotations( - isa.unannotated_function, format=annotationlib.Format.SOURCE + isa.unannotated_function, format=Format.STRING ), {}, ) @@ -620,13 +597,11 @@ def test_stock_annotations_on_wrapper(self): {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations( - wrapped, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations(wrapped, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(wrapped, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( @@ -643,12 +618,12 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.SOURCE}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, - {"format": annotationlib.Format.SOURCE, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -681,7 +656,7 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -767,9 +742,9 @@ def f(x: int): annotationlib.get_annotations(f, format=Format.FORWARDREF), {"x": str}, ) - # ... but not in SOURCE which always uses __annotate__ + # ... but not in STRING which always uses __annotate__ self.assertEqual( - annotationlib.get_annotations(f, format=Format.SOURCE), + annotationlib.get_annotations(f, format=Format.STRING), {"x": "int"}, ) @@ -804,7 +779,7 @@ def __annotations__(self): ) self.assertEqual( - annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": "int"} + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} ) def test_raising_annotations_on_custom_object(self): @@ -844,7 +819,7 @@ def __annotate__(self): annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} ) self.assertEqual( - annotationlib.get_annotations(hb, format=Format.SOURCE), {"x": str} + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} ) def test_pep695_generic_class_with_future_annotations(self): @@ -974,15 +949,13 @@ def evaluate(format, exc=NotImplementedError): return undefined with self.assertRaises(NameError): - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.VALUE) + annotationlib.call_evaluate_function(evaluate, Format.VALUE) self.assertEqual( - annotationlib.call_evaluate_function( - evaluate, annotationlib.Format.FORWARDREF - ), + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), annotationlib.ForwardRef("undefined"), ) self.assertEqual( - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.SOURCE), + annotationlib.call_evaluate_function(evaluate, Format.STRING), "undefined", ) @@ -1093,25 +1066,25 @@ class C: class TestToSource(unittest.TestCase): - def test_value_to_source(self): - self.assertEqual(value_to_source(int), "int") - self.assertEqual(value_to_source(MyClass), "test.test_annotationlib.MyClass") - self.assertEqual(value_to_source(len), "len") - self.assertEqual(value_to_source(value_to_source), "value_to_source") - self.assertEqual(value_to_source(times_three), "times_three") - self.assertEqual(value_to_source(...), "...") - self.assertEqual(value_to_source(None), "None") - self.assertEqual(value_to_source(1), "1") - self.assertEqual(value_to_source("1"), "'1'") - self.assertEqual(value_to_source(Format.VALUE), repr(Format.VALUE)) - self.assertEqual(value_to_source(MyClass()), "my repr") - - def test_annotations_to_source(self): - self.assertEqual(annotations_to_source({}), {}) - self.assertEqual(annotations_to_source({"x": int}), {"x": "int"}) - self.assertEqual(annotations_to_source({"x": "int"}), {"x": "int"}) + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) self.assertEqual( - annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"} + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} ) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 91082e6b23c04b..257b7fa95dcb76 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -375,7 +375,7 @@ class X: with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): - annotate(annotationlib.Format.SOURCE) + annotate(annotationlib.Format.STRING) with self.assertRaises(NotImplementedError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 8c21553e410d8a..433b19593bdd04 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1440,7 +1440,7 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(case(1), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) - self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.SOURCE), 'int') + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') def test_constraints(self): def f[T: (int, str)](): pass @@ -1451,7 +1451,7 @@ def f[T: (int, str)](): pass self.assertEqual(case.evaluate_constraints(1), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) - self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.SOURCE), '(int, str)') + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') def test_const_evaluator(self): T = TypeVar("T", bound=int) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3ac6b97383fcef..2f1f9e86a0bce4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7059,7 +7059,7 @@ class C: self.assertIsInstance(annos['x'], annotationlib.ForwardRef) self.assertEqual(annos['x'].__arg__, 'undefined') - self.assertEqual(get_type_hints(C, format=annotationlib.Format.SOURCE), + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), {'x': 'undefined'}) @@ -7898,7 +7898,7 @@ class Z(NamedTuple): self.assertEqual(Z.__annotations__, annos) self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Z.__annotate__(annotationlib.Format.SOURCE), {"a": "None", "b": "str"}) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) def test_future_annotations(self): code = """ @@ -8241,7 +8241,7 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__annotations__, annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Emp.__annotate__(annotationlib.Format.SOURCE), {'name': 'str', 'id': 'int'}) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8603,7 +8603,7 @@ class A[T](TypedDict): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8616,7 +8616,7 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8628,7 +8628,7 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) self.assertEqual(A2.__annotations__, {'a': T}) - self.assertEqual(A2.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8640,7 +8640,7 @@ class B(A[KT], total=False): self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) - self.assertEqual(B.__annotate__(annotationlib.Format.SOURCE), {'a': 'T', 'b': 'KT'}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8665,7 +8665,7 @@ class C(B[int]): 'b': KT, 'c': int, }) - self.assertEqual(C.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8689,7 +8689,7 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) - self.assertEqual(Point3D.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'T', 'c': 'KT', @@ -8725,7 +8725,7 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) - self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8929,7 +8929,7 @@ class A(TypedDict): A.__annotations__ self.assertEqual( - A.__annotate__(annotationlib.Format.SOURCE), + A.__annotate__(annotationlib.Format.STRING), {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', 'z': 'Required[undefined]'}, ) diff --git a/Lib/typing.py b/Lib/typing.py index 252eef32cd88a4..c924c767042552 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -245,7 +245,7 @@ def _type_repr(obj): if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return annotationlib.value_to_source(obj) + return annotationlib.value_to_string(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -1036,7 +1036,7 @@ def evaluate_forward_ref( * Recursively evaluates forward references nested within the type hint. * Rejects certain objects that are not valid type hints. * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *SOURCE* formats. + * Supports the *FORWARDREF* and *STRING* formats. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -1053,7 +1053,7 @@ def evaluate_forward_ref( if type_params is _sentinel: _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") type_params = () - if format == annotationlib.Format.SOURCE: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref @@ -2380,7 +2380,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, hints = {} for base in reversed(obj.__mro__): ann = annotationlib.get_annotations(base, format=format) - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: hints.update(ann) continue if globalns is None: @@ -2404,7 +2404,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, value = _eval_type(value, base_globals, base_locals, base.__type_params__, format=format, owner=obj) hints[name] = value - if include_extras or format is annotationlib.Format.SOURCE: + if include_extras or format is annotationlib.Format.STRING: return hints else: return {k: _strip_annotations(t) for k, t in hints.items()} @@ -2418,7 +2418,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, and not hasattr(obj, '__annotate__') ): raise TypeError(f"{obj!r} is not a module, class, or callable.") - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: return hints if globalns is None: @@ -2937,7 +2937,7 @@ def annotate(format): if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): return checked_types else: - return annotationlib.annotations_to_source(types) + return annotationlib.annotations_to_string(types) return annotate @@ -2972,7 +2972,7 @@ def __new__(cls, typename, bases, ns): def annotate(format): annos = annotationlib.call_annotate_function(original_annotate, format) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3220,13 +3220,13 @@ def __annotate__(format): annos.update(base_annos) if own_annotate is not None: own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == annotationlib.Format.SOURCE: - own = annotationlib.annotations_to_source(own_annotations) + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) else: own = own_checked_annotations annos.update(own) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 552c4745590194..51d93ed8b5ba8c 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // SOURCE + if (format == 3) { // STRING PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; From 5e7eba09bcdafe361b491b3f8cf30d7dd2df0a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:15:28 +0200 Subject: [PATCH 17/66] gh-89683: add tests for `deepcopy` on frozen dataclasses (gh-123098) Co-authored-by: Eric V. Smith --- Lib/test/test_dataclasses/__init__.py | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 69e86162e0c11a..bd2f87819a8eb0 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,6 +17,7 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict +from copy import deepcopy from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. @@ -3175,6 +3176,48 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): def test_simple(self): From 66cc6d4c502074ddbfeda1be28fae6aa4535e4a8 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 26 Sep 2024 14:23:41 -0700 Subject: [PATCH 18/66] Remove some unused files related to WASM/WASI (GH-124635) --- Tools/wasm/Setup.local.example | 13 ---------- Tools/wasm/build_wasi.sh | 44 ---------------------------------- Tools/wasm/wasi-env | 2 ++ 3 files changed, 2 insertions(+), 57 deletions(-) delete mode 100644 Tools/wasm/Setup.local.example delete mode 100755 Tools/wasm/build_wasi.sh diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 95eda863cb62c6..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK" From 83e5dc0f4d0d8d71288f162840b36f210fb03abf Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 26 Sep 2024 15:10:36 -0700 Subject: [PATCH 19/66] gh-124628: Pyrepl inputs on Windows shouldn't always be blocking reads (#124629) --- Lib/_pyrepl/windows_console.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f7a0095d795ac6..d457d2b5a338eb 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -371,15 +371,19 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self) -> INPUT_RECORD | None: + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): raise WinError(GetLastError()) - if read.value == 0: - return None - return rec def get_event(self, block: bool = True) -> Event | None: @@ -390,10 +394,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.pop() while True: - rec = self._read_input() + rec = self._read_input(block) if rec is None: - if block: - continue return None if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: @@ -464,8 +466,8 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" - while self._read_input() is not None: - pass + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -590,6 +592,14 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -602,5 +612,7 @@ def _win_only(*args, **kwargs): ScrollConsoleScreenBuffer = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only OutHandle = 0 InHandle = 0 From b65f2cdfa77d8d12c213aec663ddaaa30d75a4b2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 26 Sep 2024 16:57:19 -0700 Subject: [PATCH 20/66] gh-84559: Change the multiprocessing start method default to `forkserver` (GH-101556) Change the default multiprocessing start method away from fork to forkserver or spawn on the remaining platforms where it was fork. See the issue for context. This makes the default far more thread safe (other than for people spawning threads at import time... - don't do that!). Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/concurrent.futures.rst | 14 +++++----- Doc/library/multiprocessing.rst | 21 ++++++++++----- Doc/whatsnew/3.14.rst | 8 ++++++ Lib/multiprocessing/context.py | 26 +++++++++---------- Lib/test/_test_multiprocessing.py | 24 +++++++++++++---- Lib/test/support/__init__.py | 10 ++++++- ...4-09-19-00-09-48.gh-issue-84559.IrxvQe.rst | 5 ++++ 7 files changed, 75 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e3b24451188cc4..ce72127127c7a6 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -286,14 +286,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +302,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 80d6e4dae24463..036b8f44b9ff3b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,11 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for @@ -162,6 +164,13 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5b9b01860045fd..6875c4c909b3c7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -385,6 +385,14 @@ Deprecated as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`multiprocessing` and :mod:`concurrent.futures`: + The default start method (see :ref:`multiprocessing-start-methods`) changed + away from *fork* to *forkserver* on platforms where it was not already + *spawn* (Windows & macOS). If you require the threading incompatible *fork* + start method you must explicitly specify it when using :mod:`multiprocessing` + or :mod:`concurrent.futures` APIs. + (Contributed by Gregory P. Smith in :gh:`84559`.) + * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index ddcc7e7900999e..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b3a0645cfc84a..a059a6b8340448 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5553,15 +5553,29 @@ def test_set_get(self): multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 628529b8664c77..99cb10fc7b5f7b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2209,7 +2209,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") diff --git a/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst new file mode 100644 index 00000000000000..a4428e20f3ccdd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst @@ -0,0 +1,5 @@ +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to ``"forkserver"`` +(when the platform supports sending file handles over pipes as most do) or +``"spawn"``. Mac and Windows are unchanged as they already default to +``"spawn"``. From 23e812b84ae688a56a1011ed69a0d178c70e35ea Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:57:38 -0700 Subject: [PATCH 21/66] Docs: Update and proofread `library/venv.rst` (#124121) Co-authored-by: C.A.M. Gerlach Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/venv.rst | 142 +++++++++++++++++++++++++++++++++++--- Doc/using/venv-create.inc | 121 -------------------------------- 2 files changed, 131 insertions(+), 132 deletions(-) delete mode 100644 Doc/using/venv-create.inc diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index cf6c5437be4fd1..c0edb3f2705d8a 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -37,14 +37,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -61,7 +61,127 @@ See :pep:`405` for more background on Python virtual environments. Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -117,7 +237,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -345,8 +465,8 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: install_scripts(context, path) *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,7 +491,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 @@ -387,16 +507,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 354eb1541ceac2..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. From 65f12370982b9982b204d07f9f26ca8740f21845 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 26 Sep 2024 17:35:42 -0700 Subject: [PATCH 22/66] GH-123516: Improve JIT memory consumption by invalidating cold executors (GH-124443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Include/internal/pycore_ceval.h | 1 + Include/internal/pycore_interp.h | 2 +- Include/internal/pycore_optimizer.h | 14 +++- Include/internal/pycore_uop_ids.h | 71 ++++++++++--------- Include/internal/pycore_uop_metadata.h | 4 ++ ...-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst | 2 + Python/bytecodes.c | 8 +++ Python/ceval_gil.c | 6 ++ Python/executor_cases.c.h | 9 +++ Python/optimizer.c | 42 +++++++++++ Python/optimizer_cases.c.h | 4 ++ Python/pystate.c | 1 + Tools/cases_generator/analyzer.py | 1 + Tools/jit/_targets.py | 3 + 14 files changed, 129 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a97b53028c8f59..363845106e40dc 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -283,6 +283,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra #define _PY_GC_SCHEDULED_BIT (1U << 4) #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 36366429e8db25..a1898d926ac39f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -261,7 +261,7 @@ struct _is { struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 19e54bf122a8bb..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -29,9 +29,10 @@ typedef struct { typedef struct { uint8_t opcode; uint8_t oparg; - uint16_t valid:1; - uint16_t linked:1; - uint16_t chain_depth:14; // Must be big engough for MAX_CHAIN_DEPTH - 1. + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -123,11 +124,18 @@ PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #ifdef _Py_TIER2 PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + #endif +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 800 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b950f760d74ac7..927dae88c1fa73 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -222,64 +222,65 @@ extern "C" { #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 439 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 439 -#define _MONITOR_CALL 440 -#define _MONITOR_JUMP_BACKWARD 441 -#define _MONITOR_RESUME 442 +#define _MAYBE_EXPAND_METHOD 440 +#define _MONITOR_CALL 441 +#define _MONITOR_JUMP_BACKWARD 442 +#define _MONITOR_RESUME 443 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 443 -#define _POP_JUMP_IF_TRUE 444 +#define _POP_JUMP_IF_FALSE 444 +#define _POP_JUMP_IF_TRUE 445 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 445 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 446 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 446 +#define _PUSH_FRAME 447 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 447 -#define _PY_FRAME_KW 448 -#define _QUICKEN_RESUME 449 -#define _REPLACE_WITH_TRUE 450 +#define _PY_FRAME_GENERAL 448 +#define _PY_FRAME_KW 449 +#define _QUICKEN_RESUME 450 +#define _REPLACE_WITH_TRUE 451 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 451 -#define _SEND 452 -#define _SEND_GEN_FRAME 453 +#define _SAVE_RETURN_OFFSET 452 +#define _SEND 453 +#define _SEND_GEN_FRAME 454 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 454 -#define _STORE_ATTR 455 -#define _STORE_ATTR_INSTANCE_VALUE 456 -#define _STORE_ATTR_SLOT 457 -#define _STORE_ATTR_WITH_HINT 458 +#define _START_EXECUTOR 455 +#define _STORE_ATTR 456 +#define _STORE_ATTR_INSTANCE_VALUE 457 +#define _STORE_ATTR_SLOT 458 +#define _STORE_ATTR_WITH_HINT 459 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 459 -#define _STORE_FAST_0 460 -#define _STORE_FAST_1 461 -#define _STORE_FAST_2 462 -#define _STORE_FAST_3 463 -#define _STORE_FAST_4 464 -#define _STORE_FAST_5 465 -#define _STORE_FAST_6 466 -#define _STORE_FAST_7 467 +#define _STORE_FAST 460 +#define _STORE_FAST_0 461 +#define _STORE_FAST_1 462 +#define _STORE_FAST_2 463 +#define _STORE_FAST_3 464 +#define _STORE_FAST_4 465 +#define _STORE_FAST_5 466 +#define _STORE_FAST_6 467 +#define _STORE_FAST_7 468 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 468 -#define _STORE_SUBSCR 469 +#define _STORE_SLICE 469 +#define _STORE_SUBSCR 470 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 470 -#define _TO_BOOL 471 +#define _TIER2_RESUME_CHECK 471 +#define _TO_BOOL 472 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -289,14 +290,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 472 +#define _UNPACK_SEQUENCE 473 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 472 +#define MAX_UOP_ID 473 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4d0ab22e6aa8f3..07606135d7a356 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -274,6 +274,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, + [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, @@ -481,6 +482,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", @@ -1062,6 +1064,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _START_EXECUTOR: return 0; + case _MAKE_WARM: + return 0; case _FATAL_ERROR: return 0; case _CHECK_VALIDITY_AND_SET_IP: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst new file mode 100644 index 00000000000000..de62875e16475d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst @@ -0,0 +1,2 @@ +Improved JIT memory consumption by periodically freeing memory used by infrequently-executed code. +This change is especially likely to improve the memory footprint of long-running programs. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0fd396f1319e78..8535306d9c7a03 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4836,6 +4836,14 @@ dummy_func( assert(((_PyExecutorObject *)executor)->vm_data.valid); } + tier2 op(_MAKE_WARM, (--)) { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + } + tier2 op(_FATAL_ERROR, (--)) { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6f4476d055b5ec..1d9381d09dfb62 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1289,6 +1289,12 @@ _Py_HandlePending(PyThreadState *tstate) _Py_RunGC(tstate); } + if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + _Py_Executors_InvalidateCold(tstate->interp); + tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + } + /* GIL drop request */ if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7a9c6ab89c38cc..650bf4533a3a86 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5435,6 +5435,15 @@ break; } + case _MAKE_WARM: { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + break; + } + case _FATAL_ERROR: { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/optimizer.c b/Python/optimizer.c index bb7a90b3204f40..978649faa04d45 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -565,6 +565,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; for (;;) { @@ -1194,6 +1195,9 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil executor->jit_code = NULL; executor->jit_side_entry = NULL; executor->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + executor->vm_data.warm = true; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); return NULL; @@ -1659,4 +1663,42 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) } } +void +_Py_Executors_InvalidateCold(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + + if (!exec->vm_data.warm && PyList_Append(invalidate, (PyObject *)exec) < 0) { + goto error; + } + else { + exec->vm_data.warm = false; + } + + exec = next; + } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback + _Py_Executors_InvalidateAll(interp, 0); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a6cfa271ae6758..4d172e3c762704 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2381,6 +2381,10 @@ break; } + case _MAKE_WARM: { + break; + } + case _FATAL_ERROR: { break; } diff --git a/Python/pystate.c b/Python/pystate.c index 6bf7ebeb75ff73..6b85e5a64fefcf 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -660,6 +660,7 @@ init_interpreter(PyInterpreterState *interp, #ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; + interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; #endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index aabe205125856c..a4ce207703edcd 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -540,6 +540,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyList_FromStackRefSteal", "_PyTuple_FromArraySteal", "_PyTuple_FromStackRefSteal", + "_Py_set_eval_breaker_bit" ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index e37ee943999785..6c7b48f1f37865 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -139,6 +139,9 @@ async def _compile( "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", + # On aarch64 Linux, intrinsics were being emitted and this flag + # was required to disable them. + "-mno-outline-atomics", "-std=c11", *self.args, ] From 98b2ed7e239c807f379cd2bf864f372d79064aac Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 26 Sep 2024 19:16:51 -0700 Subject: [PATCH 23/66] gh-116510: Fix crash due to shared immortal interned strings. (gh-124646) --- ...-09-26-18-21-06.gh-issue-116510.FacUWO.rst | 5 ++ Objects/unicodeobject.c | 48 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst new file mode 100644 index 00000000000000..e3741321006548 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst @@ -0,0 +1,5 @@ +Fix a crash caused by immortal interned strings being shared between +sub-interpreters that use basic single-phase init. In that case, the string +can be used by an interpreter that outlives the interpreter that created and +interned it. For interpreters that share obmalloc state, also share the +interned dict with the main interpreter. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e9589cfe44f3bf..0f502ccdaf5767 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -282,13 +282,37 @@ hashtable_unicode_compare(const void *key1, const void *key2) } } +/* Return true if this interpreter should share the main interpreter's + intern_dict. That's important for interpreters which load basic + single-phase init extension modules (m_size == -1). There could be interned + immortal strings that are shared between interpreters, due to the + PyDict_Update(mdict, m_copy) call in import_find_extension(). + + It's not safe to deallocate those strings until all interpreters that + potentially use them are freed. By storing them in the main interpreter, we + ensure they get freed after all other interpreters are freed. +*/ +static bool +has_shared_intern_dict(PyInterpreterState *interp) +{ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + return interp != main_interp && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC; +} + static int init_interned_dict(PyInterpreterState *interp) { assert(get_interned_dict(interp) == NULL); - PyObject *interned = interned = PyDict_New(); - if (interned == NULL) { - return -1; + PyObject *interned; + if (has_shared_intern_dict(interp)) { + interned = get_interned_dict(_PyInterpreterState_Main()); + Py_INCREF(interned); + } + else { + interned = PyDict_New(); + if (interned == NULL) { + return -1; + } } _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = interned; return 0; @@ -299,7 +323,10 @@ clear_interned_dict(PyInterpreterState *interp) { PyObject *interned = get_interned_dict(interp); if (interned != NULL) { - PyDict_Clear(interned); + if (!has_shared_intern_dict(interp)) { + // only clear if the dict belongs to this interpreter + PyDict_Clear(interned); + } Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } @@ -15618,6 +15645,13 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); + if (has_shared_intern_dict(interp)) { + // the dict doesn't belong to this interpreter, skip the debug + // checks on it and just clear the pointer to it + clear_interned_dict(interp); + return; + } + #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -16126,8 +16160,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) { struct _Py_unicode_state *state = &interp->unicode; - // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() - assert(get_interned_dict(interp) == NULL); + if (!has_shared_intern_dict(interp)) { + // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() + assert(get_interned_dict(interp) == NULL); + } _PyUnicode_FiniEncodings(&state->fs_codec); From 6f9525dd3f0ef5809106ca0923a7512d666a04bb Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 26 Sep 2024 19:33:07 -0700 Subject: [PATCH 24/66] gh-116510: Fix crash during sub-interpreter shutdown (gh-124645) Fix a bug that can cause a crash when sub-interpreters use "basic" single-phase extension modules. Shared objects could refer to PyGC_Head nodes that had been freed as part of interpreter shutdown. --- ...-09-26-17-55-34.gh-issue-116510.dhn8w8.rst | 3 ++ Python/gc.c | 29 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst new file mode 100644 index 00000000000000..fc3f8af72d87bf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst @@ -0,0 +1,3 @@ +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. diff --git a/Python/gc.c b/Python/gc.c index 024d041437be4a..028657eb8999c1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1944,6 +1944,13 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } +static void +finalize_unlink_gc_head(PyGC_Head *gc) { + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); +} void _PyGC_Fini(PyInterpreterState *interp) @@ -1952,9 +1959,25 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ + /* Prevent a subtle bug that affects sub-interpreters that use basic + * single-phase init extensions (m_size == -1). Those extensions cause objects + * to be shared between interpreters, via the PyDict_Update(mdict, m_copy) call + * in import_find_extension(). + * + * If they are GC objects, their GC head next or prev links could refer to + * the interpreter _gc_runtime_state PyGC_Head nodes. Those nodes go away + * when the interpreter structure is freed and so pointers to them become + * invalid. If those objects are still used by another interpreter and + * UNTRACK is called on them, a crash will happen. We untrack the nodes + * here to avoid that. + * + * This bug was originally fixed when reported as gh-90228. The bug was + * re-introduced in gh-94673. + */ + finalize_unlink_gc_head(&gcstate->young.head); + finalize_unlink_gc_head(&gcstate->old[0].head); + finalize_unlink_gc_head(&gcstate->old[1].head); + finalize_unlink_gc_head(&gcstate->permanent_generation.head); } /* for debugging */ From 25189188bfcb48be653eb9fb3aee892f2b9539b9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 26 Sep 2024 21:37:21 -0700 Subject: [PATCH 25/66] generate_global_objects.py: Fix name of macro in error message (#124464) _PyID does not exist but _Py_ID does. --- Tools/build/generate_global_objects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 882918fafb1edd..b5b6de0e7dc2dc 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -433,7 +433,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - 'do not use &_PyID or &_Py_STR for one-character latin-1 ' + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name @@ -442,7 +442,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': overlap = identifiers & set(strings.keys()) if overlap: raise ValueError( - 'do not use both _PyID and _Py_DECLARE_STR for the same string: ' + 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: ' + repr(overlap)) return identifiers, strings From 08e1bbe4a329e5961716f030c6ccfe92c736bf28 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 27 Sep 2024 08:21:15 +0200 Subject: [PATCH 26/66] gh-86673: Harden `test_ttk.test_element_create_image` (#123335) Co-authored-by: Serhiy Storchaka --- Lib/test/test_ttk/test_style.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..eeaf5de2e303f6 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -227,13 +227,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) From 3a0e7f57628466aedcaaf6c5ff7c8224f5155a2c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 27 Sep 2024 09:48:31 +0300 Subject: [PATCH 27/66] gh-124176: Add special support for dataclasses to `create_autospec` (#124429) --- .../test_unittest/testmock/testhelpers.py | 72 +++++++++++++++++++ Lib/unittest/mock.py | 22 ++++-- ...-09-24-13-32-16.gh-issue-124176.6hmOPz.rst | 4 ++ 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index c9c20f008ca5a2..f260769eb8c35e 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index df3901f9660ac1..21ca061a77c26f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -34,6 +34,7 @@ import pkgutil from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -2756,7 +2757,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2813,7 +2822,7 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _name='()', _parent=mock, wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2827,10 +2836,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue child_kwargs = {'spec': original} # Wrap child attributes also. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst new file mode 100644 index 00000000000000..38c030668b6b42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst @@ -0,0 +1,4 @@ +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. From 5329d1b74a86b3a22ff36f7976bfe720ee06d10d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 27 Sep 2024 06:17:25 -0400 Subject: [PATCH 28/66] Docs: for for/else clarify that return or raise also skip the else (#124591) Co-authored-by: Jelle Zijlstra --- Doc/tutorial/controlflow.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index c97c65f7a3988e..fd765e58ff2485 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -209,8 +209,10 @@ after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: From b79a21ea429844e84509430e636d808ea9cff244 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Fri, 27 Sep 2024 12:35:55 +0200 Subject: [PATCH 29/66] GH-95079: document error behaviour for some unicode C APIs (#95080) --- Doc/c-api/unicode.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 30e26fe52178d7..646c1b07222561 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -614,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -657,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -666,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -674,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 From 365dffbaada421db8fdb684a84d1fb311b75ec40 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 27 Sep 2024 05:49:43 -0700 Subject: [PATCH 30/66] gh-119180: No longer set `__annotations__` in `__main__` (#124634) --- Lib/test/test__interpreters.py | 1 - Lib/test/test_pyrepl/test_pyrepl.py | 2 +- .../2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst | 2 ++ Python/pylifecycle.c | 8 +------- 4 files changed, 4 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f493a92e0ddce8..14cd50bd30502c 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -849,7 +849,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 0f3e9996e77e45..36f940eaea4eac 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1080,7 +1080,7 @@ def setUp(self): @force_not_colorized def test_exposed_globals_in_repl(self): - pre = "['__annotations__', '__builtins__'" + pre = "['__builtins__'" post = "'__loader__', '__name__', '__package__', '__spec__']" output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) if "can't use pyrepl" in output: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst new file mode 100644 index 00000000000000..4cdbb205c962c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst @@ -0,0 +1,2 @@ +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27faf723745c21..8aebbe5c405ffe 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2503,18 +2503,12 @@ finalize_subinterpreters(void) static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *ann_dict; + PyObject *m, *d; m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); d = PyModule_GetDict(m); - ann_dict = PyDict_New(); - if ((ann_dict == NULL) || - (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - return _PyStatus_ERR("Failed to initialize __main__.__annotations__"); - } - Py_DECREF(ann_dict); int has_builtins = PyDict_ContainsString(d, "__builtins__"); if (has_builtins < 0) { From 0a3577bdfcb7132c92a3f7fb2ac231bc346383c0 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 27 Sep 2024 15:35:18 +0100 Subject: [PATCH 31/66] gh-123017: Add Android to the list of platforms where `strftime` doesn't support negative years (#124467) Add Android to the list of platforms where `strftime` doesn't support negative years --- Lib/test/test_time.py | 3 +-- .../Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst | 2 ++ Modules/timemodule.c | 7 ++++++- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 293799ff68ea05..530c317a852e77 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -654,8 +654,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: diff --git a/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst new file mode 100644 index 00000000000000..45fe4786fa6563 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst @@ -0,0 +1,2 @@ +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ee59fb73ac1e31..9720c201a184a8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -813,7 +813,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); From 9c7657f09914254724683d91177aed7947637be5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 27 Sep 2024 19:20:49 +0300 Subject: [PATCH 32/66] gh-113878: Add `doc` parameter to `dataclasses.field` (gh-114051) If using `slots=True`, the `doc` parameter ends up in the `__slots__` dict. The `doc` parameter is also in the corresponding `Field` object. --- Doc/library/dataclasses.rst | 6 ++- Lib/dataclasses.py | 54 +++++++++++++------ Lib/test/test_dataclasses/__init__.py | 25 +++++++-- Lib/test/test_pydoc/test_pydoc.py | 8 +++ ...-01-14-11-43-31.gh-issue-113878.dmEIN3.rst | 9 ++++ 5 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 1457392ce6e86c..51c1a427b63787 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -231,7 +231,7 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -300,6 +300,10 @@ Module contents .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f5cb97edaf72cd..bdda7cc6c00f5d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -283,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -300,6 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -315,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -382,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -394,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -402,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -1174,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1239,7 +1242,32 @@ def _update_func_cell_for__class__(f, oldcls, newcls): return False -def _add_slots(cls, is_frozen, weakref_slot): +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # Need to create a new class, since we can't set __slots__ after a # class has been created, and the @dataclass decorator is called # after the class is created. @@ -1255,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index bd2f87819a8eb0..2984f4261bd2c4 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -61,7 +61,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -69,6 +69,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -3304,7 +3305,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3314,7 +3315,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3322,6 +3323,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2dba077cdea6a7..776e02f41a1cec 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -463,6 +463,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) diff --git a/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst new file mode 100644 index 00000000000000..8e1937ab73c31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst @@ -0,0 +1,9 @@ +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format +in dataclasses from :class:`tuple` to :class:`dict` +when documentation / metadata is present. From 26a74203f08c59922b4889931c108b34c4ee8e72 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 27 Sep 2024 09:38:04 -0700 Subject: [PATCH 33/66] GH-118093: Fix off-by-one errors in tier-up thresholds (GH-124447) --- Include/internal/pycore_backoff.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3db3aa3eb77879..a9d1bce127e63d 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -108,7 +108,7 @@ backoff_counter_triggers(_Py_BackoffCounter counter) /* Initial JUMP_BACKWARD counter. * This determines when we create a trace for a loop. * Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ -#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_VALUE 15 #define JUMP_BACKWARD_INITIAL_BACKOFF 4 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -122,7 +122,7 @@ initial_jump_backoff_counter(void) * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 64 +#define SIDE_EXIT_INITIAL_VALUE 63 #define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter From 4b89c5ebfc7d5d4f008eee0ae6da765dfc28e3a9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 27 Sep 2024 09:56:37 -0700 Subject: [PATCH 34/66] Improve accuracy of kde() invcdf estimates (gh-124637) --- Lib/statistics.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index d3dd0d530c31cf..f193fcdc241aa9 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -870,9 +870,12 @@ def f_inv(y): return f_inv def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: + if p < 0.499: x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) return x * sign @@ -886,8 +889,11 @@ def quartic_kernel(): return pdf, cdf, invcdf, support def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) return x * sign @register('triweight') From 6716dd1c33ae6fe43cf14a2a54be143b1de3fa64 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 27 Sep 2024 10:35:09 -0700 Subject: [PATCH 35/66] Fixup indentation for docs on `ModuleSpec` attributes (#124681) Co-authored-by: Jelle Zijlstra --- Doc/library/importlib.rst | 79 ++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e4cef1f3e3b7c0..27d31f66b12495 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1166,10 +1166,9 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is no synchronization between the two objects. For example, it is possible to update the module's :attr:`__file__` at runtime and this will not be automatically @@ -1179,66 +1178,60 @@ find and load modules. .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name + (see :attr:`__name__` attributes on modules). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module + (see :attr:`__loader__` attributes on modules). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`__file__` attributes on modules). + For example, for modules loaded from a .py file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) - - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The list of locations where the package's submodules will be found + (see :attr:`__path__` attributes on modules). + Most of the time this is a single directory. + The :term:`finder` should set this attribute to a list, even an empty one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`__cached__` attributes on modules). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`__package__` attributes on modules. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`origin` is interpreted + and how the module's :attr:`__file__` is populated. .. class:: AppleFrameworkLoader(name, path) From e349f73a5ad2856b0a7cbe4aef7cc081c7aed777 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 27 Sep 2024 19:38:40 +0200 Subject: [PATCH 36/66] gh-121277: Raise nice error on `next` as second argument to deprecated-removed (GH-124623) --- Doc/tools/extensions/pyspecific.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1f725c2377035b..c89b1693343b4e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -288,6 +288,9 @@ def run(self): version_deprecated = expand_version_arg(self.arguments[0], self.config.release) version_removed = self.arguments.pop(1) + if version_removed == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') self.arguments[0] = version_deprecated, version_removed # Set the label based on if we have reached the removal version From 10d504aecc56f9481114fe3d0a8d1721d38db7e3 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 27 Sep 2024 10:49:35 -0700 Subject: [PATCH 37/66] gh-124682: Disable test that is prone to intermittent failure on iOS. (#124683) Disable test that is prone to intermittent failure on iOS. --- Lib/test/test_support.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e60e5477d32e1f..9a3cf140d81241 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -548,13 +548,14 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) From 2e155536caf8a090c06d62dd92647abc62362463 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 27 Sep 2024 11:43:46 -0700 Subject: [PATCH 38/66] Itertool docs: Minor clarifications, wording tweaks, spacing, and active voice. (gh-124690) Minor clarifications, wording tweaks, spacing, and active voice. --- Doc/library/itertools.rst | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c1299ebfe8d27a..9a62249816c9bf 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -58,7 +58,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','ABC'], len) → (1, A B) (3, ABC)`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` @@ -93,7 +93,7 @@ Examples Results Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. @@ -131,11 +131,12 @@ loops that truncate the stream. total = function(total, element) yield total - The *function* argument can be set to :func:`min` for a running - minimum, :func:`max` for a running maximum, or :func:`operator.mul` - for a running product. `Amortization tables - `_ - can be built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `Amortization table + `_, + accumulate the interest and apply payments: .. doctest:: @@ -202,10 +203,10 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): # chain('ABC', 'DEF') → A B C D E F @@ -353,10 +354,12 @@ loops that truncate the stream. def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: yield element @@ -396,8 +399,10 @@ loops that truncate the stream. def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -474,7 +479,7 @@ loops that truncate the stream. If *start* is zero or ``None``, iteration starts at zero. Otherwise, elements from the iterable are skipped until *start* is reached. - If *stop* is ``None``, iteration continues until the iterable is + If *stop* is ``None``, iteration continues until the input is exhausted, if at all. Otherwise, it stops at the specified position. If *step* is ``None``, the step defaults to one. Elements are returned @@ -520,8 +525,10 @@ loops that truncate the stream. def pairwise(iterable): # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) a = next(iterator, None) + for b in iterator: yield a, b a = b @@ -584,7 +591,8 @@ loops that truncate the stream. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. From 0881e2d3b1212d988733f1d3acca4011ce5e6280 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 27 Sep 2024 19:52:23 +0100 Subject: [PATCH 39/66] gh-124609: Fix _Py_ThreadId for Windows builds using MinGW (#124663) --- Include/Python.h | 4 ++++ Include/object.h | 6 ++++++ Misc/ACKS | 1 + .../Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst | 1 + 4 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst diff --git a/Include/Python.h b/Include/Python.h index 8fffa22df9da48..e1abdd16f031fb 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -55,6 +55,10 @@ # include // __readgsqword() #endif +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" diff --git a/Include/object.h b/Include/object.h index 7124f58f6bdb37..418f2196062df7 100644 --- a/Include/object.h +++ b/Include/object.h @@ -180,6 +180,12 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) diff --git a/Misc/ACKS b/Misc/ACKS index b2529601a2f71a..d94cbacf888468 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1552,6 +1552,7 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson Izan "TizzySaurus" Robinson Jim Robinson diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst new file mode 100644 index 00000000000000..203868a8fee39c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst @@ -0,0 +1 @@ +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. From 81a253b929258f17e89adc1aeb2c2ccdbcdc2945 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 27 Sep 2024 21:31:59 +0200 Subject: [PATCH 40/66] Fix typo in InternalDocs/string_interning.md (GH-124699) --- InternalDocs/string_interning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 358e2c070cd5fa..e0d20632516142 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -72,7 +72,7 @@ We currently also immortalize strings contained in code objects and similar, specifically in the compiler and in `marshal`. These are “close enough” to immortal: even in use cases like hot reloading or `eval`-ing user input, the number of distinct identifiers and string -constants expected to stay low. +constants is expected to stay low. ## Internal API From 34158c2c7a80d7b1113beca69473e38349f0c96e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 27 Sep 2024 12:33:16 -0700 Subject: [PATCH 41/66] Drop code ownership for decimal (gh-124695) --- .github/CODEOWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 680f2ed5be031a..7e9c3caf23f079 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -207,7 +207,6 @@ Doc/c-api/stable.rst @encukou **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith From d8cf587dc749cf21eafc1064237970ee7460634f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 27 Sep 2024 22:13:29 +0200 Subject: [PATCH 42/66] doc: PyUnicode_AsUTF8String() fails if string contains surrogates (#124605) --- Doc/c-api/unicode.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 646c1b07222561..b2ac0c903c2bd7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -999,6 +999,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -1011,6 +1014,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -1438,8 +1444,9 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. From 3387f76b8f0b9f5ef89f9526c583bcc3dc36f486 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 27 Sep 2024 22:13:53 +0200 Subject: [PATCH 43/66] gh-124520: What's New entry for ctypes metaclass __new__/__init__ change (GH-124546) --- Doc/whatsnew/3.13.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45817799b542bc..52fe749697cfa4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -824,6 +824,24 @@ copy (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + + dbm --- From 6cba6e1df2c20846347b705eff7fb28caeeb17fd Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 27 Sep 2024 13:42:32 -0700 Subject: [PATCH 44/66] gh-124457: Remove coverity from CPython repo (GH-124460) Remove coverity from CPython repo. --- ...-09-24-11-52-36.gh-issue-124457.yrCjSV.rst | 2 + Misc/README | 1 - Misc/README.coverity | 22 --- Misc/coverity_model.c | 179 ------------------ 4 files changed, 2 insertions(+), 202 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst delete mode 100644 Misc/README.coverity delete mode 100644 Misc/coverity_model.c diff --git a/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst new file mode 100644 index 00000000000000..f9da7b8a5724f5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst @@ -0,0 +1,2 @@ +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 90c72c7baa3f9e..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tainted. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - From 626668912f3102a96d3f251f5304ad2f8326f3cc Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Fri, 27 Sep 2024 13:59:26 -0700 Subject: [PATCH 45/66] gh-81263: Add assignment expressions to `help` (#124641) * Add assignment expression (:=) to `help` * Update index for Assignment Expressions to include pair of `assignment; expression` --- Doc/reference/expressions.rst | 1 + Lib/pydoc.py | 2 ++ Lib/pydoc_data/topics.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f734221a2cdec5..ab72ad49d041e1 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1807,6 +1807,7 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression Assignment expressions ====================== diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d376592d69d40d..eec7b0770f56ca 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1870,6 +1870,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1963,6 +1964,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 4643df80e44aaf..97bb4eb52f4386 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -416,6 +416,34 @@ 'some expressions (like un-parenthesized tuple expressions) ' 'caused a\n' 'syntax error.\n', + 'assignment-expressions': 'Assignment expressions\n' + '**********************\n' + '\n' + 'An assignment expression (sometimes also called a “named expression”' + '\nor “walrus”) assigns an expression to an identifier, while also\n' + 'returning the value of the expression.\n' + '\n' + 'One common use case is when handling matched regular expressions:\n' + '\n' + ' if matching := pattern.search(data):\n' + ' do_something(matching)\n' + '\n' + 'Or, when processing a file stream in chunks:\n' + '\n' + ' while chunk := file.read(9000):\n' + ' process(chunk)\n' + '\n' + 'Assignment expressions must be surrounded by parentheses when used as\n' + 'expression statements and when used as sub-expressions in slicing,\n' + 'conditional, lambda, keyword-argument, and comprehension-if\n' + 'expressions and in assert, with, and assignment statements. In all\n' + 'other places where they can be used, parentheses are not required,\n' + 'including in if and while statements.\n' + '\n' + 'Added in version 3.8.\n' + 'See also:\n' + '\n' + ' **PEP 572** - Assignment Expressions\n', 'async': 'Coroutines\n' '**********\n' '\n' From 2357d5ba48cd9685cb36bcf92a0eaed86a85f4de Mon Sep 17 00:00:00 2001 From: Matt Delengowski Date: Fri, 27 Sep 2024 17:10:29 -0400 Subject: [PATCH 46/66] gh-90190: Add doc for using `singledispatch` with precise collection type hints (#116544) --- Doc/library/functools.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index c2c25ca67f338a..46136def06dc05 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -543,6 +543,25 @@ The :mod:`functools` module defines the following functions: ... print(arg.real, arg.imag) ... + For code that dispatches on a collections type (e.g., ``list``), but wants + to typehint the items of the collection (e.g., ``list[int]``), the + dispatch type should be passed explicitly to the decorator itself with the + typehint going into the function definition:: + + >>> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: From 0e21cc6cf820679439d72e3ebd06227ee2a085f9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 27 Sep 2024 14:51:01 -0700 Subject: [PATCH 47/66] GH-124547: Clear instance dictionary if memory error occurs during object dealloc (GH-124627) --- Lib/test/test_class.py | 15 +++++++++++++++ ...2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst | 3 +++ Objects/dictobject.c | 11 +++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 902f788edc22f0..d2b6a23cc3c10d 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +import test.support testmeths = [ @@ -932,6 +933,20 @@ class C: C.a = X() C.a = X() + def test_detach_materialized_dict_no_memory(self): + import _testcapi + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + self.assertEqual(ex.unraisable.exc_type, MemoryError) + with self.assertRaises(KeyError): + d["a"] if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst new file mode 100644 index 00000000000000..1005c651849f45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst @@ -0,0 +1,3 @@ +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f38ab1b2865e99..e50cf8bf1787f9 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3930,13 +3930,13 @@ dict_copy_impl(PyDictObject *self) } /* Copies the values, but does not change the reference - * counts of the objects in the array. */ + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ static PyDictValues * copy_values(PyDictValues *values) { PyDictValues *newvalues = new_values(values->capacity); if (newvalues == NULL) { - PyErr_NoMemory(); return NULL; } newvalues->size = values->size; @@ -7216,6 +7216,13 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) PyDictValues *values = copy_values(mp->ma_values); if (values == NULL) { + /* Out of memory. Clear the dict */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = mp->ma_keys; + set_keys(mp, Py_EMPTY_KEYS); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + STORE_USED(mp, 0); + PyErr_NoMemory(); return -1; } mp->ma_values = values; From 702c4a247360b43348a95c6fc76eb932483c33b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 27 Sep 2024 23:51:50 +0200 Subject: [PATCH 48/66] gh-111178: fix some USAN failures - mismatched function pointers (GH-123004) --- Objects/exceptions.c | 5 +++-- Objects/rangeobject.c | 6 +++--- Objects/tupleobject.c | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index fda62f159c1540..b3910855165494 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3387,8 +3387,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3447,7 +3448,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1318ce0319d438..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f14f10ab9c0a46..4d8cca68df946a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -999,8 +999,9 @@ tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *obj) { + _PyTupleIterObject *it = (_PyTupleIterObject *)obj; PyTupleObject *seq; PyObject *item; @@ -1101,7 +1102,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; From 1ba35ea38562bfc0301bab4e098aa124e114b886 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 27 Sep 2024 15:25:33 -0700 Subject: [PATCH 49/66] gh-123299: Copy-edit the 3.14 What's New (#124670) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/deprecations/pending-removal-in-3.16.rst | 8 ++ Doc/whatsnew/3.14.rst | 93 ++++++++++---------- 2 files changed, 56 insertions(+), 45 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 446cc63cb34ff9..fc2ef33de5e5cc 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -18,6 +18,14 @@ Pending Removal in Python 3.16 Use the ``'w'`` format code (:c:type:`Py_UCS4`) for Unicode characters instead. +* :mod:`asyncio`: + + * :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`shutil`: * The :class:`!ExecError` exception diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 6875c4c909b3c7..ffc001241ac5ec 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -185,7 +185,7 @@ Other Language Changes ``python -O -c 'assert (__debug__ := 1)'`` now produces a :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) -* Added class methods :meth:`float.from_number` and :meth:`complex.from_number` +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. They raise an error if the argument is a string. (Contributed by Serhiy Storchaka in :gh:`84978`.) @@ -206,7 +206,7 @@ Improved Modules ast --- -* Added :func:`ast.compare` for comparing two ASTs. +* Add :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) * Add support for :func:`copy.replace` for AST nodes. @@ -215,6 +215,9 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + ctypes ------ @@ -233,9 +236,9 @@ ctypes dis --- -* Added support for rendering full source location information of +* Add support for rendering full source location information of :class:`instructions `, rather than only the line number. - This feature is added to the following interfaces via the ``show_positions`` + This feature is added to the following interfaces via the *show_positions* keyword argument: - :class:`dis.Bytecode`, @@ -243,22 +246,21 @@ dis - :func:`dis.disassemble`. This feature is also exposed via :option:`dis --show-positions`. - (Contributed by Bénédikt Tran in :gh:`123165`.) fractions --------- -Added support for converting any objects that have the -:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. -(Contributed by Serhiy Storchaka in :gh:`82017`.) +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) functools --------- -* Added support to :func:`functools.partial` and +* Add support to :func:`functools.partial` and :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels to reserve a place for positional arguments. (Contributed by Dominykas Grigonis in :gh:`119127`.) @@ -267,27 +269,27 @@ functools http ---- -Directory lists and error pages generated by the :mod:`http.server` -module allow the browser to apply its default dark mode. -(Contributed by Yorik Hansen in :gh:`123430`.) +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) json ---- -Add notes for JSON serialization errors that allow to identify the source -of the error. -(Contributed by Serhiy Storchaka in :gh:`122163`.) +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) -Enable :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. -See the :ref:`JSON command-line interface ` documentation. -(Contributed by Trey Hunner in :gh:`122873`.) +* Enable the :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) operator -------- -* Two new functions ``operator.is_none`` and ``operator.is_not_none`` +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` have been added, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. @@ -297,13 +299,13 @@ operator datetime -------- -Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. -(Contributed by Wannes Boeykens in :gh:`41431`.) +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) os -- -* Added the :data:`os.environ.refresh() ` method to update +* Add the :data:`os.environ.refresh() ` method to update :data:`os.environ` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) @@ -333,7 +335,7 @@ pdb :pdbcmd:`commands` are preserved across hard-coded breakpoints. (Contributed by Tian Gao in :gh:`121450`.) -* Added a new argument ``mode`` to :class:`pdb.Pdb`. Disabled ``restart`` +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` command when :mod:`pdb` is in ``inline`` mode. (Contributed by Tian Gao in :gh:`123757`.) @@ -341,7 +343,7 @@ pickle ------ * Set the default protocol version on the :mod:`pickle` module to 5. - For more details, please see :ref:`pickle protocols `. + For more details, see :ref:`pickle protocols `. * Add notes for pickle serialization errors that allow to identify the source of the error. @@ -379,6 +381,12 @@ asyncio Deprecated ========== +* :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed @@ -437,7 +445,7 @@ ast user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom :class:`ast.NodeVisitor` subclasses will no longer be called when the - ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` method instead. Also, remove the following deprecated properties on :class:`ast.Constant`, @@ -588,18 +596,18 @@ New Features * Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` object: - * :c:func:`PyUnicodeWriter_Create`. - * :c:func:`PyUnicodeWriter_Discard`. - * :c:func:`PyUnicodeWriter_Finish`. - * :c:func:`PyUnicodeWriter_WriteChar`. - * :c:func:`PyUnicodeWriter_WriteUTF8`. - * :c:func:`PyUnicodeWriter_WriteUCS4`. - * :c:func:`PyUnicodeWriter_WriteWideChar`. - * :c:func:`PyUnicodeWriter_WriteStr`. - * :c:func:`PyUnicodeWriter_WriteRepr`. - * :c:func:`PyUnicodeWriter_WriteSubstring`. - * :c:func:`PyUnicodeWriter_Format`. - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteWideChar` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` (Contributed by Victor Stinner in :gh:`119182`.) @@ -611,11 +619,11 @@ New Features is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will - normally result in crashes during the exectuion of the subsequent call to + normally result in crashes during the execution of the subsequent call to :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use an address sanitizer to identify any use-after-free coming from an interned string and deallocate it during module shutdown. - (Contribued by Eddie Elizondo in :gh:`113601`.) + (Contributed by Eddie Elizondo in :gh:`113601`.) * Add new functions to convert C ```` numbers from/to Python :class:`int`: @@ -691,12 +699,7 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) -* :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - -.. Add deprecations above alphabetically, not here at the end. +.. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst From 425587a110eb214a097c634d4b6d944ac478923e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 28 Sep 2024 01:40:50 +0200 Subject: [PATCH 50/66] gh-124385: Document and soft-deprecate PyLong_AS_LONG (GH-124386) --- Doc/c-api/long.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 188eec4592a270..e0ae0f77a01db9 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -159,7 +159,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 -.. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -181,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) From 02b49c51501f5eeef3ab5d74fb9eace1151a1359 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 27 Sep 2024 16:50:16 -0700 Subject: [PATCH 51/66] gh-107954: Fix configuration type for the perf profiler (#124636) --- Doc/c-api/init_config.rst | 21 +++++++++++++-------- Lib/test/test_capi/test_config.py | 2 +- Lib/test/test_embed.py | 13 ++++++++----- Programs/_testembed.c | 5 +++++ Python/initconfig.c | 2 +- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0ef7d015be9b93..9dc9ba61e7a60f 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1248,19 +1248,24 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``1``, enable support for the Linux ``perf`` profiler. - Set by :option:`-X perf <-X>` command-line option and by the - :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support - with stack pointers and :option:`-X perf_jit <-X>` command-line option - and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf - support with DWARF JIT information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. + + Set to ``1`` by :option:`-X perf <-X>` command-line option and the + :envvar:`PYTHONPERFSUPPORT` environment variable. + + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 .. c:member:: int use_environment diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 01637e1cb7b6e5..71fb9ae45c7c30 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -68,7 +68,7 @@ def test_config_get(self): ("parser_debug", bool, None), ("parse_argv", bool, None), ("pathconfig_warnings", bool, None), - ("perf_profiling", bool, None), + ("perf_profiling", int, None), ("platlibdir", str, "platlibdir"), ("prefix", str | None, "prefix"), ("program_name", str, None), diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7c5cb855a397ab..3edc19d8254754 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -560,7 +560,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'cpu_count': -1, 'faulthandler': False, 'tracemalloc': 0, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': False, 'code_debug_ranges': True, 'show_ref_count': False, @@ -652,7 +652,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): use_hash_seed=False, faulthandler=False, tracemalloc=False, - perf_profiling=False, + perf_profiling=0, pathconfig_warnings=False, ) if MS_WINDOWS: @@ -966,7 +966,7 @@ def test_init_from_config(self): 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'show_ref_count': True, @@ -1031,7 +1031,7 @@ def test_init_compat_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1051,6 +1051,7 @@ def test_init_compat_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 @@ -1066,7 +1067,7 @@ def test_init_python_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1086,6 +1087,7 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = True @@ -1763,6 +1765,7 @@ def test_initconfig_api(self): 'xoptions': {'faulthandler': True}, 'hash_seed': 10, 'use_hash_seed': True, + 'perf_profiling': 2, } config_dev_mode(preconfig, config) self.check_all_configs("test_initconfig_api", config, preconfig, diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 10ee6b7be23e21..ab2b2d06cca15d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -810,6 +810,7 @@ static void set_most_env_vars(void) #ifdef Py_STATS putenv("PYTHONSTATS=1"); #endif + putenv("PYTHONPERFSUPPORT=1"); } @@ -1844,6 +1845,10 @@ static int test_initconfig_api(void) goto error; } + if (PyInitConfig_SetInt(config, "perf_profiling", 2) < 0) { + goto error; + } + // Set a UTF-8 string (program_name) if (PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) < 0) { goto error; diff --git a/Python/initconfig.c b/Python/initconfig.c index d93244f7f41084..58ac5e7d7eaeff 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -150,7 +150,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), SPEC(program_name, WSTR, READ_ONLY, NO_SYS), SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), From 165ed68c26759b817388add52a7aa2d26755d451 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 27 Sep 2024 17:19:44 -0700 Subject: [PATCH 52/66] Sorting techniques edits (#124701) --- Doc/howto/sorting.rst | 73 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index b98f91e023bdfc..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -47,11 +47,14 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: @@ -272,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= From fae5058ec13aa3b4f1acc549fadfbbbc2628f1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20B=2E?= <2589111+jfbu@users.noreply.github.com> Date: Sat, 28 Sep 2024 12:30:28 +0200 Subject: [PATCH 53/66] Doc: Fix default ``latex_elements['papersize']`` (#124525) https://www.sphinx-doc.org/en/master/latex.html#the-latex-elements-configuration-setting It should be 'letterpaper' or 'a4paper' not 'letter' or 'a4' (not to be confused with PAPER env variable). --- Doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 27cf03d6bea05a..5f22340ac434c9 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -413,8 +413,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } From c976d789a98047ae7ddec6d13c9ea7086d9fa3f9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 05:50:09 -0700 Subject: [PATCH 54/66] docs: improve venv docs (#124540) - Move "versionchanged" notes that apply to the whole class to the end of the class docs - Remove or move notes next to the method list that apply to individual methods. - Mark up parameters using the appropriate syntax - Do not capitalize "boolean" - Shorten some text --- Doc/library/venv.rst | 53 +++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index c0edb3f2705d8a..e2c77963ff3040 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -288,31 +288,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -330,10 +330,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -433,14 +430,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -448,22 +445,12 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories ``common``, ``posix``, ``nt``; each containing scripts destined for the ``bin`` directory in the environment. The contents of ``common`` and the @@ -495,6 +482,16 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ From dc12237ab092f0eee7ec5882196997804e635075 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 29 Sep 2024 01:12:53 +0900 Subject: [PATCH 55/66] gh-124688: _decimal: Get module state from ctx for performance (#124691) Get a module state from ctx objects for performance. --- Modules/_decimal/_decimal.c | 121 ++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 68d1da9faab867..a33c9793b5ad17 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -123,6 +123,7 @@ get_module_state(PyObject *mod) static struct PyModuleDef _decimal_module; static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -190,6 +191,7 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; typedef struct { @@ -210,6 +212,15 @@ typedef struct { #define CTX(v) (&((PyDecContextObject *)v)->ctx) #define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} + Py_LOCAL_INLINE(PyObject *) incr_true(void) @@ -564,7 +575,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -859,7 +870,7 @@ static PyObject * context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } @@ -1018,7 +1029,7 @@ context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1077,7 +1088,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1097,7 +1108,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1142,7 +1153,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1163,7 +1174,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1393,6 +1404,7 @@ context_new(PyTypeObject *type, CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; if (type == state->PyDecContext_Type) { PyObject_GC_Track(self); @@ -1470,7 +1482,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx((PyObject *)self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1561,7 +1573,7 @@ context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1581,7 +1593,7 @@ context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -2022,11 +2034,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2052,7 +2063,7 @@ PyDecType_New(PyTypeObject *type) assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2155,7 +2166,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2179,7 +2191,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2266,7 +2279,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2287,7 +2301,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2305,13 +2320,13 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { PyObject *dec; PyLongObject *l = (PyLongObject *)v; - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2356,7 +2371,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2385,7 +2401,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2417,7 +2434,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2438,7 +2455,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, sign = (copysign(1.0, x) == 1.0) ? 0 : 1; if (isnan(x) || isinf(x)) { - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2555,12 +2572,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2844,7 +2861,7 @@ dec_from_float(PyObject *type, PyObject *pyfloat) static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2855,7 +2872,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2882,7 +2899,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2917,7 +2934,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -3004,7 +3021,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3107,7 +3124,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3162,7 +3179,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3181,7 +3198,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -4421,12 +4438,11 @@ dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4436,6 +4452,13 @@ dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +static PyObject * +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + static PyObject * dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { @@ -5138,7 +5161,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5170,7 +5193,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5206,7 +5229,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5235,7 +5258,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5301,7 +5324,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5356,7 +5379,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5391,7 +5414,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5408,7 +5432,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5434,7 +5458,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5451,7 +5475,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5484,7 +5508,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5581,7 +5605,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5736,6 +5760,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, From 69a4063ca516360b5eb96f5432ad9f9dfc32a72e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 28 Sep 2024 20:51:49 +0300 Subject: [PATCH 56/66] gh-123339: Fix cases of inconsistency of __module__ and __firstlineno__ in classes (GH-123613) * Setting the __module__ attribute for a class now removes the __firstlineno__ item from the type's dict. * The _collections_abc and _pydecimal modules now completely replace the collections.abc and decimal modules after importing them. This allows to get the source of classes and functions defined in these modules. * inspect.findsource() now checks whether the first line number for a class is out of bound. --- Doc/reference/datamodel.rst | 5 +- Lib/collections/abc.py | 6 +- Lib/decimal.py | 7 ++- Lib/inspect.py | 6 +- Lib/test/test_builtin.py | 12 ++++ Lib/test/test_decimal.py | 3 +- Lib/test/test_inspect/inspect_fodder2.py | 12 ++++ Lib/test/test_inspect/test_inspect.py | 61 ++++++++++++++++++- ...-09-02-20-36-45.gh-issue-123339.QcmpSs.rst | 3 + ...-09-02-20-34-04.gh-issue-123339.czgcSu.rst | 4 ++ Objects/typeobject.c | 3 + 11 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst create mode 100644 Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 5ce6bf17db41ea..513199d21456bf 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1080,7 +1080,10 @@ Special attributes .. versionadded:: 3.13 * - .. attribute:: type.__firstlineno__ - - The line number of the first line of the class definition, including decorators. + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. .. versionadded:: 3.13 diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index bff76291634604..034ba377a0dbec 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ -from _collections_abc import * -from _collections_abc import __all__ # noqa: F401 -from _collections_abc import _CallableGenericAlias # noqa: F401 +import _collections_abc +import sys +sys.modules[__name__] = _collections_abc diff --git a/Lib/decimal.py b/Lib/decimal.py index f8c548eb1c6ecf..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -103,6 +103,7 @@ from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __version__ # noqa: F401 - from _pydecimal import __libmpdec_version__ # noqa: F401 + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b25300fcb2509..17314564f35397 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,12 @@ def findsource(object): if isclass(object): try: - firstlineno = vars(object)['__firstlineno__'] + lnum = vars(object)['__firstlineno__'] - 1 except (TypeError, KeyError): raise OSError('source code not available') - return lines, firstlineno - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 2ea97e797a4892..d884f54940b471 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2607,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2685,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 12479e32d0f5db..c591fd54430b18 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4381,7 +4381,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43e9f852022934..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -357,3 +357,15 @@ class td354(typing.TypedDict): # line 358 td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index aeee504fb8b555..d2dc9e147d29c2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -835,6 +835,47 @@ class C: nonlocal __firstlineno__ self.assertRaises(OSError, inspect.getsource, C) +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -947,6 +988,11 @@ def test_typeddict(self): self.assertSourceEqual(mod2.td354, 354, 356) self.assertRaises(OSError, inspect.getsource, mod2.td359) + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1010,7 +1056,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -1018,9 +1064,20 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst new file mode 100644 index 00000000000000..25b47d5fbaefa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst @@ -0,0 +1,3 @@ +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. diff --git a/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst new file mode 100644 index 00000000000000..e388541f1c2c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst @@ -0,0 +1,4 @@ +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3368c1ef577d14..0e2d9758a5ffae 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1435,6 +1435,9 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } From 04c837d9d8a474777ef9c1412fbba14f0682366c Mon Sep 17 00:00:00 2001 From: Kira Date: Sun, 29 Sep 2024 01:15:43 +0300 Subject: [PATCH 57/66] gh-124442: make `__static_attributes__` deterministic by sorting (#124492) Signed-off-by: kp2pml30 Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/test/test_compile.py | 17 +++++++++++++++++ ...24-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst | 2 ++ Python/compile.c | 12 +++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index b81d847c824273..f7ef7a1c26f7bd 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import contextlib import dis import io +import itertools import math import opcode import os @@ -2687,6 +2688,22 @@ def test_nested(self): self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst new file mode 100644 index 00000000000000..58e79f22ac0f90 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst @@ -0,0 +1,2 @@ +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. diff --git a/Python/compile.c b/Python/compile.c index 7b3e6f336e44b1..9826d3fbbde976 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -911,7 +911,17 @@ PyObject * _PyCompile_StaticAttributesAsTuple(compiler *c) { assert(c->u->u_static_attributes); - return PySequence_Tuple(c->u->u_static_attributes); + PyObject *static_attributes_unsorted = PySequence_List(c->u->u_static_attributes); + if (static_attributes_unsorted == NULL) { + return NULL; + } + if (PyList_Sort(static_attributes_unsorted) != 0) { + Py_DECREF(static_attributes_unsorted); + return NULL; + } + PyObject *static_attributes = PySequence_Tuple(static_attributes_unsorted); + Py_DECREF(static_attributes_unsorted); + return static_attributes; } int From c00964ecd508ba6ae43498017d5a3873844a058a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 29 Sep 2024 02:25:23 +0200 Subject: [PATCH 58/66] gh-124665: Add `_PyCodec_UnregisterError` and `_codecs._unregister_error` (#124677) --- Include/internal/pycore_codecs.h | 11 ++++++ Lib/test/test_codeccallbacks.py | 27 ++++++++++++++- Modules/_codecsmodule.c | 25 +++++++++++++ Modules/clinic/_codecsmodule.c.h | 52 +++++++++++++++++++++++++++- Python/codecs.c | 22 ++++++++++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + 6 files changed, 136 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 5e2d5c5ce9d868..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -21,6 +21,17 @@ extern void _PyCodec_Fini(PyInterpreterState *interp); extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 32373f0799bfeb..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,6 +1068,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..01855aec5e123e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2683,6 +2683,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2796,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index 9c0a3fad314cb5..68dc232bb86163 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,6 +16,12 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +static const char *codecs_builtin_error_handlers[] = { + "strict", "ignore", "replace", + "xmlcharrefreplace", "backslashreplace", "namereplace", + "surrogatepass", "surrogateescape", +}; + const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ @@ -618,6 +624,20 @@ int PyCodec_RegisterError(const char *name, PyObject *error) name, error); } +int _PyCodec_UnregisterError(const char *name) +{ + for (size_t i = 0; i < Py_ARRAY_LENGTH(codecs_builtin_error_handlers); ++i) { + if (strcmp(name, codecs_builtin_error_handlers[i]) == 0) { + PyErr_Format(PyExc_ValueError, + "cannot un-register built-in error handler '%s'", name); + return -1; + } + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->codecs.initialized); + return PyDict_PopString(interp->codecs.error_registry, name, NULL); +} + /* Lookup the error handling callback function registered under the name error. As a special case NULL can be passed, in which case the error handling callback for strict encoding will be returned. */ @@ -1470,6 +1490,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) } } }; + // ensure that the built-in error handlers' names are kept in sync + assert(Py_ARRAY_LENGTH(methods) == Py_ARRAY_LENGTH(codecs_builtin_error_handlers)); assert(interp->codecs.initialized == 0); interp->codecs.search_path = PyList_New(0); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index f4dc807198a8ef..e6c599a2ac4a46 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -345,6 +345,7 @@ Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - +Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LOCATION - From e0a41a5dd12cb6e9277b05abebac5c70be684dd7 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sun, 29 Sep 2024 08:42:46 +0530 Subject: [PATCH 59/66] GH-124639: add back loop param to staggered_race (#124700) --- Lib/asyncio/staggered.py | 10 ++++++++-- Lib/test/test_asyncio/test_staggered.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 4458d01dece0e6..6ccf5c3c269ff0 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -11,7 +11,7 @@ class _Done(Exception): pass -async def staggered_race(coro_fns, delay): +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is @@ -82,7 +82,13 @@ async def run_one_coro(this_index, coro_fn, this_failed): raise _Done try: - async with taskgroups.TaskGroup() as tg: + tg = taskgroups.TaskGroup() + # Intentionally override the loop in the TaskGroup to avoid + # using the running loop, preserving backwards compatibility + # TaskGroup only starts using `_loop` after `__aenter__` + # so overriding it here is safe. + tg._loop = loop + async with tg: for this_index, coro_fn in enumerate(coro_fns): this_failed = locks.Event() exceptions.append(None) diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 21a39b3f911747..8cd98394aea8f8 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -121,6 +121,25 @@ async def coro(index): self.assertIsInstance(excs[0], ValueError) self.assertIsNone(excs[1]) + def test_loop_argument(self): + loop = asyncio.new_event_loop() + async def coro(): + self.assertEqual(loop, asyncio.get_running_loop()) + return 'coro' + + async def main(): + winner, index, excs = await staggered_race( + [coro], + delay=0.1, + loop=loop + ) + + self.assertEqual(winner, 'coro') + self.assertEqual(index, 0) + + loop.run_until_complete(main()) + loop.close() + if __name__ == "__main__": unittest.main() From d08c7888229e78533648191dfe42e2d2d3ecea25 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:40:20 +0300 Subject: [PATCH 60/66] gh-123497: New limit for Python integers on 64-bit platforms (GH-123724) Instead of be limited just by the size of addressable memory (2**63 bytes), Python integers are now also limited by the number of bits, so the number of bit now always fit in a 64-bit integer. Both limits are much larger than what might be available in practice, so it doesn't affect users. _PyLong_NumBits() and _PyLong_Frexp() are now always successful. --- Include/cpython/longobject.h | 5 +- Include/internal/pycore_long.h | 11 +- Modules/_pickle.c | 6 +- Modules/_randommodule.c | 6 +- Modules/mathmodule.c | 25 ++--- Objects/floatobject.c | 11 +- Objects/longobject.c | 196 +++++++++++---------------------- Python/ast_opt.c | 23 ++-- 8 files changed, 108 insertions(+), 175 deletions(-) diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 82f8cc8a159c77..b239f7c557e016 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -71,10 +71,9 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); absolute value of a long. For example, this returns 1 for 1 and -1, 2 for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. v must not be NULL, and must be a normalized long. - (uint64_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. + Always successful. */ -PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v); +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 8822147b636dd4..196b4152280a35 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -79,11 +79,10 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); @@ -105,10 +104,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 18affdd4875f3b..b2bd9545c1b130 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2146,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - uint64_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; @@ -2161,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (uint64_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 3835a3072d96c6..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -295,7 +295,7 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - uint64_t bits; + int64_t bits; size_t keyused; int res; @@ -335,8 +335,8 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (uint64_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index baf2dc439b8959..058f57770755aa 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - uint64_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1680,14 +1680,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (uint64_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1704,13 +1703,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1727,12 +1726,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - uint64_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1742,7 +1741,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2202,8 +2201,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index dc3d8a3e5d0f4b..a48a210adee3b9 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -406,19 +406,16 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - uint64_t nbits64 = _PyLong_NumBits(w); - if (nbits64 > (unsigned int)DBL_MAX_EXP) { + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { /* This Python integer is larger than any finite C double. * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) { - /* This Python integer is so large that uint64_t isn't - * big enough to hold the # of bits. */ - PyErr_Clear(); - } i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; diff --git a/Objects/longobject.c b/Objects/longobject.c index d34c8b6d71ab3f..9beb5884a6932b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -133,8 +133,16 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif PyLongObject * _PyLong_New(Py_ssize_t size) @@ -804,11 +812,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -uint64_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - uint64_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -818,21 +826,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((uint64_t)(ndigits - 1) > UINT64_MAX / (uint64_t)PyLong_SHIFT) - goto Overflow; - result = (uint64_t)(ndigits - 1) * (uint64_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (UINT64_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - /* Very unlikely. Such integer would require more than 2 exbibytes of RAM. */ - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a 64-bit integer"); - return (uint64_t)-1; } PyObject * @@ -1247,15 +1246,12 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) /* Calculates the number of bits required for the *absolute* value * of v. This does not take sign into account, only magnitude. */ - uint64_t nb = _PyLong_NumBits((PyObject *)v); - if (nb == (uint64_t)-1) { - res = -1; - } else { - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up - * multiples of 8 to the next byte, but we add an implied bit for - * the sign and it cancels out. */ - res = (Py_ssize_t)(nb / 8) + 1; - } + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; /* Two edge cases exist that are best handled after extracting the * bits. These may result in us reporting overflow when the value @@ -3415,7 +3411,8 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) double _PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; @@ -3432,14 +3429,7 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) *e = 0; return 0.0; } - int msd_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + msd_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (INT64_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (INT64_MAX - 1) / PyLong_SHIFT + 1 || - msd_bits > (INT64_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (int64_t)(a_size - 1) * PyLong_SHIFT + msd_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3468,18 +3458,18 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) */ if (a_bits <= DBL_MANT_DIG + 2) { shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) % PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); - shift_bits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3505,21 +3495,13 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == INT64_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3547,7 +3529,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -5217,39 +5201,6 @@ long_bool(PyLongObject *v) return !_PyLong_IsZero(v); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ -static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; -} - /* Inner function for both long_rshift and _PyLong_Rshift, shifting an integer right by PyLong_SHIFT*wordshift + remshift bits. wordshift should be nonnegative. */ @@ -5343,8 +5294,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5355,24 +5305,35 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, uint64_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { if (_PyLong_IsNegative((PyLongObject *)a)) { return PyLong_FromLong(-1); } @@ -5430,8 +5391,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_lshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5442,24 +5402,30 @@ long_lshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return _PyLong_Lshift(a, shiftby); } /* Return a << shiftby. */ PyObject * -_PyLong_Lshift(PyObject *a, uint64_t shiftby) +_PyLong_Lshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); return NULL; @@ -6213,11 +6179,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - uint64_t nbits = _PyLong_NumBits(self); - if (nbits == (uint64_t)-1) { - return NULL; - } - return PyLong_FromUnsignedLongLong(nbits); + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -6251,40 +6216,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f5b04757e08bf3..01e208b88eca8b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -169,11 +169,10 @@ safe_multiply(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); - uint64_t wbits = _PyLong_NumBits(w); - if (vbits == (uint64_t)-1 || wbits == (uint64_t)-1) { - return NULL; - } + int64_t vbits = _PyLong_NumBits(v); + int64_t wbits = _PyLong_NumBits(w); + assert(vbits >= 0); + assert(wbits >= 0); if (vbits + wbits > MAX_INT_SIZE) { return NULL; } @@ -215,12 +214,13 @@ safe_power(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && _PyLong_IsPositive((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (vbits > MAX_INT_SIZE / wbits) { + if ((uint64_t)vbits > MAX_INT_SIZE / wbits) { return NULL; } } @@ -234,12 +234,13 @@ safe_lshift(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (wbits > MAX_INT_SIZE || vbits > MAX_INT_SIZE - wbits) { + if (wbits > MAX_INT_SIZE || (uint64_t)vbits > MAX_INT_SIZE - wbits) { return NULL; } } From 49e105f9488de18d3d92948232fcbd956cbe0c6e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:44:34 +0300 Subject: [PATCH 61/66] gh-104860: Fix allow_abbrev=False for single-dash long options (GH-124340) --- Lib/argparse.py | 2 +- Lib/test/test_argparse.py | 17 +++++++++++++++++ ...24-09-23-17-33-47.gh-issue-104860.O86OSc.rst | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 690b2a9db9481b..4506a5e3a378f4 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2328,7 +2328,7 @@ def _get_option_tuples(self, option_string): action = self._option_string_actions[option_string] tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] tup = action, option_string, None, None result.append(tup) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ef05a6fefcffcc..26e32c30b4d104 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -914,6 +914,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" diff --git a/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst new file mode 100644 index 00000000000000..707c4d651cb5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst @@ -0,0 +1,2 @@ +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. From 9bcadf589ab6f7b9d309290de7a80156b6905d35 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:47:06 +0300 Subject: [PATCH 62/66] gh-80259: Fix conflict between type and default=SUPPRESS in argparse (GH-124519) type() no longer called for SUPPRESS. This only affects positional arguments with nargs='?'. --- Lib/argparse.py | 2 +- Lib/test/test_argparse.py | 20 ++++++++++++------- ...4-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst | 2 ++ 3 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 4506a5e3a378f4..3c2023e69e9ad6 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2483,7 +2483,7 @@ def _get_values(self, action, arg_strings): value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) self._check_value(action, value) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 26e32c30b4d104..08f31c239950e0 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1587,18 +1587,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst new file mode 100644 index 00000000000000..bb451cdd9ae44c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. From 61180446eee2aef07b042c7e8892c45afabd1499 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:52:07 +0300 Subject: [PATCH 63/66] gh-124345: Support abbreviated single-dash long options with = in argparse (GH-124428) --- Lib/argparse.py | 6 ++++-- Lib/test/test_argparse.py | 11 ++++++++++- .../2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 3c2023e69e9ad6..715d2ba39075e5 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2319,7 +2319,9 @@ def _get_option_tuples(self, option_string): # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] @@ -2330,7 +2332,7 @@ def _get_option_tuples(self, option_string): result.append(tup) elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, None, None + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 08f31c239950e0..01dc99c9434043 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -380,15 +380,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -875,7 +882,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] diff --git a/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst new file mode 100644 index 00000000000000..dff902d8c6139a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst @@ -0,0 +1,2 @@ +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. From dac4ec52866e4068f3ac33b4da1e1a1fe6fc2cba Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:52:52 +0300 Subject: [PATCH 64/66] gh-53834: Fix support of arguments with choices in argparse (GH-124495) Positional arguments with nargs equal to '?' or '*' no longer check default against choices. Optional arguments with nargs equal to '?' no longer check const against choices. --- Lib/argparse.py | 6 +----- Lib/test/test_argparse.py | 16 ++++++++-------- ...2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst | 4 ++++ 3 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 715d2ba39075e5..f428512d4007ef 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2487,7 +2487,6 @@ def _get_values(self, action, arg_strings): value = action.default if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2495,11 +2494,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 01dc99c9434043..5c8f0ec0b58e9a 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -628,9 +628,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -1027,8 +1027,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1061,8 +1061,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1075,9 +1075,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), diff --git a/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst new file mode 100644 index 00000000000000..20ba1534f5e99d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst @@ -0,0 +1,4 @@ +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. From f1a2417b9e2993e584610851ac004c8b0599b323 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 10:57:21 +0300 Subject: [PATCH 65/66] gh-61181: Fix support of choices with string value in argparse (GH-124578) Substrings of the specified string no longer considered valid values. --- Doc/library/argparse.rst | 2 +- Lib/argparse.py | 14 +++++++++----- Lib/test/test_argparse.py | 10 +++++----- .../2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst | 2 ++ 4 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d5a21899ae4f99..a4683bccf651cd 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1823,7 +1823,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) diff --git a/Lib/argparse.py b/Lib/argparse.py index f428512d4007ef..a671c8b56b55bb 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2552,11 +2552,15 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + choices = action.choices + if choices is not None: + if isinstance(choices, str): + choices = iter(choices) + if value not in choices: + args = {'value': value, + 'choices': ', '.join(map(repr, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 5c8f0ec0b58e9a..91f04fa1358a9d 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -686,7 +686,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -2270,14 +2270,14 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser @@ -4618,7 +4618,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -5281,7 +5281,7 @@ def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: + dict(choices=['a', 'b'])]: self.assertTypeError('-x', action=action, **attrs) def test_no_argument_no_const_actions(self): diff --git a/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst new file mode 100644 index 00000000000000..801a5fdd4abd4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst @@ -0,0 +1,2 @@ +Fix support of :ref:`choices` with string value in :mod:`argparse`. Substrings +of the specified string no longer considered valid values. From 95e92ef6c74e973ea13d15180190d0fa2af82fbf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Sep 2024 11:01:10 +0300 Subject: [PATCH 66/66] gh-116850: Fix argparse for namespaces with not directly writable dict (GH-124667) It now always uses setattr() instead of setting the dict item to modify the namespace. This allows to use a class as a namespace. --- Lib/argparse.py | 3 ++- Lib/test/test_argparse.py | 12 ++++++++++++ .../2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index a671c8b56b55bb..504289192f3f96 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1224,7 +1224,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 91f04fa1358a9d..15805937baaa97 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2344,6 +2344,18 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + def test_parse_known_args_with_single_dash_option(self): parser = ErrorRaisingArgumentParser() parser.add_argument('-k', '--known', action='count', default=0) diff --git a/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst new file mode 100644 index 00000000000000..62639a16c52aa0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes).