diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ada5fb0fe64dc2..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:40 - -ENV CC=clang - -ENV WASI_SDK_VERSION=24 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=22.0.0 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 680f2ed5be031a..7e9c3caf23f079 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -207,7 +207,6 @@ Doc/c-api/stable.rst @encukou **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0ef7d015be9b93..9dc9ba61e7a60f 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1248,19 +1248,24 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``1``, enable support for the Linux ``perf`` profiler. - Set by :option:`-X perf <-X>` command-line option and by the - :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support - with stack pointers and :option:`-X perf_jit <-X>` command-line option - and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf - support with DWARF JIT information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. + + Set to ``1`` by :option:`-X perf <-X>` command-line option and the + :envvar:`PYTHONPERFSUPPORT` environment variable. + + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 .. c:member:: int use_environment diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 188eec4592a270..e0ae0f77a01db9 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -159,7 +159,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 -.. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -181,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 30e26fe52178d7..b2ac0c903c2bd7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -614,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -657,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -666,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -674,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 @@ -990,6 +999,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -1002,6 +1014,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -1429,8 +1444,9 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. diff --git a/Doc/conf.py b/Doc/conf.py index 27cf03d6bea05a..5f22340ac434c9 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -413,8 +413,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index 446cc63cb34ff9..fc2ef33de5e5cc 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -18,6 +18,14 @@ Pending Removal in Python 3.16 Use the ``'w'`` format code (:c:type:`Py_UCS4`) for Unicode characters instead. +* :mod:`asyncio`: + + * :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`shutil`: * The :class:`!ExecError` exception diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4a6f1ca57d89e3..fa7b22bde1dc6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:attr:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index b98f91e023bdfc..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -47,11 +47,14 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: @@ -272,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 2219e37f6b0677..37490456d13312 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -32,7 +32,7 @@ This module supports retrieving annotations in three main formats for annotations that cannot be resolved, allowing you to inspect the annotations without evaluating them. This is useful when you need to work with annotations that may contain unresolved forward references. -* :attr:`~Format.SOURCE` returns the annotations as a string, similar +* :attr:`~Format.STRING` returns the annotations as a string, similar to how it would appear in the source file. This is useful for documentation generators that want to display annotations in a readable way. @@ -135,7 +135,7 @@ Classes values. Real objects may contain references to, :class:`ForwardRef` proxy objects. - .. attribute:: SOURCE + .. attribute:: STRING :value: 3 Values are the text string of the annotation as it appears in the @@ -197,23 +197,23 @@ Classes Functions --------- -.. function:: annotations_to_source(annotations) +.. function:: annotations_to_string(annotations) Convert an annotations dict containing runtime values to a dict containing only strings. If the values are not already strings, - they are converted using :func:`value_to_source`. + they are converted using :func:`value_to_string`. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. - For example, this is used to implement the :attr:`~Format.SOURCE` for + For example, this is used to implement the :attr:`~Format.STRING` for :class:`typing.TypedDict` classes created through the functional syntax: .. doctest:: >>> from typing import TypedDict >>> Movie = TypedDict("movie", {"name": str, "year": int}) - >>> get_annotations(Movie, format=Format.SOURCE) + >>> get_annotations(Movie, format=Format.STRING) {'name': 'str', 'year': 'int'} .. versionadded:: 3.14 @@ -282,7 +282,7 @@ Functions NameError: name 'undefined' is not defined >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) ForwardRef('undefined') - >>> call_evaluate_function(Alias.evaluate_value, Format.SOURCE) + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) 'undefined' .. versionadded:: 3.14 @@ -369,14 +369,14 @@ Functions .. versionadded:: 3.14 -.. function:: value_to_source(value) +.. function:: value_to_string(value) Convert an arbitrary Python value to a format suitable for use by the - :attr:`~Format.SOURCE` format. This calls :func:`repr` for most + :attr:`~Format.STRING` format. This calls :func:`repr` for most objects, but has special handling for some objects, such as type objects. This is meant as a helper for user-provided - annotate functions that support the :attr:`~Format.SOURCE` format but + annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. It can also be used to provide a user-friendly string representation for other objects that contain values that are commonly encountered in annotations. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d5a21899ae4f99..a4683bccf651cd 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1823,7 +1823,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a8a18ad31fb773..a9518859b83478 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -178,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -199,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -214,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -243,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -1771,9 +1771,9 @@ aliases. .. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. *name* is the name of the type variable. - *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. *default_value* + A :class:`typing.TypeVar`. ``name`` is the name of the type variable. + ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. ``default_value`` is the default value; if the :class:`!TypeVar` has no default, this attribute will be set to ``None``. @@ -1801,8 +1801,8 @@ aliases. .. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. - *default_value* is the default value; if the :class:`!ParamSpec` has no default, + A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, this attribute will be set to ``None``. .. doctest:: @@ -1836,8 +1836,8 @@ aliases. .. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. - *default_value* is the default value; if the :class:`!TypeVarTuple` has no + A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no default, this attribute will be set to ``None``. .. doctest:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e3b24451188cc4..ce72127127c7a6 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -286,14 +286,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +302,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 1457392ce6e86c..51c1a427b63787 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -231,7 +231,7 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -300,6 +300,10 @@ Module contents .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 774b3262117723..46136def06dc05 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -354,7 +354,7 @@ The :mod:`functools` module defines the following functions: newfunc.keywords = keywords return newfunc - The :func:`partial` function is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument @@ -368,10 +368,11 @@ The :mod:`functools` module defines the following functions: 18 If :data:`Placeholder` sentinels are present in *args*, they will be filled first - when :func:`partial` is called. This allows custom selection of positional arguments - to be pre-filled when constructing a :ref:`partial object `. + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, only the + first positional argument can be pre-filled. - If :data:`!Placeholder` sentinels are present, all of them must be filled at call time: + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: .. doctest:: @@ -379,14 +380,15 @@ The :mod:`functools` module defines the following functions: >>> say_to_world('Hello', 'dear') Hello dear world! - Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because - only one positional argument is provided, while there are two placeholders - in :ref:`partial object `. + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. - Successive :func:`partial` applications fill :data:`!Placeholder` sentinels - of the input :func:`partial` objects with new positional arguments. - A place for positional argument can be retained by inserting new - :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: .. doctest:: @@ -402,8 +404,8 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - Note, :data:`!Placeholder` has no special treatment when used for keyword - argument of :data:`!Placeholder`. + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. @@ -541,6 +543,25 @@ The :mod:`functools` module defines the following functions: ... print(arg.real, arg.imag) ... + For code that dispatches on a collections type (e.g., ``list``), but wants + to typehint the items of the collection (e.g., ``list[int]``), the + dispatch type should be passed explicitly to the decorator itself with the + typehint going into the function definition:: + + >>> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: @@ -791,7 +812,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes are not created automatically. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e4cef1f3e3b7c0..27d31f66b12495 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1166,10 +1166,9 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is no synchronization between the two objects. For example, it is possible to update the module's :attr:`__file__` at runtime and this will not be automatically @@ -1179,66 +1178,60 @@ find and load modules. .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name + (see :attr:`__name__` attributes on modules). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module + (see :attr:`__loader__` attributes on modules). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`__file__` attributes on modules). + For example, for modules loaded from a .py file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) - - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The list of locations where the package's submodules will be found + (see :attr:`__path__` attributes on modules). + Most of the time this is a single directory. + The :term:`finder` should set this attribute to a list, even an empty one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`__cached__` attributes on modules). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`__package__` attributes on modules. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`origin` is interpreted + and how the module's :attr:`__file__` is populated. .. class:: AppleFrameworkLoader(name, path) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c1299ebfe8d27a..9a62249816c9bf 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -58,7 +58,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','ABC'], len) → (1, A B) (3, ABC)`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` @@ -93,7 +93,7 @@ Examples Results Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. @@ -131,11 +131,12 @@ loops that truncate the stream. total = function(total, element) yield total - The *function* argument can be set to :func:`min` for a running - minimum, :func:`max` for a running maximum, or :func:`operator.mul` - for a running product. `Amortization tables - `_ - can be built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `Amortization table + `_, + accumulate the interest and apply payments: .. doctest:: @@ -202,10 +203,10 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): # chain('ABC', 'DEF') → A B C D E F @@ -353,10 +354,12 @@ loops that truncate the stream. def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: yield element @@ -396,8 +399,10 @@ loops that truncate the stream. def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -474,7 +479,7 @@ loops that truncate the stream. If *start* is zero or ``None``, iteration starts at zero. Otherwise, elements from the iterable are skipped until *start* is reached. - If *stop* is ``None``, iteration continues until the iterable is + If *stop* is ``None``, iteration continues until the input is exhausted, if at all. Otherwise, it stops at the specified position. If *step* is ``None``, the step defaults to one. Elements are returned @@ -520,8 +525,10 @@ loops that truncate the stream. def pairwise(iterable): # pairwise('ABCDEFG') → AB BC CD DE EF FG + iterator = iter(iterable) a = next(iterator, None) + for b in iterator: yield a, b a = b @@ -584,7 +591,8 @@ loops that truncate the stream. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 80d6e4dae24463..036b8f44b9ff3b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,11 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for @@ -162,6 +164,13 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1f316307965c11..57a1f920523035 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -574,11 +574,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c08b10d67bb031..640bc2c9d503bc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3427,7 +3427,7 @@ Introspection helpers * Replaces type hints that evaluate to :const:`!None` with :class:`types.NoneType`. * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.SOURCE` formats. + :attr:`~annotationlib.Format.STRING` formats. *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. *owner*, if given, should be the object that holds the annotations that diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d603a163c2e82d..cc2b1b4299553c 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -401,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -409,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: + + .. doctest:: + + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + Note that :meth:`reset_mock` *doesn't* clear the :attr:`return_value`, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - :attr:`return_value` or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + set using normal assignment by default. - .. note:: *return_value*, and *side_effect* are keyword-only - arguments. + Child mocks are reset as well. + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index cf6c5437be4fd1..e2c77963ff3040 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -37,14 +37,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -61,7 +61,127 @@ See :pep:`405` for more background on Python virtual environments. Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -117,7 +237,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -168,31 +288,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -210,10 +330,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -313,14 +430,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -328,25 +445,15 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,10 +478,20 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ @@ -387,16 +504,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 097a39cea31c67..1b1e9f479cbe08 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1217,9 +1217,10 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1326,11 +1327,16 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the semantics of a function. See :ref:`annotations` for more information on annotations. +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. + .. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 5ce6bf17db41ea..513199d21456bf 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1080,7 +1080,10 @@ Special attributes .. versionadded:: 3.13 * - .. attribute:: type.__firstlineno__ - - The line number of the first line of the class definition, including decorators. + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. .. versionadded:: 3.13 diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b5f5523d368964..ab72ad49d041e1 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -284,7 +284,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -309,7 +309,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -454,7 +454,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -485,9 +485,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -576,7 +576,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -695,7 +695,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -759,7 +759,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -892,7 +892,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -904,9 +904,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1803,6 +1807,7 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression Assignment expressions ====================== @@ -1905,10 +1910,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1929,6 +1936,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1f725c2377035b..c89b1693343b4e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -288,6 +288,9 @@ def run(self): version_deprecated = expand_version_arg(self.arguments[0], self.config.release) version_removed = self.arguments.pop(1) + if version_removed == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') self.arguments[0] = version_deprecated, version_removed # Set the label based on if we have reached the removal version diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index c978e61b16a49e..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -13,7 +13,7 @@ {% endif %} {% block body %} -

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index c97c65f7a3988e..fd765e58ff2485 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -209,8 +209,10 @@ after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 354eb1541ceac2..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 45817799b542bc..52fe749697cfa4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -824,6 +824,24 @@ copy (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + + dbm --- diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3d6084e6ecc19b..ffc001241ac5ec 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -91,7 +91,7 @@ annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VA format (which evaluates annotations to runtime values, similar to the behavior in earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format (which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.SOURCE` format (which returns annotations as strings). +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). This example shows how these formats behave: @@ -106,7 +106,7 @@ This example shows how these formats behave: NameError: name 'Undefined' is not defined >>> get_annotations(func, format=Format.FORWARDREF) {'arg': ForwardRef('Undefined')} - >>> get_annotations(func, format=Format.SOURCE) + >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} Implications for annotated code @@ -185,7 +185,7 @@ Other Language Changes ``python -O -c 'assert (__debug__ := 1)'`` now produces a :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) -* Added class methods :meth:`float.from_number` and :meth:`complex.from_number` +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. They raise an error if the argument is a string. (Contributed by Serhiy Storchaka in :gh:`84978`.) @@ -206,7 +206,7 @@ Improved Modules ast --- -* Added :func:`ast.compare` for comparing two ASTs. +* Add :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) * Add support for :func:`copy.replace` for AST nodes. @@ -215,6 +215,9 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + ctypes ------ @@ -233,9 +236,9 @@ ctypes dis --- -* Added support for rendering full source location information of +* Add support for rendering full source location information of :class:`instructions `, rather than only the line number. - This feature is added to the following interfaces via the ``show_positions`` + This feature is added to the following interfaces via the *show_positions* keyword argument: - :class:`dis.Bytecode`, @@ -243,22 +246,21 @@ dis - :func:`dis.disassemble`. This feature is also exposed via :option:`dis --show-positions`. - (Contributed by Bénédikt Tran in :gh:`123165`.) fractions --------- -Added support for converting any objects that have the -:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. -(Contributed by Serhiy Storchaka in :gh:`82017`.) +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) functools --------- -* Added support to :func:`functools.partial` and +* Add support to :func:`functools.partial` and :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels to reserve a place for positional arguments. (Contributed by Dominykas Grigonis in :gh:`119127`.) @@ -267,27 +269,27 @@ functools http ---- -Directory lists and error pages generated by the :mod:`http.server` -module allow the browser to apply its default dark mode. -(Contributed by Yorik Hansen in :gh:`123430`.) +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) json ---- -Add notes for JSON serialization errors that allow to identify the source -of the error. -(Contributed by Serhiy Storchaka in :gh:`122163`.) +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) -Enable :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. -See the :ref:`JSON command-line interface ` documentation. -(Contributed by Trey Hunner in :gh:`122873`.) +* Enable the :mod:`json` module to work as a script using the :option:`-m` switch: ``python -m json``. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) operator -------- -* Two new functions ``operator.is_none`` and ``operator.is_not_none`` +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` have been added, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. @@ -297,13 +299,13 @@ operator datetime -------- -Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. -(Contributed by Wannes Boeykens in :gh:`41431`.) +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) os -- -* Added the :data:`os.environ.refresh() ` method to update +* Add the :data:`os.environ.refresh() ` method to update :data:`os.environ` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) @@ -333,7 +335,7 @@ pdb :pdbcmd:`commands` are preserved across hard-coded breakpoints. (Contributed by Tian Gao in :gh:`121450`.) -* Added a new argument ``mode`` to :class:`pdb.Pdb`. Disabled ``restart`` +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` command when :mod:`pdb` is in ``inline`` mode. (Contributed by Tian Gao in :gh:`123757`.) @@ -341,7 +343,7 @@ pickle ------ * Set the default protocol version on the :mod:`pickle` module to 5. - For more details, please see :ref:`pickle protocols `. + For more details, see :ref:`pickle protocols `. * Add notes for pickle serialization errors that allow to identify the source of the error. @@ -379,12 +381,26 @@ asyncio Deprecated ========== +* :mod:`asyncio`: + :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16, + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`builtins`: Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`multiprocessing` and :mod:`concurrent.futures`: + The default start method (see :ref:`multiprocessing-start-methods`) changed + away from *fork* to *forkserver* on platforms where it was not already + *spawn* (Windows & macOS). If you require the threading incompatible *fork* + start method you must explicitly specify it when using :mod:`multiprocessing` + or :mod:`concurrent.futures` APIs. + (Contributed by Gregory P. Smith in :gh:`84559`.) + * :mod:`os`: :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to @@ -429,7 +445,7 @@ ast user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom :class:`ast.NodeVisitor` subclasses will no longer be called when the - ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` method instead. Also, remove the following deprecated properties on :class:`ast.Constant`, @@ -580,18 +596,18 @@ New Features * Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` object: - * :c:func:`PyUnicodeWriter_Create`. - * :c:func:`PyUnicodeWriter_Discard`. - * :c:func:`PyUnicodeWriter_Finish`. - * :c:func:`PyUnicodeWriter_WriteChar`. - * :c:func:`PyUnicodeWriter_WriteUTF8`. - * :c:func:`PyUnicodeWriter_WriteUCS4`. - * :c:func:`PyUnicodeWriter_WriteWideChar`. - * :c:func:`PyUnicodeWriter_WriteStr`. - * :c:func:`PyUnicodeWriter_WriteRepr`. - * :c:func:`PyUnicodeWriter_WriteSubstring`. - * :c:func:`PyUnicodeWriter_Format`. - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteWideChar` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` (Contributed by Victor Stinner in :gh:`119182`.) @@ -603,11 +619,11 @@ New Features is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will - normally result in crashes during the exectuion of the subsequent call to + normally result in crashes during the execution of the subsequent call to :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use an address sanitizer to identify any use-after-free coming from an interned string and deallocate it during module shutdown. - (Contribued by Eddie Elizondo in :gh:`113601`.) + (Contributed by Eddie Elizondo in :gh:`113601`.) * Add new functions to convert C ```` numbers from/to Python :class:`int`: @@ -683,12 +699,7 @@ Deprecated :c:macro:`!isfinite` available from :file:`math.h` since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) -* :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - -.. Add deprecations above alphabetically, not here at the end. +.. Add C API deprecations above alphabetically, not here at the end. .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst diff --git a/Include/Python.h b/Include/Python.h index 8fffa22df9da48..e1abdd16f031fb 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -55,6 +55,10 @@ # include // __readgsqword() #endif +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 82f8cc8a159c77..b239f7c557e016 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -71,10 +71,9 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); absolute value of a long. For example, this returns 1 for 1 and -1, 2 for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. v must not be NULL, and must be a normalized long. - (uint64_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. + Always successful. */ -PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v); +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3db3aa3eb77879..a9d1bce127e63d 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -108,7 +108,7 @@ backoff_counter_triggers(_Py_BackoffCounter counter) /* Initial JUMP_BACKWARD counter. * This determines when we create a trace for a loop. * Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */ -#define JUMP_BACKWARD_INITIAL_VALUE 16 +#define JUMP_BACKWARD_INITIAL_VALUE 15 #define JUMP_BACKWARD_INITIAL_BACKOFF 4 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -122,7 +122,7 @@ initial_jump_backoff_counter(void) * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 64 +#define SIDE_EXIT_INITIAL_VALUE 63 #define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a97b53028c8f59..363845106e40dc 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -283,6 +283,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra #define _PY_GC_SCHEDULED_BIT (1U << 4) #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 5e2d5c5ce9d868..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -21,6 +21,17 @@ extern void _PyCodec_Fini(PyInterpreterState *interp); extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 36366429e8db25..a1898d926ac39f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -261,7 +261,7 @@ struct _is { struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 8822147b636dd4..196b4152280a35 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -79,11 +79,10 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); @@ -105,10 +104,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 19e54bf122a8bb..f92c0a0cddf906 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -29,9 +29,10 @@ typedef struct { typedef struct { uint8_t opcode; uint8_t oparg; - uint16_t valid:1; - uint16_t linked:1; - uint16_t chain_depth:14; // Must be big engough for MAX_CHAIN_DEPTH - 1. + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -123,11 +124,18 @@ PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #ifdef _Py_TIER2 PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + #endif +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 800 diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index ca5a1e2adb4787..118bc98b35d5e3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -209,7 +209,6 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *); // Export for _testinternalcapi extension. PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b950f760d74ac7..927dae88c1fa73 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -222,64 +222,65 @@ extern "C" { #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 439 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 439 -#define _MONITOR_CALL 440 -#define _MONITOR_JUMP_BACKWARD 441 -#define _MONITOR_RESUME 442 +#define _MAYBE_EXPAND_METHOD 440 +#define _MONITOR_CALL 441 +#define _MONITOR_JUMP_BACKWARD 442 +#define _MONITOR_RESUME 443 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 443 -#define _POP_JUMP_IF_TRUE 444 +#define _POP_JUMP_IF_FALSE 444 +#define _POP_JUMP_IF_TRUE 445 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 445 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 446 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 446 +#define _PUSH_FRAME 447 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 447 -#define _PY_FRAME_KW 448 -#define _QUICKEN_RESUME 449 -#define _REPLACE_WITH_TRUE 450 +#define _PY_FRAME_GENERAL 448 +#define _PY_FRAME_KW 449 +#define _QUICKEN_RESUME 450 +#define _REPLACE_WITH_TRUE 451 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 451 -#define _SEND 452 -#define _SEND_GEN_FRAME 453 +#define _SAVE_RETURN_OFFSET 452 +#define _SEND 453 +#define _SEND_GEN_FRAME 454 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 454 -#define _STORE_ATTR 455 -#define _STORE_ATTR_INSTANCE_VALUE 456 -#define _STORE_ATTR_SLOT 457 -#define _STORE_ATTR_WITH_HINT 458 +#define _START_EXECUTOR 455 +#define _STORE_ATTR 456 +#define _STORE_ATTR_INSTANCE_VALUE 457 +#define _STORE_ATTR_SLOT 458 +#define _STORE_ATTR_WITH_HINT 459 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 459 -#define _STORE_FAST_0 460 -#define _STORE_FAST_1 461 -#define _STORE_FAST_2 462 -#define _STORE_FAST_3 463 -#define _STORE_FAST_4 464 -#define _STORE_FAST_5 465 -#define _STORE_FAST_6 466 -#define _STORE_FAST_7 467 +#define _STORE_FAST 460 +#define _STORE_FAST_0 461 +#define _STORE_FAST_1 462 +#define _STORE_FAST_2 463 +#define _STORE_FAST_3 464 +#define _STORE_FAST_4 465 +#define _STORE_FAST_5 466 +#define _STORE_FAST_6 467 +#define _STORE_FAST_7 468 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 468 -#define _STORE_SUBSCR 469 +#define _STORE_SLICE 469 +#define _STORE_SUBSCR 470 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 470 -#define _TO_BOOL 471 +#define _TIER2_RESUME_CHECK 471 +#define _TO_BOOL 472 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -289,14 +290,14 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 472 +#define _UNPACK_SEQUENCE 473 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE #define __DO_CALL_FUNCTION_EX _DO_CALL_FUNCTION_EX -#define MAX_UOP_ID 472 +#define MAX_UOP_ID 473 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4d0ab22e6aa8f3..07606135d7a356 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -274,6 +274,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, + [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, @@ -481,6 +482,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", @@ -1062,6 +1064,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _START_EXECUTOR: return 0; + case _MAKE_WARM: + return 0; case _FATAL_ERROR: return 0; case _CHECK_VALIDITY_AND_SET_IP: diff --git a/Include/object.h b/Include/object.h index 7124f58f6bdb37..418f2196062df7 100644 --- a/Include/object.h +++ b/Include/object.h @@ -180,6 +180,12 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 358e2c070cd5fa..e0d20632516142 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -72,7 +72,7 @@ We currently also immortalize strings contained in code objects and similar, specifically in the compiler and in `marshal`. These are “close enough” to immortal: even in use cases like hot reloading or `eval`-ing user input, the number of distinct identifiers and string -constants expected to stay low. +constants is expected to stay low. ## Internal API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 4139cbadf93e13..c2edf6c8856c21 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -485,10 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() - from annotationlib import value_to_source + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([value_to_source(a) for a in self.__args__[:-1]])}], ' - f'{value_to_source(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f7a0095d795ac6..d457d2b5a338eb 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -371,15 +371,19 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self) -> INPUT_RECORD | None: + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): raise WinError(GetLastError()) - if read.value == 0: - return None - return rec def get_event(self, block: bool = True) -> Event | None: @@ -390,10 +394,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.pop() while True: - rec = self._read_input() + rec = self._read_input(block) if rec is None: - if block: - continue return None if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: @@ -464,8 +466,8 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" - while self._read_input() is not None: - pass + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -590,6 +592,14 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -602,5 +612,7 @@ def _win_only(*args, **kwargs): ScrollConsoleScreenBuffer = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only OutHandle = 0 InHandle = 0 diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a027f4de3dfed6..a11188722487b2 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -15,15 +15,15 @@ "call_evaluate_function", "get_annotate_function", "get_annotations", - "annotations_to_source", - "value_to_source", + "annotations_to_string", + "value_to_string", ] class Format(enum.IntEnum): VALUE = 1 FORWARDREF = 2 - SOURCE = 3 + STRING = 3 _Union = None @@ -291,9 +291,21 @@ def __convert_to_ast(self, other): return other.__ast_node__ elif isinstance(other, slice): return ast.Slice( - lower=self.__convert_to_ast(other.start) if other.start is not None else None, - upper=self.__convert_to_ast(other.stop) if other.stop is not None else None, - step=self.__convert_to_ast(other.step) if other.step is not None else None, + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), ) else: return ast.Constant(value=other) @@ -469,7 +481,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): can be called with any of the format arguments in the Format enum, but compiler-generated __annotate__ functions only support the VALUE format. This function provides additional functionality to call __annotate__ - functions with the FORWARDREF and SOURCE formats. + functions with the FORWARDREF and STRING formats. *annotate* must be an __annotate__ function, which takes a single argument and returns a dict of annotations. @@ -487,8 +499,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): return annotate(format) except NotImplementedError: pass - if format == Format.SOURCE: - # SOURCE is implemented by calling the annotate function in a special + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special # environment where every name lookup results in an instance of _Stringifier. # _Stringifier supports every dunder operation and returns a new _Stringifier. # At the end, we get a dictionary that mostly contains _Stringifier objects (or @@ -524,9 +536,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): for key, val in annos.items() } elif format == Format.FORWARDREF: - # FORWARDREF is implemented similarly to SOURCE, but there are two changes, + # FORWARDREF is implemented similarly to STRING, but there are two changes, # at the beginning and the end of the process. - # First, while SOURCE uses an empty dictionary as the namespace, so that all + # First, while STRING uses an empty dictionary as the namespace, so that all # name lookups result in _Stringifier objects, FORWARDREF uses the globals # and builtins, so that defined names map to their real values. # Second, instead of returning strings, we want to return either real values @@ -688,14 +700,14 @@ def get_annotations( # __annotations__ threw NameError and there is no __annotate__. In that case, # we fall back to trying __annotations__ again. return dict(_get_dunder_annotations(obj)) - case Format.SOURCE: - # For SOURCE, we try to call __annotate__ + case Format.STRING: + # For STRING, we try to call __annotate__ ann = _get_and_call_annotate(obj, format) if ann is not None: return ann # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) - return annotations_to_source(ann) + return annotations_to_string(ann) case _: raise ValueError(f"Unsupported format {format!r}") @@ -764,10 +776,10 @@ def get_annotations( return return_value -def value_to_source(value): - """Convert a Python value to a format suitable for use with the SOURCE format. +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. - This is inteded as a helper for tools that support the SOURCE format but do + This is inteded as a helper for tools that support the STRING format but do not have access to the code that originally produced the annotations. It uses repr() for most objects. @@ -783,10 +795,10 @@ def value_to_source(value): return repr(value) -def annotations_to_source(annotations): - """Convert an annotation dict containing values to approximately the SOURCE format.""" +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" return { - n: t if isinstance(t, str) else value_to_source(t) + n: t if isinstance(t, str) else value_to_string(t) for n, t in annotations.items() } diff --git a/Lib/argparse.py b/Lib/argparse.py index bf58c0a0b4648d..874f271959c4fe 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1224,7 +1224,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -2316,7 +2317,9 @@ def _get_option_tuples(self, option_string): # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] @@ -2325,9 +2328,9 @@ def _get_option_tuples(self, option_string): action = self._option_string_actions[option_string] tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, None, None + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here @@ -2477,9 +2480,8 @@ def _get_values(self, action, arg_strings): value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2487,11 +2489,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2548,11 +2547,15 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + choices = action.choices + if choices is not None: + if isinstance(choices, str): + choices = iter(choices) + if value not in choices: + args = {'value': value, + 'choices': ', '.join(map(repr, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 4458d01dece0e6..6ccf5c3c269ff0 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -11,7 +11,7 @@ class _Done(Exception): pass -async def staggered_race(coro_fns, delay): +async def staggered_race(coro_fns, delay, *, loop=None): """Run coroutines with staggered start times and take the first to finish. This method takes an iterable of coroutine functions. The first one is @@ -82,7 +82,13 @@ async def run_one_coro(this_index, coro_fn, this_failed): raise _Done try: - async with taskgroups.TaskGroup() as tg: + tg = taskgroups.TaskGroup() + # Intentionally override the loop in the TaskGroup to avoid + # using the running loop, preserving backwards compatibility + # TaskGroup only starts using `_loop` after `__aenter__` + # so overriding it here is safe. + tg._loop = loop + async with tg: for this_index, coro_fn in enumerate(coro_fns): this_failed = locks.Event() exceptions.append(None) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index bff76291634604..034ba377a0dbec 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ -from _collections_abc import * -from _collections_abc import __all__ # noqa: F401 -from _collections_abc import _CallableGenericAlias # noqa: F401 +import _collections_abc +import sys +sys.modules[__name__] = _collections_abc diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f5cb97edaf72cd..bdda7cc6c00f5d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -283,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -300,6 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None @recursive_repr() @@ -315,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -382,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -394,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -402,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -1174,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1239,7 +1242,32 @@ def _update_func_cell_for__class__(f, oldcls, newcls): return False -def _add_slots(cls, is_frozen, weakref_slot): +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # Need to create a new class, since we can't set __slots__ after a # class has been created, and the @dataclass decorator is called # after the class is created. @@ -1255,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: diff --git a/Lib/decimal.py b/Lib/decimal.py index f8c548eb1c6ecf..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -103,6 +103,7 @@ from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __version__ # noqa: F401 - from _pydecimal import __libmpdec_version__ # noqa: F401 + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b25300fcb2509..17314564f35397 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -970,10 +970,12 @@ def findsource(object): if isclass(object): try: - firstlineno = vars(object)['__firstlineno__'] + lnum = vars(object)['__firstlineno__'] - 1 except (TypeError, KeyError): raise OSError('source code not available') - return lines, firstlineno - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index ddcc7e7900999e..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/pdb.py b/Lib/pdb.py index 443160eaaae887..28270167565564 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -403,13 +403,6 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary used to be updated from the actual frame - # locals whenever the .f_locals accessor was called, so it was - # cached here to ensure that modifications were not overwritten. While - # the caching is no longer required now that f_locals is a direct proxy - # on optimized frames, it's also harmless, so the code structure has - # been left unchanged. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) if self._chained_exceptions: @@ -732,7 +725,7 @@ def _exec_in_closure(self, source, globals, locals): def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: buffer = line @@ -960,7 +953,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -991,7 +984,7 @@ def completedefault(self, text, line, begidx, endidx): # Use rlcompleter to do the completion state = 0 matches = [] - completer = Completer(self.curframe.f_globals | self.curframe_locals) + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) while (match := completer.complete(text, state)) is not None: matches.append(match) state += 1 @@ -1153,7 +1146,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1458,7 +1451,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1704,7 +1696,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1749,7 +1741,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1769,15 +1761,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1785,7 +1777,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -2029,7 +2021,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") diff --git a/Lib/pydoc.py b/Lib/pydoc.py index d376592d69d40d..eec7b0770f56ca 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1870,6 +1870,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1963,6 +1964,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 4643df80e44aaf..97bb4eb52f4386 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -416,6 +416,34 @@ 'some expressions (like un-parenthesized tuple expressions) ' 'caused a\n' 'syntax error.\n', + 'assignment-expressions': 'Assignment expressions\n' + '**********************\n' + '\n' + 'An assignment expression (sometimes also called a “named expression”' + '\nor “walrus”) assigns an expression to an identifier, while also\n' + 'returning the value of the expression.\n' + '\n' + 'One common use case is when handling matched regular expressions:\n' + '\n' + ' if matching := pattern.search(data):\n' + ' do_something(matching)\n' + '\n' + 'Or, when processing a file stream in chunks:\n' + '\n' + ' while chunk := file.read(9000):\n' + ' process(chunk)\n' + '\n' + 'Assignment expressions must be surrounded by parentheses when used as\n' + 'expression statements and when used as sub-expressions in slicing,\n' + 'conditional, lambda, keyword-argument, and comprehension-if\n' + 'expressions and in assert, with, and assignment statements. In all\n' + 'other places where they can be used, parentheses are not required,\n' + 'including in if and while statements.\n' + '\n' + 'Added in version 3.8.\n' + 'See also:\n' + '\n' + ' **PEP 572** - Assignment Expressions\n', 'async': 'Coroutines\n' '**********\n' '\n' diff --git a/Lib/statistics.py b/Lib/statistics.py index d3dd0d530c31cf..f193fcdc241aa9 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -870,9 +870,12 @@ def f_inv(y): return f_inv def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: + if p < 0.499: x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) return x * sign @@ -886,8 +889,11 @@ def quartic_kernel(): return pdf, cdf, invcdf, support def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) return x * sign @register('triweight') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b3a0645cfc84a..a059a6b8340448 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5553,15 +5553,29 @@ def test_set_get(self): multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 628529b8664c77..99cb10fc7b5f7b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2209,7 +2209,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f493a92e0ddce8..14cd50bd30502c 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -849,7 +849,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index dc1106aee1e2f1..eedf2506a14912 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -12,8 +12,8 @@ ForwardRef, get_annotations, get_annotate_function, - annotations_to_source, - value_to_source, + annotations_to_string, + value_to_string, ) from typing import Unpack @@ -39,14 +39,14 @@ def __repr__(self): class TestFormat(unittest.TestCase): def test_enum(self): - self.assertEqual(annotationlib.Format.VALUE.value, 1) - self.assertEqual(annotationlib.Format.VALUE, 1) + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) - self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) - self.assertEqual(annotationlib.Format.FORWARDREF, 2) + self.assertEqual(Format.FORWARDREF.value, 2) + self.assertEqual(Format.FORWARDREF, 2) - self.assertEqual(annotationlib.Format.SOURCE.value, 3) - self.assertEqual(annotationlib.Format.SOURCE, 3) + self.assertEqual(Format.STRING.value, 3) + self.assertEqual(Format.STRING, 3) class TestForwardRefFormat(unittest.TestCase): @@ -54,9 +54,7 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) fwdref = anno["arg"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) self.assertEqual(fwdref.__forward_arg__, "x") @@ -66,16 +64,14 @@ def inner(arg: x): x = 1 self.assertEqual(fwdref.evaluate(), x) - anno = annotationlib.get_annotations( - inner, format=annotationlib.Format.FORWARDREF - ) + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) self.assertIs(anno["x"], int) fwdref = anno["y"] self.assertIsInstance(fwdref, annotationlib.ForwardRef) @@ -92,14 +88,14 @@ def test_closure(self): def inner(arg: x): pass - anno = annotationlib.get_annotations(inner, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(inner, format=Format.STRING) self.assertEqual(anno, {"arg": "x"}) def test_function(self): def f(x: int, y: doesntexist): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) def test_expressions(self): @@ -133,7 +129,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -184,7 +180,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -218,7 +214,7 @@ def f( ): pass - anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + anno = annotationlib.get_annotations(f, format=Format.STRING) self.assertEqual( anno, { @@ -241,13 +237,13 @@ def f(fstring: f"{a}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) def f(fstring_format: f"{a:02d}"): pass with self.assertRaisesRegex(TypeError, format_msg): - annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + annotationlib.get_annotations(f, format=Format.STRING) class TestForwardRefClass(unittest.TestCase): @@ -276,7 +272,7 @@ class Gen[T]: with self.assertRaises(NameError): ForwardRef("T").evaluate(owner=int) - T, = Gen.__type_params__ + (T,) = Gen.__type_params__ self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) @@ -294,8 +290,7 @@ class Gen[T]: def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( - ForwardRef("Counter", module="collections").evaluate(), - collections.Counter + ForwardRef("Counter", module="collections").evaluate(), collections.Counter ) self.assertEqual( ForwardRef("Counter[int]", module="collections").evaluate(), @@ -383,22 +378,20 @@ class C1(metaclass=NoDict): self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(C1, format=Format.FORWARDREF), {"a": int}, ) self.assertEqual( - annotationlib.get_annotations(C1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(C1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) self.assertEqual( - annotationlib.get_annotations( - NoDict, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), {"b": str}, ) self.assertEqual( - annotationlib.get_annotations(NoDict, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(NoDict, format=Format.STRING), {"b": "str"}, ) @@ -410,20 +403,20 @@ def f2(a: undefined): pass self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.VALUE), + annotationlib.get_annotations(f1, format=Format.VALUE), {"a": int}, ) self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) fwd = annotationlib.ForwardRef("undefined") self.assertEqual( - annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), + annotationlib.get_annotations(f2, format=Format.FORWARDREF), {"a": fwd}, ) self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) self.assertEqual( - annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(f1, format=Format.STRING), {"a": "int"}, ) self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) @@ -446,30 +439,26 @@ def foo(): pass with self.assertRaises(ValueError): - annotationlib.get_annotations( - foo, format=annotationlib.Format.FORWARDREF, eval_str=True - ) - annotationlib.get_annotations( - foo, format=annotationlib.Format.SOURCE, eval_str=True - ) + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) def test_stock_annotations(self): def foo(a: int, b: str): pass - for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + for format in (Format.VALUE, Format.FORWARDREF): with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), {"a": int, "b": str}, ) self.assertEqual( - annotationlib.get_annotations(foo, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(foo, format=Format.STRING), {"a": "int", "b": "str"}, ) foo.__annotations__ = {"a": "foo", "b": "str"} - for format in annotationlib.Format: + for format in Format: with self.subTest(format=format): self.assertEqual( annotationlib.get_annotations(foo, format=format), @@ -491,10 +480,10 @@ def test_stock_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -529,7 +518,7 @@ def test_stock_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -563,48 +552,36 @@ def test_stock_annotations_in_module(self): ) self.assertEqual( - annotationlib.get_annotations(isa, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(isa, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.MyClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), {"a": "int", "b": "str"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function2, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function2, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - isa.function3, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.function3, format=Format.STRING), {"a": "int", "b": "str", "c": "MyClass"}, ) self.assertEqual( - annotationlib.get_annotations( - annotationlib, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(annotationlib, format=Format.STRING), {}, ) self.assertEqual( - annotationlib.get_annotations( - isa.UnannotatedClass, format=annotationlib.Format.SOURCE - ), + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), {}, ) self.assertEqual( annotationlib.get_annotations( - isa.unannotated_function, format=annotationlib.Format.SOURCE + isa.unannotated_function, format=Format.STRING ), {}, ) @@ -620,13 +597,11 @@ def test_stock_annotations_on_wrapper(self): {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations( - wrapped, format=annotationlib.Format.FORWARDREF - ), + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), {"a": int, "b": str, "return": isa.MyClass}, ) self.assertEqual( - annotationlib.get_annotations(wrapped, format=annotationlib.Format.SOURCE), + annotationlib.get_annotations(wrapped, format=Format.STRING), {"a": "int", "b": "str", "return": "MyClass"}, ) self.assertEqual( @@ -643,12 +618,12 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {}, {"eval_str": False}, - {"format": annotationlib.Format.VALUE}, - {"format": annotationlib.Format.FORWARDREF}, - {"format": annotationlib.Format.SOURCE}, - {"format": annotationlib.Format.VALUE, "eval_str": False}, - {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, - {"format": annotationlib.Format.SOURCE, "eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -681,7 +656,7 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, - {"format": annotationlib.Format.VALUE, "eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): self.assertEqual( @@ -767,9 +742,9 @@ def f(x: int): annotationlib.get_annotations(f, format=Format.FORWARDREF), {"x": str}, ) - # ... but not in SOURCE which always uses __annotate__ + # ... but not in STRING which always uses __annotate__ self.assertEqual( - annotationlib.get_annotations(f, format=Format.SOURCE), + annotationlib.get_annotations(f, format=Format.STRING), {"x": "int"}, ) @@ -804,7 +779,7 @@ def __annotations__(self): ) self.assertEqual( - annotationlib.get_annotations(ha, format=Format.SOURCE), {"x": "int"} + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} ) def test_raising_annotations_on_custom_object(self): @@ -844,7 +819,7 @@ def __annotate__(self): annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} ) self.assertEqual( - annotationlib.get_annotations(hb, format=Format.SOURCE), {"x": str} + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} ) def test_pep695_generic_class_with_future_annotations(self): @@ -974,15 +949,13 @@ def evaluate(format, exc=NotImplementedError): return undefined with self.assertRaises(NameError): - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.VALUE) + annotationlib.call_evaluate_function(evaluate, Format.VALUE) self.assertEqual( - annotationlib.call_evaluate_function( - evaluate, annotationlib.Format.FORWARDREF - ), + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), annotationlib.ForwardRef("undefined"), ) self.assertEqual( - annotationlib.call_evaluate_function(evaluate, annotationlib.Format.SOURCE), + annotationlib.call_evaluate_function(evaluate, Format.STRING), "undefined", ) @@ -1093,25 +1066,25 @@ class C: class TestToSource(unittest.TestCase): - def test_value_to_source(self): - self.assertEqual(value_to_source(int), "int") - self.assertEqual(value_to_source(MyClass), "test.test_annotationlib.MyClass") - self.assertEqual(value_to_source(len), "len") - self.assertEqual(value_to_source(value_to_source), "value_to_source") - self.assertEqual(value_to_source(times_three), "times_three") - self.assertEqual(value_to_source(...), "...") - self.assertEqual(value_to_source(None), "None") - self.assertEqual(value_to_source(1), "1") - self.assertEqual(value_to_source("1"), "'1'") - self.assertEqual(value_to_source(Format.VALUE), repr(Format.VALUE)) - self.assertEqual(value_to_source(MyClass()), "my repr") - - def test_annotations_to_source(self): - self.assertEqual(annotations_to_source({}), {}) - self.assertEqual(annotations_to_source({"x": int}), {"x": "int"}) - self.assertEqual(annotations_to_source({"x": "int"}), {"x": "int"}) + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) self.assertEqual( - annotations_to_source({"x": int, "y": str}), {"x": "int", "y": "str"} + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} ) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index b68e73101326b0..a972ed0cc9053b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -380,15 +380,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -621,9 +628,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -679,7 +686,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -875,7 +882,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -914,6 +923,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -1001,8 +1027,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1035,8 +1061,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1049,9 +1075,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1570,18 +1596,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -2238,14 +2270,14 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser @@ -2312,6 +2344,18 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + def test_abbreviation(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foodle') @@ -4608,7 +4652,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -5271,7 +5315,7 @@ def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: + dict(choices=['a', 'b'])]: self.assertTypeError('-x', action=action, **attrs) def test_no_argument_no_const_actions(self): diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 21a39b3f911747..8cd98394aea8f8 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -121,6 +121,25 @@ async def coro(index): self.assertIsInstance(excs[0], ValueError) self.assertIsNone(excs[1]) + def test_loop_argument(self): + loop = asyncio.new_event_loop() + async def coro(): + self.assertEqual(loop, asyncio.get_running_loop()) + return 'coro' + + async def main(): + winner, index, excs = await staggered_race( + [coro], + delay=0.1, + loop=loop + ) + + self.assertEqual(winner, 'coro') + self.assertEqual(index, 0) + + loop.run_until_complete(main()) + loop.close() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 2ea97e797a4892..d884f54940b471 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2607,6 +2607,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2685,6 +2686,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 01637e1cb7b6e5..71fb9ae45c7c30 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -68,7 +68,7 @@ def test_config_get(self): ("parser_debug", bool, None), ("parse_argv", bool, None), ("pathconfig_warnings", bool, None), - ("perf_profiling", bool, None), + ("perf_profiling", int, None), ("platlibdir", str, "platlibdir"), ("prefix", str | None, "prefix"), ("program_name", str, None), diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 902f788edc22f0..d2b6a23cc3c10d 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,6 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest +import test.support testmeths = [ @@ -932,6 +933,20 @@ class C: C.a = X() C.a = X() + def test_detach_materialized_dict_no_memory(self): + import _testcapi + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + self.assertEqual(ex.unraisable.exc_type, MemoryError) + with self.assertRaises(KeyError): + d["a"] if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index b81d847c824273..f7ef7a1c26f7bd 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import contextlib import dis import io +import itertools import math import opcode import os @@ -2687,6 +2688,22 @@ def test_nested(self): self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 69e86162e0c11a..2984f4261bd2c4 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,6 +17,7 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict +from copy import deepcopy from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. @@ -60,7 +61,7 @@ class C: x: int = field(default=1, default_factory=int) def test_field_repr(self): - int_field = field(default=1, init=True, repr=False) + int_field = field(default=1, init=True, repr=False, doc='Docstring') int_field.name = "id" repr_output = repr(int_field) expected_output = "Field(name='id',type=None," \ @@ -68,6 +69,7 @@ def test_field_repr(self): "init=True,repr=False,hash=None," \ "compare=True,metadata=mappingproxy({})," \ f"kw_only={MISSING!r}," \ + "doc='Docstring'," \ "_field_type=None)" self.assertEqual(repr_output, expected_output) @@ -3175,6 +3177,48 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): def test_simple(self): @@ -3261,7 +3305,7 @@ class Base(Root4): j: str h: str - self.assertEqual(Base.__slots__, ('y', )) + self.assertEqual(Base.__slots__, ('y',)) @dataclass(slots=True) class Derived(Base): @@ -3271,7 +3315,7 @@ class Derived(Base): k: str h: str - self.assertEqual(Derived.__slots__, ('z', )) + self.assertEqual(Derived.__slots__, ('z',)) @dataclass class AnotherDerived(Base): @@ -3279,6 +3323,24 @@ class AnotherDerived(Base): self.assertNotIn('__slots__', AnotherDerived.__dict__) + def test_slots_with_docs(self): + class Root: + __slots__ = {'x': 'x'} + + @dataclass(slots=True) + class Base(Root): + y1: int = field(doc='y1') + y2: int + + self.assertEqual(Base.__slots__, {'y1': 'y1', 'y2': None}) + + @dataclass(slots=True) + class Child(Base): + z1: int = field(doc='z1') + z2: int + + self.assertEqual(Child.__slots__, {'z1': 'z1', 'z2': None}) + def test_cant_inherit_from_iterator_slots(self): class Root: diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 12479e32d0f5db..c591fd54430b18 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4381,7 +4381,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7c5cb855a397ab..3edc19d8254754 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -560,7 +560,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'cpu_count': -1, 'faulthandler': False, 'tracemalloc': 0, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': False, 'code_debug_ranges': True, 'show_ref_count': False, @@ -652,7 +652,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): use_hash_seed=False, faulthandler=False, tracemalloc=False, - perf_profiling=False, + perf_profiling=0, pathconfig_warnings=False, ) if MS_WINDOWS: @@ -966,7 +966,7 @@ def test_init_from_config(self): 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'show_ref_count': True, @@ -1031,7 +1031,7 @@ def test_init_compat_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1051,6 +1051,7 @@ def test_init_compat_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 @@ -1066,7 +1067,7 @@ def test_init_python_env(self): 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': False, + 'perf_profiling': 0, 'import_time': True, 'code_debug_ranges': False, 'malloc_stats': True, @@ -1086,6 +1087,7 @@ def test_init_python_env(self): 'module_search_paths': self.IGNORE_CONFIG, 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = True @@ -1763,6 +1765,7 @@ def test_initconfig_api(self): 'xoptions': {'faulthandler': True}, 'hash_seed': 10, 'use_hash_seed': True, + 'perf_profiling': 2, } config_dev_mode(preconfig, config) self.check_all_configs("test_initconfig_api", config, preconfig, diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index c6b58fcd86f449..a705161369e8dd 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -3,10 +3,13 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper +NTHREAD = 10 +OBJECT_COUNT = 5_000 + + class C: def __init__(self, v): self.v = v @@ -14,11 +17,8 @@ def __init__(self, v): @threading_helper.requires_working_threading() class TestList(TestCase): - @support.requires_resource('cpu') def test_racing_iter_append(self): - l = [] - OBJECT_COUNT = 10000 def writer_func(): for i in range(OBJECT_COUNT): @@ -34,7 +34,7 @@ def reader_func(): writer = Thread(target=writer_func) readers = [] - for x in range(30): + for x in range(NTHREAD): reader = Thread(target=reader_func) readers.append(reader) reader.start() @@ -44,39 +44,32 @@ def reader_func(): for reader in readers: reader.join() - @support.requires_resource('cpu') def test_racing_iter_extend(self): - iters = [ - lambda x: [x], - ] - for iter_case in iters: - with self.subTest(iter=iter_case): - l = [] - OBJECT_COUNT = 10000 - - def writer_func(): - for i in range(OBJECT_COUNT): - l.extend(iter_case(C(i + OBJECT_COUNT))) - - def reader_func(): - while True: - count = len(l) - for i, x in enumerate(l): - self.assertEqual(x.v, i + OBJECT_COUNT) - if count == OBJECT_COUNT: - break - - writer = Thread(target=writer_func) - readers = [] - for x in range(30): - reader = Thread(target=reader_func) - readers.append(reader) - reader.start() - - writer.start() - writer.join() - for reader in readers: - reader.join() + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend([C(i + OBJECT_COUNT)]) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() if __name__ == "__main__": diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index be582455d118ac..8fec01715531cb 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,7 +7,6 @@ import weakref from sys import monitoring -from test import support from test.support import threading_helper from threading import Thread, _PyRLock from unittest import TestCase @@ -15,7 +14,7 @@ class InstrumentationMultiThreadedMixin: thread_count = 10 - func_count = 200 + func_count = 50 fib = 12 def after_threads(self): @@ -37,14 +36,13 @@ def work(self, n, funcs): def start_work(self, n, funcs): # With the GIL builds we need to make sure that the hooks have # a chance to run as it's possible to run w/o releasing the GIL. - time.sleep(1) + time.sleep(0.1) self.work(n, funcs) def after_test(self): """Runs once after the test is done""" pass - @support.requires_resource('cpu') def test_instrumentation(self): # Setup a bunch of functions which will need instrumentation... funcs = [] @@ -220,29 +218,31 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) - @support.requires_resource('cpu') def test_set_local_trace_opcodes(self): def trace(frame, event, arg): frame.f_trace_opcodes = True return trace + loops = 1_000 + sys.settrace(trace) try: l = _PyRLock() def f(): - for i in range(3000): + for i in range(loops): with l: pass t = Thread(target=f) t.start() - for i in range(3000): + for i in range(loops): with l: pass t.join() finally: sys.settrace(None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 977bfd2c7fd2f7..51463b6bb8c1b4 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -5,7 +5,6 @@ from threading import Thread from unittest import TestCase -from test import support from test.support import threading_helper @@ -97,8 +96,9 @@ def reader_func(): self.run_one(writer_func, reader_func) - @support.requires_resource('cpu') def test___class___modification(self): + loops = 200 + class Foo: pass @@ -108,7 +108,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(5000): + for _ in range(loops): foo.__class__ = Bar type(foo) foo.__class__ = Foo diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 906f9884d6792f..bb7df1f5cfa7f7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1048,6 +1048,24 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + class IncrementalGCTests(unittest.TestCase): diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43e9f852022934..43fda6622537fc 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -357,3 +357,15 @@ class td354(typing.TypedDict): # line 358 td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index aeee504fb8b555..d2dc9e147d29c2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -835,6 +835,47 @@ class C: nonlocal __firstlineno__ self.assertRaises(OSError, inspect.getsource, C) +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; @@ -947,6 +988,11 @@ def test_typeddict(self): self.assertSourceEqual(mod2.td354, 354, 356) self.assertRaises(OSError, inspect.getsource, mod2.td359) + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -1010,7 +1056,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -1018,9 +1064,20 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2dba077cdea6a7..776e02f41a1cec 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -463,6 +463,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 0f3e9996e77e45..36f940eaea4eac 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1080,7 +1080,7 @@ def setUp(self): @force_not_colorized def test_exposed_globals_in_repl(self): - pre = "['__annotations__', '__builtins__'" + pre = "['__builtins__'" post = "'__loader__', '__name__', '__package__', '__spec__']" output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) if "can't use pyrepl" in output: diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index b0d1f12513d404..1222ec6a3c4109 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -4,7 +4,6 @@ import threading import unittest from unittest.mock import patch -from test import support from test.support import import_helper, threading_helper @@ -515,10 +514,6 @@ def test___class___modification_multithreaded(self): an audit hook. """ - if support.Py_GIL_DISABLED: - # gh-124402: On a Free Threaded build, the test takes a few minutes - support.requires('cpu') - class Foo: pass @@ -528,7 +523,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(5000): + for _ in range(200): foo.__class__ = Bar type(foo) foo.__class__ = Foo diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e60e5477d32e1f..9a3cf140d81241 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -548,13 +548,14 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 293799ff68ea05..530c317a852e77 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -654,8 +654,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..eeaf5de2e303f6 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -227,13 +227,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 49d6aa810304fb..ebb65d8c6cf81b 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -211,6 +211,19 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] def test_keywords(self): TA = TypeAliasType(name="TA", value=int) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 91082e6b23c04b..257b7fa95dcb76 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -375,7 +375,7 @@ class X: with self.assertRaises(NotImplementedError): annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): - annotate(annotationlib.Format.SOURCE) + annotate(annotationlib.Format.STRING) with self.assertRaises(NotImplementedError): annotate(None) self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 8c21553e410d8a..433b19593bdd04 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1440,7 +1440,7 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(case(1), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) - self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.SOURCE), 'int') + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') def test_constraints(self): def f[T: (int, str)](): pass @@ -1451,7 +1451,7 @@ def f[T: (int, str)](): pass self.assertEqual(case.evaluate_constraints(1), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) - self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.SOURCE), '(int, str)') + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') def test_const_evaluator(self): T = TypeVar("T", bound=int) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3ac6b97383fcef..2f1f9e86a0bce4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7059,7 +7059,7 @@ class C: self.assertIsInstance(annos['x'], annotationlib.ForwardRef) self.assertEqual(annos['x'].__arg__, 'undefined') - self.assertEqual(get_type_hints(C, format=annotationlib.Format.SOURCE), + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), {'x': 'undefined'}) @@ -7898,7 +7898,7 @@ class Z(NamedTuple): self.assertEqual(Z.__annotations__, annos) self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Z.__annotate__(annotationlib.Format.SOURCE), {"a": "None", "b": "str"}) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) def test_future_annotations(self): code = """ @@ -8241,7 +8241,7 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__annotations__, annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) - self.assertEqual(Emp.__annotate__(annotationlib.Format.SOURCE), {'name': 'str', 'id': 'int'}) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8603,7 +8603,7 @@ class A[T](TypedDict): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8616,7 +8616,7 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) self.assertEqual(A.__annotations__, {'a': T}) - self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8628,7 +8628,7 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) self.assertEqual(A2.__annotations__, {'a': T}) - self.assertEqual(A2.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8640,7 +8640,7 @@ class B(A[KT], total=False): self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) - self.assertEqual(B.__annotate__(annotationlib.Format.SOURCE), {'a': 'T', 'b': 'KT'}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8665,7 +8665,7 @@ class C(B[int]): 'b': KT, 'c': int, }) - self.assertEqual(C.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8689,7 +8689,7 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) - self.assertEqual(Point3D.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'T', 'c': 'KT', @@ -8725,7 +8725,7 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) - self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.SOURCE), { + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { 'a': 'T', 'b': 'KT', 'c': 'int', @@ -8929,7 +8929,7 @@ class A(TypedDict): A.__annotations__ self.assertEqual( - A.__annotate__(annotationlib.Format.SOURCE), + A.__annotate__(annotationlib.Format.STRING), {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', 'z': 'Required[undefined]'}, ) diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index c9c20f008ca5a2..f260769eb8c35e 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 252eef32cd88a4..c924c767042552 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -245,7 +245,7 @@ def _type_repr(obj): if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return annotationlib.value_to_source(obj) + return annotationlib.value_to_string(obj) def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): @@ -1036,7 +1036,7 @@ def evaluate_forward_ref( * Recursively evaluates forward references nested within the type hint. * Rejects certain objects that are not valid type hints. * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *SOURCE* formats. + * Supports the *FORWARDREF* and *STRING* formats. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -1053,7 +1053,7 @@ def evaluate_forward_ref( if type_params is _sentinel: _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") type_params = () - if format == annotationlib.Format.SOURCE: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref @@ -2380,7 +2380,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, hints = {} for base in reversed(obj.__mro__): ann = annotationlib.get_annotations(base, format=format) - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: hints.update(ann) continue if globalns is None: @@ -2404,7 +2404,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, value = _eval_type(value, base_globals, base_locals, base.__type_params__, format=format, owner=obj) hints[name] = value - if include_extras or format is annotationlib.Format.SOURCE: + if include_extras or format is annotationlib.Format.STRING: return hints else: return {k: _strip_annotations(t) for k, t in hints.items()} @@ -2418,7 +2418,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, and not hasattr(obj, '__annotate__') ): raise TypeError(f"{obj!r} is not a module, class, or callable.") - if format is annotationlib.Format.SOURCE: + if format is annotationlib.Format.STRING: return hints if globalns is None: @@ -2937,7 +2937,7 @@ def annotate(format): if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): return checked_types else: - return annotationlib.annotations_to_source(types) + return annotationlib.annotations_to_string(types) return annotate @@ -2972,7 +2972,7 @@ def __new__(cls, typename, bases, ns): def annotate(format): annos = annotationlib.call_annotate_function(original_annotate, format) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3220,13 +3220,13 @@ def __annotate__(format): annos.update(base_annos) if own_annotate is not None: own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) - if format != annotationlib.Format.SOURCE: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == annotationlib.Format.SOURCE: - own = annotationlib.annotations_to_source(own_annotations) + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) else: own = own_checked_annotations annos.update(own) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index bb34c7436047ad..21ca061a77c26f 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -34,6 +34,7 @@ import pkgutil from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -628,7 +629,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None, *, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -2218,7 +2221,7 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() - def reset_mock(self, /, *args, return_value=False, **kwargs): + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): if ( return_value and self._mock_name @@ -2754,7 +2757,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2811,7 +2822,7 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, _name='()', _parent=mock, wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2825,10 +2836,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue child_kwargs = {'spec': original} # Wrap child attributes also. diff --git a/Misc/ACKS b/Misc/ACKS index b2529601a2f71a..d94cbacf888468 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1552,6 +1552,7 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson Izan "TizzySaurus" Robinson Jim Robinson diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst new file mode 100644 index 00000000000000..de62875e16475d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-27-21-44-14.gh-issue-116017.ZY3yBY.rst @@ -0,0 +1,2 @@ +Improved JIT memory consumption by periodically freeing memory used by infrequently-executed code. +This change is especially likely to improve the memory footprint of long-running programs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst new file mode 100644 index 00000000000000..25b47d5fbaefa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-36-45.gh-issue-123339.QcmpSs.rst @@ -0,0 +1,3 @@ +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst new file mode 100644 index 00000000000000..58e79f22ac0f90 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-25-11-53-22.gh-issue-124442.EXC1Ve.rst @@ -0,0 +1,2 @@ +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst new file mode 100644 index 00000000000000..1005c651849f45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-12-19-13.gh-issue-124547.P_SHfU.rst @@ -0,0 +1,3 @@ +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst new file mode 100644 index 00000000000000..4cdbb205c962c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-13-25-01.gh-issue-119180.k_JCX0.rst @@ -0,0 +1,2 @@ +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst new file mode 100644 index 00000000000000..fc3f8af72d87bf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-17-55-34.gh-issue-116510.dhn8w8.rst @@ -0,0 +1,3 @@ +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst new file mode 100644 index 00000000000000..e3741321006548 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-26-18-21-06.gh-issue-116510.FacUWO.rst @@ -0,0 +1,5 @@ +Fix a crash caused by immortal interned strings being shared between +sub-interpreters that use basic single-phase init. In that case, the string +can be used by an interpreter that outlives the interpreter that created and +interned it. For interpreters that share obmalloc state, also share the +interned dict with the main interpreter. diff --git a/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst new file mode 100644 index 00000000000000..f9da7b8a5724f5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-24-11-52-36.gh-issue-124457.yrCjSV.rst @@ -0,0 +1,2 @@ +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. diff --git a/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst new file mode 100644 index 00000000000000..8e1937ab73c31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-14-11-43-31.gh-issue-113878.dmEIN3.rst @@ -0,0 +1,9 @@ +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format +in dataclasses from :class:`tuple` to :class:`dict` +when documentation / metadata is present. diff --git a/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst new file mode 100644 index 00000000000000..e388541f1c2c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-02-20-34-04.gh-issue-123339.czgcSu.rst @@ -0,0 +1,4 @@ +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. diff --git a/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst new file mode 100644 index 00000000000000..a4428e20f3ccdd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-19-00-09-48.gh-issue-84559.IrxvQe.rst @@ -0,0 +1,5 @@ +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to ``"forkserver"`` +(when the platform supports sending file handles over pipes as most do) or +``"spawn"``. Mac and Windows are unchanged as they already default to +``"spawn"``. diff --git a/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst new file mode 100644 index 00000000000000..707c4d651cb5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-17-33-47.gh-issue-104860.O86OSc.rst @@ -0,0 +1,2 @@ +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst new file mode 100644 index 00000000000000..dff902d8c6139a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-12-34-48.gh-issue-124345.s3vKql.rst @@ -0,0 +1,2 @@ +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst new file mode 100644 index 00000000000000..38c030668b6b42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-13-32-16.gh-issue-124176.6hmOPz.rst @@ -0,0 +1,4 @@ +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. diff --git a/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst new file mode 100644 index 00000000000000..45fe4786fa6563 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-21-15-27.gh-issue-123017.dSAr2f.rst @@ -0,0 +1,2 @@ +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst new file mode 100644 index 00000000000000..20ba1534f5e99d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-10-25-57.gh-issue-53834.uyIckw.rst @@ -0,0 +1,4 @@ +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst new file mode 100644 index 00000000000000..4dbf4eb709733d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-12-14-58.gh-issue-124498.Ozxs55.rst @@ -0,0 +1,2 @@ +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst new file mode 100644 index 00000000000000..bb451cdd9ae44c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-08-29.gh-issue-80259.kO5Tw7.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. diff --git a/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst new file mode 100644 index 00000000000000..33ae037ae56b0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-25-18-34-48.gh-issue-124538.nXZk4R.rst @@ -0,0 +1 @@ +Fixed crash when using :func:`gc.get_referents` on a capsule object. diff --git a/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst new file mode 100644 index 00000000000000..801a5fdd4abd4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-26-09-18-09.gh-issue-61181.dwjmch.rst @@ -0,0 +1,2 @@ +Fix support of :ref:`choices` with string value in :mod:`argparse`. Substrings +of the specified string no longer considered valid values. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst new file mode 100644 index 00000000000000..62639a16c52aa0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-15-16-04.gh-issue-116850.dBkR0-.rst @@ -0,0 +1,2 @@ +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes). diff --git a/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst new file mode 100644 index 00000000000000..203868a8fee39c --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-09-27-13-40-25.gh-issue-124609.WaKk8G.rst @@ -0,0 +1 @@ +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 90c72c7baa3f9e..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tainted. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 32373f0799bfeb..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,6 +1068,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbfed59995c21e..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2179,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2358,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2454,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0d91bef2ff9bda..90527d2a3e0350 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5959,6 +5959,8 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) invalid_iso_midnight: PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); return NULL; invalid_string_error: diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index e99a96ab93392e..a33c9793b5ad17 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -122,6 +122,8 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -134,10 +136,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = _PyType_GetModuleByDef2(Py_TYPE(left), Py_TYPE(right), - &_decimal_module); - assert(mod != NULL); - return get_module_state(mod); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); + } + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -183,6 +191,7 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; typedef struct { @@ -203,6 +212,15 @@ typedef struct { #define CTX(v) (&((PyDecContextObject *)v)->ctx) #define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} + Py_LOCAL_INLINE(PyObject *) incr_true(void) @@ -557,7 +575,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -745,7 +763,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -852,7 +870,7 @@ static PyObject * context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } @@ -1011,7 +1029,7 @@ context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1070,7 +1088,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1090,7 +1108,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1135,7 +1153,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1156,7 +1174,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1386,6 +1404,7 @@ context_new(PyTypeObject *type, CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; if (type == state->PyDecContext_Type) { PyObject_GC_Track(self); @@ -1463,7 +1482,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx((PyObject *)self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1554,7 +1573,7 @@ context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1574,7 +1593,7 @@ context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -2015,11 +2034,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2045,7 +2063,7 @@ PyDecType_New(PyTypeObject *type) assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2148,7 +2166,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2172,7 +2191,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2259,7 +2279,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2280,7 +2301,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2298,13 +2320,13 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { PyObject *dec; PyLongObject *l = (PyLongObject *)v; - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2349,7 +2371,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2378,7 +2401,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2410,7 +2434,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2431,7 +2455,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, sign = (copysign(1.0, x) == 1.0) ? 0 : 1; if (isnan(x) || isinf(x)) { - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2548,12 +2572,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2837,7 +2861,7 @@ dec_from_float(PyObject *type, PyObject *pyfloat) static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2848,7 +2872,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2875,7 +2899,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2910,7 +2934,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -2997,7 +3021,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3100,7 +3124,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3155,7 +3179,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3174,7 +3198,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -4414,12 +4438,11 @@ dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4429,6 +4452,13 @@ dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +static PyObject * +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + static PyObject * dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { @@ -5041,6 +5071,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, @@ -5130,7 +5161,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5162,7 +5193,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5198,7 +5229,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5227,7 +5258,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5293,7 +5324,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5348,7 +5379,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5383,7 +5414,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5400,7 +5432,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5426,7 +5458,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5443,7 +5475,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5476,7 +5508,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5573,7 +5605,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5728,6 +5760,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 2b3bd7c3de1176..31cf7bcc09782c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -26,7 +26,7 @@ typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; PyTypeObject *placeholder_type; - PyObject *placeholder; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -76,13 +76,12 @@ static PyMethodDef placeholder_methods[] = { }; static void -placeholder_dealloc(PyObject* placeholder) +placeholder_dealloc(PyObject* self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref Placeholder out of existence. Instead, - * since Placeholder is an immortal object, re-set the reference count. - */ - _Py_SetImmortal(placeholder); + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); } static PyObject * @@ -93,10 +92,26 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + if (state->placeholder == NULL) { - state->placeholder = PyType_GenericNew(type, NULL, NULL); + state->placeholder = Py_NewRef(placeholder); } - return state->placeholder; + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } static PyType_Slot placeholder_type_slots[] = { @@ -105,13 +120,14 @@ static PyType_Slot placeholder_type_slots[] = { {Py_tp_doc, (void *)placeholder_doc}, {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, {0, 0} }; static PyType_Spec placeholder_type_spec = { .name = "functools._PlaceholderType", .basicsize = sizeof(placeholderobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, .slots = placeholder_type_slots }; @@ -1717,13 +1733,17 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->placeholder_type) < 0) { return -1; } - state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); - if (state->placeholder == NULL) { + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { return -1; } - if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) { + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); return -1; } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1769,6 +1789,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1781,6 +1802,7 @@ _functools_clear(PyObject *module) _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 18affdd4875f3b..b2bd9545c1b130 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2146,7 +2146,7 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - uint64_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; @@ -2161,8 +2161,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (uint64_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 3835a3072d96c6..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -295,7 +295,7 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - uint64_t bits; + int64_t bits; size_t keyused; int res; @@ -335,8 +335,8 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (uint64_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..01855aec5e123e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2683,6 +2683,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2796,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3013d4709d96ffe input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index baf2dc439b8959..058f57770755aa 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - uint64_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1680,14 +1680,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (uint64_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1704,13 +1703,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1727,12 +1726,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - uint64_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1742,7 +1741,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2202,8 +2201,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ee59fb73ac1e31..9720c201a184a8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -813,7 +813,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..28965e0f21b7a0 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -317,10 +317,14 @@ static int capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + + if (capsule->traverse_func != NULL) { + return capsule->traverse_func((PyObject*)capsule, visit, arg); + } - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return 0; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f38ab1b2865e99..e50cf8bf1787f9 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3930,13 +3930,13 @@ dict_copy_impl(PyDictObject *self) } /* Copies the values, but does not change the reference - * counts of the objects in the array. */ + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ static PyDictValues * copy_values(PyDictValues *values) { PyDictValues *newvalues = new_values(values->capacity); if (newvalues == NULL) { - PyErr_NoMemory(); return NULL; } newvalues->size = values->size; @@ -7216,6 +7216,13 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) PyDictValues *values = copy_values(mp->ma_values); if (values == NULL) { + /* Out of memory. Clear the dict */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = mp->ma_keys; + set_keys(mp, Py_EMPTY_KEYS); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + STORE_USED(mp, 0); + PyErr_NoMemory(); return -1; } mp->ma_values = values; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index fda62f159c1540..b3910855165494 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3387,8 +3387,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3447,7 +3448,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, diff --git a/Objects/floatobject.c b/Objects/floatobject.c index dc3d8a3e5d0f4b..a48a210adee3b9 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -406,19 +406,16 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - uint64_t nbits64 = _PyLong_NumBits(w); - if (nbits64 > (unsigned int)DBL_MAX_EXP) { + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { /* This Python integer is larger than any finite C double. * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) { - /* This Python integer is so large that uint64_t isn't - * big enough to hold the # of bits. */ - PyErr_Clear(); - } i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; diff --git a/Objects/longobject.c b/Objects/longobject.c index d34c8b6d71ab3f..9beb5884a6932b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -133,8 +133,16 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif PyLongObject * _PyLong_New(Py_ssize_t size) @@ -804,11 +812,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -uint64_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - uint64_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -818,21 +826,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((uint64_t)(ndigits - 1) > UINT64_MAX / (uint64_t)PyLong_SHIFT) - goto Overflow; - result = (uint64_t)(ndigits - 1) * (uint64_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (UINT64_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - /* Very unlikely. Such integer would require more than 2 exbibytes of RAM. */ - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a 64-bit integer"); - return (uint64_t)-1; } PyObject * @@ -1247,15 +1246,12 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) /* Calculates the number of bits required for the *absolute* value * of v. This does not take sign into account, only magnitude. */ - uint64_t nb = _PyLong_NumBits((PyObject *)v); - if (nb == (uint64_t)-1) { - res = -1; - } else { - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up - * multiples of 8 to the next byte, but we add an implied bit for - * the sign and it cancels out. */ - res = (Py_ssize_t)(nb / 8) + 1; - } + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up + * multiples of 8 to the next byte, but we add an implied bit for + * the sign and it cancels out. */ + res = (Py_ssize_t)(nb / 8) + 1; /* Two edge cases exist that are best handled after extracting the * bits. These may result in us reporting overflow when the value @@ -3415,7 +3411,8 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) double _PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; @@ -3432,14 +3429,7 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) *e = 0; return 0.0; } - int msd_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + msd_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (INT64_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (INT64_MAX - 1) / PyLong_SHIFT + 1 || - msd_bits > (INT64_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (int64_t)(a_size - 1) * PyLong_SHIFT + msd_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3468,18 +3458,18 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) */ if (a_bits <= DBL_MANT_DIG + 2) { shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) % PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); - shift_bits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3505,21 +3495,13 @@ _PyLong_Frexp(PyLongObject *a, int64_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == INT64_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3547,7 +3529,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -5217,39 +5201,6 @@ long_bool(PyLongObject *v) return !_PyLong_IsZero(v); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ -static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; -} - /* Inner function for both long_rshift and _PyLong_Rshift, shifting an integer right by PyLong_SHIFT*wordshift + remshift bits. wordshift should be nonnegative. */ @@ -5343,8 +5294,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5355,24 +5305,35 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, uint64_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { if (_PyLong_IsNegative((PyLongObject *)a)) { return PyLong_FromLong(-1); } @@ -5430,8 +5391,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_lshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5442,24 +5402,30 @@ long_lshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return _PyLong_Lshift(a, shiftby); } /* Return a << shiftby. */ PyObject * -_PyLong_Lshift(PyObject *a, uint64_t shiftby) +_PyLong_Lshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } -#if PY_SSIZE_T_MAX <= UINT64_MAX / PyLong_SHIFT - if (shiftby > (uint64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); return NULL; @@ -6213,11 +6179,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - uint64_t nbits = _PyLong_NumBits(self); - if (nbits == (uint64_t)-1) { - return NULL; - } - return PyLong_FromUnsignedLongLong(nbits); + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -6251,40 +6216,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1318ce0319d438..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f14f10ab9c0a46..4d8cca68df946a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -999,8 +999,9 @@ tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *obj) { + _PyTupleIterObject *it = (_PyTupleIterObject *)obj; PyTupleObject *seq; PyObject *item; @@ -1101,7 +1102,7 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 68e481f8e5163b..0e2d9758a5ffae 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1435,6 +1435,9 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } @@ -5207,8 +5210,8 @@ PyType_GetModuleState(PyTypeObject *type) /* Get the module of the first superclass where the module has the * given PyModuleDef. */ -static inline PyObject * -get_module_by_def(PyTypeObject *type, PyModuleDef *def) +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); @@ -5241,7 +5244,7 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -5254,37 +5257,14 @@ get_module_by_def(PyTypeObject *type, PyModuleDef *def) } } END_TYPE_LOCK(); - return res; -} -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *module = get_module_by_def(type, def); - if (module == NULL) { + if (res == NULL) { PyErr_Format( PyExc_TypeError, "PyType_GetModuleByDef: No superclass of '%s' has the given module", type->tp_name); } - return module; -} - -PyObject * -_PyType_GetModuleByDef2(PyTypeObject *left, PyTypeObject *right, - PyModuleDef *def) -{ - PyObject *module = get_module_by_def(left, def); - if (module == NULL) { - module = get_module_by_def(right, def); - if (module == NULL) { - PyErr_Format( - PyExc_TypeError, - "PyType_GetModuleByDef: No superclass of '%s' nor '%s' has " - "the given module", left->tp_name, right->tp_name); - } - } - return module; + return res; } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 09e9ab39364742..51d93ed8b5ba8c 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -168,7 +168,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyObject *value = ((constevaluatorobject *)self)->value; - if (format == 3) { // SOURCE + if (format == 3) { // STRING PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 if (writer == NULL) { return NULL; @@ -1915,7 +1915,16 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + ta->type_params = NULL; + } + else { + ta->type_params = Py_NewRef(type_params); + } ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e9589cfe44f3bf..0f502ccdaf5767 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -282,13 +282,37 @@ hashtable_unicode_compare(const void *key1, const void *key2) } } +/* Return true if this interpreter should share the main interpreter's + intern_dict. That's important for interpreters which load basic + single-phase init extension modules (m_size == -1). There could be interned + immortal strings that are shared between interpreters, due to the + PyDict_Update(mdict, m_copy) call in import_find_extension(). + + It's not safe to deallocate those strings until all interpreters that + potentially use them are freed. By storing them in the main interpreter, we + ensure they get freed after all other interpreters are freed. +*/ +static bool +has_shared_intern_dict(PyInterpreterState *interp) +{ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + return interp != main_interp && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC; +} + static int init_interned_dict(PyInterpreterState *interp) { assert(get_interned_dict(interp) == NULL); - PyObject *interned = interned = PyDict_New(); - if (interned == NULL) { - return -1; + PyObject *interned; + if (has_shared_intern_dict(interp)) { + interned = get_interned_dict(_PyInterpreterState_Main()); + Py_INCREF(interned); + } + else { + interned = PyDict_New(); + if (interned == NULL) { + return -1; + } } _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = interned; return 0; @@ -299,7 +323,10 @@ clear_interned_dict(PyInterpreterState *interp) { PyObject *interned = get_interned_dict(interp); if (interned != NULL) { - PyDict_Clear(interned); + if (!has_shared_intern_dict(interp)) { + // only clear if the dict belongs to this interpreter + PyDict_Clear(interned); + } Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } @@ -15618,6 +15645,13 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); + if (has_shared_intern_dict(interp)) { + // the dict doesn't belong to this interpreter, skip the debug + // checks on it and just clear the pointer to it + clear_interned_dict(interp); + return; + } + #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -16126,8 +16160,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) { struct _Py_unicode_state *state = &interp->unicode; - // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() - assert(get_interned_dict(interp) == NULL); + if (!has_shared_intern_dict(interp)) { + // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() + assert(get_interned_dict(interp) == NULL); + } _PyUnicode_FiniEncodings(&state->fs_codec); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 10ee6b7be23e21..ab2b2d06cca15d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -810,6 +810,7 @@ static void set_most_env_vars(void) #ifdef Py_STATS putenv("PYTHONSTATS=1"); #endif + putenv("PYTHONPERFSUPPORT=1"); } @@ -1844,6 +1845,10 @@ static int test_initconfig_api(void) goto error; } + if (PyInitConfig_SetInt(config, "perf_profiling", 2) < 0) { + goto error; + } + // Set a UTF-8 string (program_name) if (PyInitConfig_SetStr(config, "program_name", PROGRAM_NAME_UTF8) < 0) { goto error; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f5b04757e08bf3..01e208b88eca8b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -169,11 +169,10 @@ safe_multiply(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); - uint64_t wbits = _PyLong_NumBits(w); - if (vbits == (uint64_t)-1 || wbits == (uint64_t)-1) { - return NULL; - } + int64_t vbits = _PyLong_NumBits(v); + int64_t wbits = _PyLong_NumBits(w); + assert(vbits >= 0); + assert(wbits >= 0); if (vbits + wbits > MAX_INT_SIZE) { return NULL; } @@ -215,12 +214,13 @@ safe_power(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && _PyLong_IsPositive((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (vbits > MAX_INT_SIZE / wbits) { + if ((uint64_t)vbits > MAX_INT_SIZE / wbits) { return NULL; } } @@ -234,12 +234,13 @@ safe_lshift(PyObject *v, PyObject *w) if (PyLong_Check(v) && PyLong_Check(w) && !_PyLong_IsZero((PyLongObject *)v) && !_PyLong_IsZero((PyLongObject *)w) ) { - uint64_t vbits = _PyLong_NumBits(v); + int64_t vbits = _PyLong_NumBits(v); size_t wbits = PyLong_AsSize_t(w); - if (vbits == (uint64_t)-1 || wbits == (size_t)-1) { + assert(vbits >= 0); + if (wbits == (size_t)-1) { return NULL; } - if (wbits > MAX_INT_SIZE || vbits > MAX_INT_SIZE - wbits) { + if (wbits > MAX_INT_SIZE || (uint64_t)vbits > MAX_INT_SIZE - wbits) { return NULL; } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0fd396f1319e78..8535306d9c7a03 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4836,6 +4836,14 @@ dummy_func( assert(((_PyExecutorObject *)executor)->vm_data.valid); } + tier2 op(_MAKE_WARM, (--)) { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + } + tier2 op(_FATAL_ERROR, (--)) { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6f4476d055b5ec..1d9381d09dfb62 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1289,6 +1289,12 @@ _Py_HandlePending(PyThreadState *tstate) _Py_RunGC(tstate); } + if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + _Py_Executors_InvalidateCold(tstate->interp); + tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + } + /* GIL drop request */ if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ diff --git a/Python/codecs.c b/Python/codecs.c index 9c0a3fad314cb5..68dc232bb86163 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,6 +16,12 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +static const char *codecs_builtin_error_handlers[] = { + "strict", "ignore", "replace", + "xmlcharrefreplace", "backslashreplace", "namereplace", + "surrogatepass", "surrogateescape", +}; + const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ @@ -618,6 +624,20 @@ int PyCodec_RegisterError(const char *name, PyObject *error) name, error); } +int _PyCodec_UnregisterError(const char *name) +{ + for (size_t i = 0; i < Py_ARRAY_LENGTH(codecs_builtin_error_handlers); ++i) { + if (strcmp(name, codecs_builtin_error_handlers[i]) == 0) { + PyErr_Format(PyExc_ValueError, + "cannot un-register built-in error handler '%s'", name); + return -1; + } + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->codecs.initialized); + return PyDict_PopString(interp->codecs.error_registry, name, NULL); +} + /* Lookup the error handling callback function registered under the name error. As a special case NULL can be passed, in which case the error handling callback for strict encoding will be returned. */ @@ -1470,6 +1490,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) } } }; + // ensure that the built-in error handlers' names are kept in sync + assert(Py_ARRAY_LENGTH(methods) == Py_ARRAY_LENGTH(codecs_builtin_error_handlers)); assert(interp->codecs.initialized == 0); interp->codecs.search_path = PyList_New(0); diff --git a/Python/compile.c b/Python/compile.c index 7b3e6f336e44b1..9826d3fbbde976 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -911,7 +911,17 @@ PyObject * _PyCompile_StaticAttributesAsTuple(compiler *c) { assert(c->u->u_static_attributes); - return PySequence_Tuple(c->u->u_static_attributes); + PyObject *static_attributes_unsorted = PySequence_List(c->u->u_static_attributes); + if (static_attributes_unsorted == NULL) { + return NULL; + } + if (PyList_Sort(static_attributes_unsorted) != 0) { + Py_DECREF(static_attributes_unsorted); + return NULL; + } + PyObject *static_attributes = PySequence_Tuple(static_attributes_unsorted); + Py_DECREF(static_attributes_unsorted); + return static_attributes; } int diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7a9c6ab89c38cc..650bf4533a3a86 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5435,6 +5435,15 @@ break; } + case _MAKE_WARM: { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + break; + } + case _FATAL_ERROR: { assert(0); Py_FatalError("Fatal error uop executed."); diff --git a/Python/gc.c b/Python/gc.c index 024d041437be4a..028657eb8999c1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1944,6 +1944,13 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } +static void +finalize_unlink_gc_head(PyGC_Head *gc) { + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); +} void _PyGC_Fini(PyInterpreterState *interp) @@ -1952,9 +1959,25 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ + /* Prevent a subtle bug that affects sub-interpreters that use basic + * single-phase init extensions (m_size == -1). Those extensions cause objects + * to be shared between interpreters, via the PyDict_Update(mdict, m_copy) call + * in import_find_extension(). + * + * If they are GC objects, their GC head next or prev links could refer to + * the interpreter _gc_runtime_state PyGC_Head nodes. Those nodes go away + * when the interpreter structure is freed and so pointers to them become + * invalid. If those objects are still used by another interpreter and + * UNTRACK is called on them, a crash will happen. We untrack the nodes + * here to avoid that. + * + * This bug was originally fixed when reported as gh-90228. The bug was + * re-introduced in gh-94673. + */ + finalize_unlink_gc_head(&gcstate->young.head); + finalize_unlink_gc_head(&gcstate->old[0].head); + finalize_unlink_gc_head(&gcstate->old[1].head); + finalize_unlink_gc_head(&gcstate->permanent_generation.head); } /* for debugging */ diff --git a/Python/initconfig.c b/Python/initconfig.c index d93244f7f41084..58ac5e7d7eaeff 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -150,7 +150,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), SPEC(program_name, WSTR, READ_ONLY, NO_SYS), SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), diff --git a/Python/optimizer.c b/Python/optimizer.c index bb7a90b3204f40..978649faa04d45 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -565,6 +565,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; for (;;) { @@ -1194,6 +1195,9 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil executor->jit_code = NULL; executor->jit_side_entry = NULL; executor->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + executor->vm_data.warm = true; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); return NULL; @@ -1659,4 +1663,42 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) } } +void +_Py_Executors_InvalidateCold(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + + if (!exec->vm_data.warm && PyList_Append(invalidate, (PyObject *)exec) < 0) { + goto error; + } + else { + exec->vm_data.warm = false; + } + + exec = next; + } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback + _Py_Executors_InvalidateAll(interp, 0); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a6cfa271ae6758..4d172e3c762704 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2381,6 +2381,10 @@ break; } + case _MAKE_WARM: { + break; + } + case _FATAL_ERROR: { break; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27faf723745c21..8aebbe5c405ffe 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2503,18 +2503,12 @@ finalize_subinterpreters(void) static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *ann_dict; + PyObject *m, *d; m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); d = PyModule_GetDict(m); - ann_dict = PyDict_New(); - if ((ann_dict == NULL) || - (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - return _PyStatus_ERR("Failed to initialize __main__.__annotations__"); - } - Py_DECREF(ann_dict); int has_builtins = PyDict_ContainsString(d, "__builtins__"); if (has_builtins < 0) { diff --git a/Python/pystate.c b/Python/pystate.c index 6bf7ebeb75ff73..6b85e5a64fefcf 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -660,6 +660,7 @@ init_interpreter(PyInterpreterState *interp, #ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; + interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; #endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 882918fafb1edd..b5b6de0e7dc2dc 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -433,7 +433,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - 'do not use &_PyID or &_Py_STR for one-character latin-1 ' + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name @@ -442,7 +442,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': overlap = identifiers & set(strings.keys()) if overlap: raise ValueError( - 'do not use both _PyID and _Py_DECLARE_STR for the same string: ' + 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: ' + repr(overlap)) return identifiers, strings diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index f4dc807198a8ef..e6c599a2ac4a46 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -345,6 +345,7 @@ Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - +Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LOCATION - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index aabe205125856c..a4ce207703edcd 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -540,6 +540,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyList_FromStackRefSteal", "_PyTuple_FromArraySteal", "_PyTuple_FromStackRefSteal", + "_Py_set_eval_breaker_bit" ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index e37ee943999785..6c7b48f1f37865 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -139,6 +139,9 @@ async def _compile( "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", + # On aarch64 Linux, intrinsics were being emitted and this flag + # was required to disable them. + "-mno-outline-atomics", "-std=c11", *self.args, ] diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 95eda863cb62c6..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK"