diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index bbae378f7b994e..0800401f4cd113 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -21,6 +21,7 @@ apt-get -yq install \ libssl-dev \ lzma \ lzma-dev \ + strace \ tk-dev \ uuid-dev \ xvfb \ diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 0a18e63c7e7a2c..75a05b4197c460 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -34,18 +34,20 @@ There are a few functions specific to Python functions. Return a new function object associated with the code object *code*. *globals* must be a dictionary with the global variables accessible to the function. - The function's docstring and name are retrieved from the code object. *__module__* + The function's docstring and name are retrieved from the code object. + :func:`~function.__module__` is retrieved from *globals*. The argument defaults, annotations and closure are - set to ``NULL``. *__qualname__* is set to the same value as the code object's - :attr:`~codeobject.co_qualname` field. + set to ``NULL``. :attr:`~function.__qualname__` is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; - if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's :attr:`~codeobject.co_qualname` field. + :attr:`~function.__qualname__` attribute. + *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the :attr:`!__qualname__` attribute is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. versionadded:: 3.3 @@ -62,11 +64,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - Return a :term:`borrowed reference` to the *__module__* attribute of the - function object *op*. It can be *NULL*. + Return a :term:`borrowed reference` to the :attr:`~function.__module__` + attribute of the :ref:`function object ` *op*. + It can be *NULL*. - This is normally a string containing the module name, but can be set to any - other object by Python code. + This is normally a :class:`string ` containing the module name, + but can be set to any other object by Python code. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 528813c255c0a5..7d82f7839dfcd7 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -419,15 +419,15 @@ Accessing attributes of extension types The string should be static, no copy is made of it. - .. c:member:: Py_ssize_t offset - - The offset in bytes that the member is located on the type’s object struct. - .. c:member:: int type The type of the member in the C struct. See :ref:`PyMemberDef-types` for the possible values. + .. c:member:: Py_ssize_t offset + + The offset in bytes that the member is located on the type’s object struct. + .. c:member:: int flags Zero or more of the :ref:`PyMemberDef-flags`, combined using bitwise OR. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 1134686c947d66..be8c7e6c827f57 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -153,7 +153,8 @@ on an arbitrary object ``o``: unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. + :attr:`o.__globals__ ` as the globals when calling + :func:`eval`. However, not all string values used as annotations can be successfully turned into Python values by :func:`eval`. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f732aaea729d40..87274a5133d1cf 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1342,7 +1342,8 @@ Using the non-data descriptor protocol, a pure Python version of The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``. +function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, +:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. .. testcode:: :hide: @@ -1522,8 +1523,9 @@ Using the non-data descriptor protocol, a pure Python version of The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look -like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``, -and ``__annotations__``. +like the wrapped function: :attr:`~function.__name__`, +:attr:`~function.__qualname__`, :attr:`~function.__doc__`, +and :attr:`~function.__annotations__`. Member objects and __slots__ diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index a2c1eb00d8b33d..e91972fe621a48 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -18,16 +18,17 @@ cookie value. The module formerly strictly applied the parsing rules described in the :rfc:`2109` and :rfc:`2068` specifications. It has since been discovered that -MSIE 3.0x doesn't follow the character rules outlined in those specs and also -many current day browsers and servers have relaxed parsing rules when comes to -Cookie handling. As a result, the parsing rules used are a bit less strict. +MSIE 3.0x didn't follow the character rules outlined in those specs; many +current-day browsers and servers have also relaxed parsing rules when it comes +to cookie handling. As a result, this module now uses parsing rules that are a +bit less strict than they once were. The character set, :data:`string.ascii_letters`, :data:`string.digits` and ``!#$%&'*+-.^_`|~:`` denote the set of valid characters allowed by this module -in Cookie name (as :attr:`~Morsel.key`). +in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed ':' as a valid Cookie name character. + Allowed ':' as a valid cookie name character. .. note:: @@ -54,9 +55,10 @@ in Cookie name (as :attr:`~Morsel.key`). .. class:: SimpleCookie([input]) - This class derives from :class:`BaseCookie` and overrides :meth:`value_decode` - and :meth:`value_encode`. SimpleCookie supports strings as cookie values. - When setting the value, SimpleCookie calls the builtin :func:`str()` to convert + This class derives from :class:`BaseCookie` and overrides :meth:`~BaseCookie.value_decode` + and :meth:`~BaseCookie.value_encode`. :class:`!SimpleCookie` supports + strings as cookie values. When setting the value, :class:`!SimpleCookie` + calls the builtin :func:`str` to convert the value to a string. Values received from HTTP are kept as strings. .. seealso:: @@ -129,17 +131,17 @@ Morsel Objects Abstract a key/value pair, which has some :rfc:`2109` attributes. Morsels are dictionary-like objects, whose set of keys is constant --- the valid - :rfc:`2109` attributes, which are - - * ``expires`` - * ``path`` - * ``comment`` - * ``domain`` - * ``max-age`` - * ``secure`` - * ``version`` - * ``httponly`` - * ``samesite`` + :rfc:`2109` attributes, which are: + + .. attribute:: expires + path + comment + domain + max-age + secure + version + httponly + samesite The attribute :attr:`httponly` specifies that the cookie is only transferred in HTTP requests, and is not accessible through JavaScript. This is intended @@ -152,7 +154,7 @@ Morsel Objects The keys are case-insensitive and their default value is ``''``. .. versionchanged:: 3.5 - :meth:`~Morsel.__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` + :meth:`!__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value` into account. .. versionchanged:: 3.7 diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8381e508139fbd..f8b3e39c4f54f0 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1225,9 +1225,10 @@ Classes and functions * If ``obj`` is a class, ``globals`` defaults to ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to ``obj.__globals__``, + * If ``obj`` is a callable, ``globals`` defaults to + :attr:`obj.__globals__ `, although if ``obj`` is a wrapped function (using - ``functools.update_wrapper()``) it is first unwrapped. + :func:`functools.update_wrapper`) it is first unwrapped. Calling ``get_annotations`` is best practice for accessing the annotations dict of any object. See :ref:`annotations-howto` for diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1265b5b12e492d..9028ff5c134fa9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5384,10 +5384,10 @@ Code objects are used by the implementation to represent "pseudo-compiled" executable Python code such as a function body. They differ from function objects because they don't contain a reference to their global execution environment. Code objects are returned by the built-in :func:`compile` function -and can be extracted from function objects through their :attr:`__code__` -attribute. See also the :mod:`code` module. +and can be extracted from function objects through their +:attr:`~function.__code__` attribute. See also the :mod:`code` module. -Accessing ``__code__`` raises an :ref:`auditing event ` +Accessing :attr:`~function.__code__` raises an :ref:`auditing event ` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: @@ -5461,8 +5461,9 @@ It is written as ``NotImplemented``. Internal Objects ---------------- -See :ref:`types` for this information. It describes stack frame objects, -traceback objects, and slice objects. +See :ref:`types` for this information. It describes +:ref:`stack frame objects `, +:ref:`traceback objects `, and slice objects. .. _specialattrs: diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index b68a78e8267bcc..9add8500c7788c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -18,7 +18,7 @@ This module creates temporary files and directories. It works on all supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, :class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and +:term:`context managers `. :func:`mkstemp` and :func:`mkdtemp` are lower-level functions which require manual cleanup. All the user-callable functions and constructors take additional arguments which @@ -41,7 +41,7 @@ The module defines the following user-callable items: this; your code should not rely on a temporary file created using this function having or not having a visible name in the file system. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the file object the temporary file will be removed from the filesystem. @@ -87,9 +87,9 @@ The module defines the following user-callable items: determine whether and how the named file should be automatically deleted. The returned object is always a :term:`file-like object` whose :attr:`!file` - attribute is the underlying true file object. This :term:`file-like object` + attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. The - name of the temporary file can be retrieved from the :attr:`name` attribute + name of the temporary file can be retrieved from the :attr:`!name` attribute of the returned file-like object. On Unix, unlike with the :func:`TemporaryFile`, the directory entry does not get unlinked immediately after the file creation. @@ -151,18 +151,20 @@ The module defines the following user-callable items: contents are written to disk and operation proceeds as with :func:`TemporaryFile`. - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. + .. method:: SpooledTemporaryFile.rollover - The returned object is a file-like object whose :attr:`_file` attribute + The resulting file has one additional method, :meth:`!rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`!_file` attribute is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This + object, depending on whether :meth:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. + the truncate method now accepts a *size* argument. .. versionchanged:: 3.8 Added *errors* parameter. @@ -176,24 +178,28 @@ The module defines the following user-callable items: .. class:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False, *, delete=True) This class securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see + The resulting object can be used as a :term:`context manager` (see :ref:`tempfile-examples`). On completion of the context or destruction of the temporary directory object, the newly created temporary directory and all its contents are removed from the filesystem. - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled - exceptions during explicit or implicit cleanup (such as a - :exc:`PermissionError` removing open files on Windows) will be ignored, - and the remaining removable items deleted on a "best-effort" basis. - Otherwise, errors will be raised in whatever context cleanup occurs - (the :func:`cleanup` call, exiting the context manager, when the object - is garbage-collected or during interpreter shutdown). + .. attribute:: TemporaryDirectory.name + + The directory name can be retrieved from the :attr:`!name` attribute of the + returned object. When the returned object is used as a :term:`context manager`, the + :attr:`!name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. + + .. method:: TemporaryDirectory.cleanup + + The directory can be explicitly cleaned up by calling the + :meth:`!cleanup` method. If *ignore_cleanup_errors* is true, any unhandled + exceptions during explicit or implicit cleanup (such as a + :exc:`PermissionError` removing open files on Windows) will be ignored, + and the remaining removable items deleted on a "best-effort" basis. + Otherwise, errors will be raised in whatever context cleanup occurs + (the :meth:`!cleanup` call, exiting the context manager, when the object + is garbage-collected or during interpreter shutdown). The *delete* parameter can be used to disable cleanup of the directory tree upon exiting the context. While it may seem unusual for a context manager diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 2d5ea19b2cb892..14cade690fc773 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -16,7 +16,8 @@ interpreter. .. index:: pair: object; traceback -The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, +The module uses :ref:`traceback objects ` --- these are +objects of type :class:`types.TracebackType`, which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. .. seealso:: @@ -212,7 +213,8 @@ The module defines the following functions: .. function:: walk_tb(tb) - Walk a traceback following ``tb_next`` yielding the frame and line number + Walk a traceback following :attr:`~traceback.tb_next` yielding the frame and + line number for each frame. This helper is used with :meth:`StackSummary.extract`. .. versionadded:: 3.5 diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 22766462822af9..8ce67cf77253c3 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -378,11 +378,8 @@ Standard names are defined for the following types: .. data:: FrameType - The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a - traceback object. - - See :ref:`the language reference ` for details of the - available attributes and operations. + The type of :ref:`frame objects ` such as found in + :attr:`tb.tb_frame ` if ``tb`` is a traceback object. .. data:: GetSetDescriptorType diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 016369d2b89d2c..ca1ea455f0acfc 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -84,12 +84,12 @@ alone XML-RPC servers. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. @@ -298,12 +298,12 @@ requests sent to Python CGI scripts. Register a function that can respond to XML-RPC requests. If *name* is given, it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain + :attr:`function.__name__` will be used. *name* is a string, and may contain characters not legal in Python identifiers, including the period character. This method can also be used as a decorator. When used as a decorator, *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. + *name*. If no *name* is given, :attr:`function.__name__` will be used. .. versionchanged:: 3.7 :meth:`register_function` can be used as a decorator. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 8f6481339837a0..7a735095bdecb2 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1261,7 +1261,8 @@ except that the original function is not temporarily bound to the name ``func``. A list of :ref:`type parameters ` may be given in square brackets between the function's name and the opening parenthesis for its parameter list. This indicates to static type checkers that the function is generic. At runtime, -the type parameters can be retrieved from the function's ``__type_params__`` +the type parameters can be retrieved from the function's +:attr:`~function.__type_params__` attribute. See :ref:`generic-functions` for more. .. versionchanged:: 3.12 @@ -1868,8 +1869,8 @@ like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. are mappings. .. [#] A string literal appearing as the first statement in the function body is - transformed into the function's ``__doc__`` attribute and therefore the - function's :term:`docstring`. + transformed into the function's :attr:`~function.__doc__` attribute and + therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is transformed into the namespace's ``__doc__`` item and therefore the class's diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 27d379a8b70f31..de79d72a05c68a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -534,9 +534,34 @@ section :ref:`function`). It should be called with an argument list containing the same number of items as the function's formal parameter list. -Special attributes: +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: __closure__ (function attribute) + single: __globals__ (function attribute) + pair: global; namespace + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__globals__ + - A reference to the :class:`dictionary ` that holds the function's + :ref:`global variables ` -- the global namespace of the module + in which the function was defined. + + * - .. attribute:: function.__closure__ + - ``None`` or a :class:`tuple` of cells that contain bindings for the + function's free variables. -.. tabularcolumns:: |l|L|l| + A cell object has the attribute ``cell_contents``. + This can be used to get the value of the cell, as well as set the value. + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. index:: single: __doc__ (function attribute) @@ -544,96 +569,78 @@ Special attributes: single: __module__ (function attribute) single: __dict__ (function attribute) single: __defaults__ (function attribute) - single: __closure__ (function attribute) single: __code__ (function attribute) - single: __globals__ (function attribute) single: __annotations__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) - pair: global; namespace -+-------------------------+-------------------------------+-----------+ -| Attribute | Meaning | | -+=========================+===============================+===========+ -| :attr:`__doc__` | The function's documentation | Writable | -| | string, or ``None`` if | | -| | unavailable; not inherited by | | -| | subclasses. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's name. | Writable | -| __name__` | | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's | Writable | -| __qualname__` | :term:`qualified name`. | | -| | | | -| | .. versionadded:: 3.3 | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__module__` | The name of the module the | Writable | -| | function was defined in, or | | -| | ``None`` if unavailable. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__defaults__` | A tuple containing default | Writable | -| | argument values for those | | -| | arguments that have defaults, | | -| | or ``None`` if no arguments | | -| | have a default value. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__code__` | The code object representing | Writable | -| | the compiled function body. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__globals__` | A reference to the dictionary | Read-only | -| | that holds the function's | | -| | global variables --- the | | -| | global namespace of the | | -| | module in which the function | | -| | was defined. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~object.__dict__`| The namespace supporting | Writable | -| | arbitrary function | | -| | attributes. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | -| | that contain bindings for the | | -| | function's free variables. | | -| | See below for information on | | -| | the ``cell_contents`` | | -| | attribute. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__annotations__` | A dict containing annotations | Writable | -| | of parameters. The keys of | | -| | the dict are the parameter | | -| | names, and ``'return'`` for | | -| | the return annotation, if | | -| | provided. For more | | -| | information on working with | | -| | this attribute, see | | -| | :ref:`annotations-howto`. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__kwdefaults__` | A dict containing defaults | Writable | -| | for keyword-only parameters. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__type_params__` | A tuple containing the | Writable | -| | :ref:`type parameters | | -| | ` of a | | -| | :ref:`generic function | | -| | `. | | -+-------------------------+-------------------------------+-----------+ - -Most of the attributes labelled "Writable" check the type of the assigned value. +Most of these attributes check the type of the assigned value: + +.. list-table:: + :header-rows: 1 + + * - Attribute + - Meaning + + * - .. attribute:: function.__doc__ + - The function's documentation string, or ``None`` if unavailable. + Not inherited by subclasses. + + * - .. attribute:: function.__name__ + - The function's name. + See also: :attr:`__name__ attributes `. + + * - .. attribute:: function.__qualname__ + - The function's :term:`qualified name`. + See also: :attr:`__qualname__ attributes `. + + .. versionadded:: 3.3 + + * - .. attribute:: function.__module__ + - The name of the module the function was defined in, + or ``None`` if unavailable. + + * - .. attribute:: function.__defaults__ + - A :class:`tuple` containing default parameter values + for those parameters that have defaults, + or ``None`` if no parameters have a default value. + + * - .. attribute:: function.__code__ + - The :ref:`code object ` representing + the compiled function body. + + * - .. attribute:: function.__dict__ + - The namespace supporting arbitrary function attributes. + See also: :attr:`__dict__ attributes `. + + * - .. attribute:: function.__annotations__ + - A :class:`dictionary ` containing annotations of parameters. + The keys of the dictionary are the parameter names, + and ``'return'`` for the return annotation, if provided. + See also: :ref:`annotations-howto`. + + * - .. attribute:: function.__kwdefaults__ + - A :class:`dictionary ` containing defaults for keyword-only + parameters. + + * - .. attribute:: function.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters ` of + a :ref:`generic function `. Function objects also support getting and setting arbitrary attributes, which can be used, for example, to attach metadata to functions. Regular attribute -dot-notation is used to get and set such attributes. *Note that the current -implementation only supports function attributes on user-defined functions. -Function attributes on built-in functions may be supported in the future.* +dot-notation is used to get and set such attributes. -A cell object has the attribute ``cell_contents``. This can be used to get -the value of the cell, as well as set the value. +.. impl-detail:: + + CPython's current implementation only supports function attributes + on user-defined functions. Function attributes on + :ref:`built-in functions ` may be supported in the + future. Additional information about a function's definition can be retrieved from its -code object; see the description of internal types below. The -:data:`cell ` type can be accessed in the :mod:`types` -module. +:ref:`code object ` +(accessible via the :attr:`~function.__code__` attribute). .. _instance-methods: @@ -665,15 +672,17 @@ Special read-only attributes: :ref:`bound ` * - .. attribute:: method.__func__ - - Refers to the original function object + - Refers to the original :ref:`function object ` * - .. attribute:: method.__doc__ - - The method's documentation (same as :attr:`!method.__func__.__doc__`). + - The method's documentation + (same as :attr:`method.__func__.__doc__ `). A :class:`string ` if the original function had a docstring, else ``None``. * - .. attribute:: method.__name__ - - The name of the method (same as :attr:`!method.__func__.__name__`) + - The name of the method + (same as :attr:`method.__func__.__name__ `) * - .. attribute:: method.__module__ - The name of the module the method was defined in, or ``None`` if @@ -779,6 +788,8 @@ is raised and the asynchronous iterator will have reached the end of the set of values to be yielded. +.. _builtin-functions: + Built-in functions ^^^^^^^^^^^^^^^^^^ @@ -791,10 +802,14 @@ A built-in function object is a wrapper around a C function. Examples of built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a standard built-in module). The number and type of the arguments are determined by the C function. Special read-only attributes: -:attr:`__doc__` is the function's documentation string, or ``None`` if -unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is -set to ``None`` (but see the next item); :attr:`__module__` is the name of -the module the function was defined in or ``None`` if unavailable. + +* :attr:`!__doc__` is the function's documentation string, or ``None`` if + unavailable. See :attr:`function.__doc__`. +* :attr:`!__name__` is the function's name. See :attr:`function.__name__`. +* :attr:`!__self__` is set to ``None`` (but see the next item). +* :attr:`!__module__` is the name of + the module the function was defined in or ``None`` if unavailable. + See :attr:`function.__module__`. .. _builtin-methods: @@ -844,7 +859,8 @@ the :ref:`import system ` as invoked either by the :keyword:`import` statement, or by calling functions such as :func:`importlib.import_module` and built-in :func:`__import__`. A module object has a namespace implemented by a -dictionary object (this is the dictionary referenced by the ``__globals__`` +:class:`dictionary ` object (this is the dictionary referenced by the +:attr:`~function.__globals__` attribute of functions defined in the module). Attribute references are translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object does not contain the code object used @@ -1244,8 +1260,9 @@ Frame objects .. index:: pair: object; frame -Frame objects represent execution frames. They may occur in traceback objects -(see below), and are also passed to registered trace functions. +Frame objects represent execution frames. They may occur in +:ref:`traceback objects `, +and are also passed to registered trace functions. .. index:: single: f_back (frame attribute) @@ -1357,26 +1374,30 @@ Traceback objects single: sys.exception single: sys.last_traceback -Traceback objects represent a stack trace of an exception. A traceback object +Traceback objects represent the stack trace of an :ref:`exception `. +A traceback object is implicitly created when an exception occurs, and may also be explicitly created by calling :class:`types.TracebackType`. +.. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code. + For implicitly created tracebacks, when the search for an exception handler unwinds the execution stack, at each unwound level a traceback object is inserted in front of the current traceback. When an exception handler is entered, the stack trace is made available to the program. (See section :ref:`try`.) It is accessible as the third item of the -tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute +tuple returned by :func:`sys.exc_info`, and as the ``__traceback__`` attribute of the caught exception. When the program contains no suitable handler, the stack trace is written (nicely formatted) to the standard error stream; if the interpreter is interactive, it is also made available to the user -as ``sys.last_traceback``. +as :data:`sys.last_traceback`. For explicitly created tracebacks, it is up to the creator of the traceback -to determine how the ``tb_next`` attributes should be linked to form a -full stack trace. +to determine how the :attr:`~traceback.tb_next` attributes should be linked to +form a full stack trace. .. index:: single: tb_frame (traceback attribute) @@ -1385,27 +1406,40 @@ full stack trace. pair: statement; try Special read-only attributes: -:attr:`tb_frame` points to the execution frame of the current level; -:attr:`tb_lineno` gives the line number where the exception occurred; -:attr:`tb_lasti` indicates the precise instruction. + +.. list-table:: + + * - .. attribute:: traceback.tb_frame + - Points to the execution :ref:`frame ` of the current + level. + + Accessing this attribute raises an + :ref:`auditing event ` ``object.__getattr__`` with arguments + ``obj`` and ``"tb_frame"``. + + * - .. attribute:: traceback.tb_lineno + - Gives the line number where the exception occurred + + * - .. attribute:: traceback.tb_lasti + - Indicates the "precise instruction". + The line number and last instruction in the traceback may differ from the -line number of its frame object if the exception occurred in a +line number of its :ref:`frame object ` if the exception +occurred in a :keyword:`try` statement with no matching except clause or with a -finally clause. - -Accessing ``tb_frame`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. +:keyword:`finally` clause. .. index:: single: tb_next (traceback attribute) -Special writable attribute: :attr:`tb_next` is the next level in the stack -trace (towards the frame where the exception occurred), or ``None`` if -there is no next level. +.. attribute:: traceback.tb_next -.. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. + The special writable attribute :attr:`!tb_next` is the next level in the + stack trace (towards the frame where the exception occurred), or ``None`` if + there is no next level. + + .. versionchanged:: 3.7 + This attribute is now writable Slice objects @@ -1873,7 +1907,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. note:: This method may still be bypassed when looking up special methods as the - result of implicit invocation via language syntax or built-in functions. + result of implicit invocation via language syntax or + :ref:`built-in functions `. See :ref:`special-lookup`. .. audit-event:: object.__getattr__ obj,name object.__getattribute__ diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 3e07d16068a627..0adfb0365934e4 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -708,10 +708,12 @@ and formatted string literals may be concatenated with plain string literals. single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals + .. _f-strings: +.. _formatted-string-literals: -Formatted string literals -------------------------- +f-strings +--------- .. versionadded:: 3.6 diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 5ef68cc089d436..511648ab6991c6 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -57,7 +57,6 @@ Doc/library/ftplib.rst Doc/library/functools.rst Doc/library/http.client.rst Doc/library/http.cookiejar.rst -Doc/library/http.cookies.rst Doc/library/http.server.rst Doc/library/importlib.rst Doc/library/inspect.rst @@ -94,7 +93,6 @@ Doc/library/string.rst Doc/library/subprocess.rst Doc/library/syslog.rst Doc/library/tarfile.rst -Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 3551bfa4c0f133..42c2f10e0be260 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -126,7 +126,7 @@ def add_annotations(self, app, doctree): f"Object type mismatch in limited API annotation " f"for {name}: {record['role']!r} != {objtype!r}") stable_added = record['added'] - message = ' Part of the ' + message = sphinx_gettext(' Part of the ') emph_node = nodes.emphasis(message, message, classes=['stableabi']) ref_node = addnodes.pending_xref( @@ -134,40 +134,40 @@ def add_annotations(self, app, doctree): reftype='ref', refexplicit="False") struct_abi_kind = record['struct_abi_kind'] if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text('Limited API') + ref_node += nodes.Text(sphinx_gettext('Limited API')) else: - ref_node += nodes.Text('Stable ABI') + ref_node += nodes.Text(sphinx_gettext('Stable ABI')) emph_node += ref_node if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' (as an opaque struct)') + emph_node += nodes.Text(sphinx_gettext(' (as an opaque struct)')) elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' (including all members)') + emph_node += nodes.Text(sphinx_gettext(' (including all members)')) if record['ifdef_note']: emph_node += nodes.Text(' ' + record['ifdef_note']) if stable_added == '3.2': # Stable ABI was introduced in 3.2. pass else: - emph_node += nodes.Text(f' since version {stable_added}') + emph_node += nodes.Text(sphinx_gettext(' since version %s') % stable_added) emph_node += nodes.Text('.') if struct_abi_kind == 'members': emph_node += nodes.Text( - ' (Only some members are part of the stable ABI.)') + sphinx_gettext(' (Only some members are part of the stable ABI.)')) node.insert(0, emph_node) # Unstable API annotation. if name.startswith('PyUnstable'): warn_node = nodes.admonition( classes=['unstable-c-api', 'warning']) - message = 'This is ' + message = sphinx_gettext('This is ') emph_node = nodes.emphasis(message, message) ref_node = addnodes.pending_xref( 'Unstable API', refdomain="std", reftarget='unstable-c-api', reftype='ref', refexplicit="False") - ref_node += nodes.Text('Unstable API') + ref_node += nodes.Text(sphinx_gettext('Unstable API')) emph_node += ref_node - emph_node += nodes.Text('. It may change without warning in minor releases.') + emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) warn_node += emph_node node.insert(0, warn_node) diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index bab4aaeb4604b8..3a0acab8836b11 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -9,6 +9,16 @@ In extensions/c_annotations.py: +{% trans %} Part of the {% endtrans %} +{% trans %}Limited API{% endtrans %} +{% trans %}Stable ABI{% endtrans %} +{% trans %} (as an opaque struct){% endtrans %} +{% trans %} (including all members){% endtrans %} +{% trans %} since version %s{% endtrans %} +{% trans %} (Only some members are part of the stable ABI.){% endtrans %} +{% trans %}This is {% endtrans %} +{% trans %}Unstable API{% endtrans %} +{% trans %}. It may change without warning in minor releases.{% endtrans %} {% trans %}Return value: Always NULL.{% endtrans %} {% trans %}Return value: New reference.{% endtrans %} {% trans %}Return value: Borrowed reference.{% endtrans %} diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index f0e1ded75a9d27..6d2d3cc02b8768 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -424,7 +424,8 @@ PEP 232: Function Attributes In Python 2.1, functions can now have arbitrary information attached to them. People were often using docstrings to hold information about functions and -methods, because the ``__doc__`` attribute was the only way of attaching any +methods, because the :attr:`~function.__doc__` attribute was the only way of +attaching any information to a function. For example, in the Zope web application server, functions are marked as safe for public access by having a docstring, and in John Aycock's SPARK parsing framework, docstrings hold parts of the BNF grammar diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index cab321c3e54d18..a7b0c6e5d76a6f 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -324,7 +324,8 @@ function, as previously described. In other words, ``@A @B @C(args)`` becomes:: Getting this right can be slightly brain-bending, but it's not too difficult. -A small related change makes the :attr:`func_name` attribute of functions +A small related change makes the :attr:`func_name ` +attribute of functions writable. This attribute is used to display function names in tracebacks, so decorators should change the name of any new function that's constructed and returned. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index b0c2529e780213..5953b32c6aaa18 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -779,14 +779,15 @@ Operators And Special Methods * Removed support for :attr:`__members__` and :attr:`__methods__`. -* The function attributes named :attr:`func_X` have been renamed to - use the :data:`__X__` form, freeing up these names in the function +* The function attributes named :attr:`!func_X` have been renamed to + use the :attr:`!__X__` form, freeing up these names in the function attribute namespace for user-defined attributes. To wit, - :attr:`func_closure`, :attr:`func_code`, :attr:`func_defaults`, - :attr:`func_dict`, :attr:`func_doc`, :attr:`func_globals`, - :attr:`func_name` were renamed to :attr:`__closure__`, - :attr:`__code__`, :attr:`__defaults__`, :attr:`~object.__dict__`, - :attr:`__doc__`, :attr:`__globals__`, :attr:`~definition.__name__`, + :attr:`!func_closure`, :attr:`!func_code`, :attr:`!func_defaults`, + :attr:`!func_dict`, :attr:`!func_doc`, :attr:`!func_globals`, + :attr:`!func_name` were renamed to :attr:`~function.__closure__`, + :attr:`~function.__code__`, :attr:`~function.__defaults__`, + :attr:`~function.__dict__`, :attr:`~function.__doc__`, + :attr:`~function.__globals__`, :attr:`~function.__name__`, respectively. * :meth:`!__nonzero__` is now :meth:`~object.__bool__`. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4401deb0768c11..00f396846e29bd 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -483,7 +483,7 @@ Deprecated (as has always been the case for an executing frame). (Contributed by Irit Katriel in :gh:`79932`.) -* Assignment to a function's ``__code__`` attribute where the new code +* Assignment to a function's :attr:`~function.__code__` attribute where the new code object's type does not match the function's type, is deprecated. The different types are: plain function, generator, async generator and coroutine. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index df32b76b6d7b03..5ef76968e9d86b 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -791,8 +791,9 @@ functools * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to - be introspected. It also copies :attr:`__annotations__` if defined. And now - it also gracefully skips over missing attributes such as :attr:`__doc__` which + be introspected. It also copies :attr:`~function.__annotations__` if + defined. And now it also gracefully skips over missing attributes such as + :attr:`~function.__doc__` which might not be defined for the wrapped callable. In the above example, the cache can be removed by recovering the original diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 2ddab76814369e..9f9cf7fafdc19e 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2405,8 +2405,8 @@ Changes in the Python API storage). (:issue:`17094`.) * Parameter names in ``__annotations__`` dicts are now mangled properly, - similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in - :issue:`20625`.) + similarly to :attr:`~function.__kwdefaults__`. + (Contributed by Yury Selivanov in :issue:`20625`.) * :attr:`hashlib.hash.name` now always returns the identifier in lower case. Previously some builtin hashes had uppercase names, but now that it is a diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index ae6affcab664c6..a32866094ffeb5 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -1947,7 +1947,8 @@ traceback --------- New :func:`~traceback.walk_stack` and :func:`~traceback.walk_tb` -functions to conveniently traverse frame and traceback objects. +functions to conveniently traverse frame and +:ref:`traceback objects `. (Contributed by Robert Collins in :issue:`17911`.) New lightweight classes: :class:`~traceback.TracebackException`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 7a74f9c1685c31..616e51571388a8 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -525,8 +525,8 @@ Other Language Changes * In order to better support dynamic creation of stack traces, :class:`types.TracebackType` can now be instantiated from Python code, and - the ``tb_next`` attribute on :ref:`tracebacks ` is now - writable. + the :attr:`~traceback.tb_next` attribute on + :ref:`tracebacks ` is now writable. (Contributed by Nathaniel J. Smith in :issue:`30579`.) * When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index b0dbf53d4e3d15..17572dba65487d 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -665,7 +665,9 @@ struct _obmalloc_global_state { struct _obmalloc_state { struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; +#if WITH_PYMALLOC_RADIX_TREE struct _obmalloc_usage usage; +#endif }; diff --git a/Lib/dis.py b/Lib/dis.py index 8d3885d2526b70..efa935c5a6a0b6 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -336,93 +336,6 @@ class Instruction(_Instruction): covered by this instruction """ - @staticmethod - def _get_argval_argrepr(op, arg, offset, co_consts, names, varname_from_oparg, - labels_map): - get_name = None if names is None else names.__getitem__ - argval = None - argrepr = '' - deop = _deoptop(op) - if arg is not None: - # Set argval to the dereferenced value of the argument when - # available, and argrepr to the string representation of argval. - # _disassemble_bytes needs the string repr of the - # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. - argval = arg - if deop in hasconst: - argval, argrepr = _get_const_info(deop, arg, co_consts) - elif deop in hasname: - if deop == LOAD_GLOBAL: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL" - elif deop == LOAD_ATTR: - argval, argrepr = _get_name_info(arg//2, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - elif deop == LOAD_SUPER_ATTR: - argval, argrepr = _get_name_info(arg//4, get_name) - if (arg & 1) and argrepr: - argrepr = f"{argrepr} + NULL|self" - else: - argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = f"to L{labels_map[argval]}" - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - caches = _get_cache_size(_all_opname[deop]) - argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 - argrepr = f"to L{labels_map[argval]}" - elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): - arg1 = arg >> 4 - arg2 = arg & 15 - val1, argrepr1 = _get_name_info(arg1, varname_from_oparg) - val2, argrepr2 = _get_name_info(arg2, varname_from_oparg) - argrepr = argrepr1 + ", " + argrepr2 - argval = val1, val2 - elif deop in haslocal or deop in hasfree: - argval, argrepr = _get_name_info(arg, varname_from_oparg) - elif deop in hascompare: - argval = cmp_op[arg >> 5] - argrepr = argval - if arg & 16: - argrepr = f"bool({argrepr})" - elif deop == CONVERT_VALUE: - argval = (None, str, repr, ascii)[arg] - argrepr = ('', 'str', 'repr', 'ascii')[arg] - elif deop == SET_FUNCTION_ATTRIBUTE: - argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) - if arg & (1<' marker arrow as part of the line *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field + + *line_offset* the line number (within the code unit) """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + + + def print_instruction(self, instr, mark_as_current=False): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + fields = [] # Column: Source code line number if lineno_width: - if self.starts_line: - lineno_fmt = "%%%dd" if self.line_number is not None else "%%%ds" + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" lineno_fmt = lineno_fmt % lineno_width - lineno = self.line_number if self.line_number is not None else '--' + lineno = _NO_LINENO if instr.line_number is None else instr.line_number fields.append(lineno_fmt % lineno) else: fields.append(' ' * lineno_width) # Column: Label - if self.label is not None: - lbl = f"L{self.label}:" + if instr.label is not None: + lbl = f"L{instr.label}:" fields.append(f"{lbl:>{label_width}}") else: fields.append(' ' * label_width) # Column: Instruction offset from start of code sequence if offset_width > 0: - fields.append(f"{repr(self.offset):>{offset_width}} ") + fields.append(f"{repr(instr.offset):>{offset_width}} ") # Column: Current instruction indicator if mark_as_current: fields.append('-->') else: fields.append(' ') # Column: Opcode name - fields.append(self.opname.ljust(_OPNAME_WIDTH)) + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument - if self.arg is not None: - arg = repr(self.arg) + if instr.arg is not None: + arg = repr(instr.arg) # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into # the space reserved for oparg. This results in fewer misaligned opargs # in the disassembly output. - opname_excess = max(0, len(self.opname) - _OPNAME_WIDTH) - fields.append(repr(self.arg).rjust(_OPARG_WIDTH - opname_excess)) + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) # Column: Opcode argument details - if self.argrepr: - fields.append('(' + self.argrepr + ')') - return ' '.join(fields).rstrip() + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts, names, varname_from_oparg, labels_map): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjabs: + argval = arg*2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + if deop == ENTER_EXECUTOR: + argval += 2 + argrepr = f"to L{self.labels_map[argval]}" + elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1< 0 - else: - show_lineno = False - if show_lineno: - maxlineno = max(linestarts_ints) + line_offset - if maxlineno >= 1000: - lineno_width = len(str(maxlineno)) - else: - lineno_width = 3 + offset_width = len(str(max(len(code) - 2, 9999))) if show_offsets else 0 - if lineno_width < len(str(None)) and None in linestarts.values(): - lineno_width = len(str(None)) - else: - lineno_width = 0 - if show_offsets: - maxoffset = len(code) - 2 - if maxoffset >= 10000: - offset_width = len(str(maxoffset)) - else: - offset_width = 4 - else: - offset_width = 0 - - label_width = -1 - for instr in _get_instructions_bytes(code, varname_from_oparg, names, - co_consts, linestarts, - line_offset=line_offset, - exception_entries=exception_entries, - co_positions=co_positions, - show_caches=show_caches, - original_code=original_code): - new_source_line = (show_lineno and - instr.starts_line and - instr.offset > 0) - if new_source_line: - print(file=file) + labels_map = _make_labels_map(original_code or code, exception_entries) + label_width = 4 + len(str(len(labels_map))) + + formatter = Formatter(file=file, + lineno_width=_get_lineno_width(linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=line_offset) + + arg_resolver = ArgResolver(co_consts, names, varname_from_oparg, labels_map) + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + show_caches=show_caches, + original_code=original_code, + labels_map=labels_map, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, + show_caches=show_caches, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, show_caches=False, lasti=-1): + for instr in instrs: if show_caches: is_current_instr = instr.offset == lasti else: # Each CACHE takes 2 bytes is_current_instr = instr.offset <= lasti \ <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) - label_width = getattr(instr, 'label_width', label_width) - assert label_width >= 0 - print(instr._disassemble(lineno_width, is_current_instr, offset_width, label_width), - file=file) - if exception_entries: - print("ExceptionTable:", file=file) - for entry in exception_entries: - lasti = " lasti" if entry.lasti else "" - start = entry.start_label - end = entry.end_label - target = entry.target_label - print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + formatter.print_instruction(instr, is_current_instr) + formatter.print_exception_table(exception_entries) def _disassemble_str(source, **kwargs): """Compile the source string, then disassemble the code object.""" @@ -927,15 +944,18 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False def __iter__(self): co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co.co_consts, co.co_names, co._varname_from_oparg, + labels_map) return _get_instructions_bytes(_get_code_array(co, self.adaptive), - co._varname_from_oparg, - co.co_names, co.co_consts, - self._linestarts, + linestarts=self._linestarts, line_offset=self._line_offset, - exception_entries=self.exception_entries, co_positions=co.co_positions(), show_caches=self.show_caches, - original_code=co.co_code) + original_code=original_code, + labels_map=labels_map, + arg_resolver=arg_resolver) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 667623ec71ac98..464126e2df0668 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -7,7 +7,7 @@ from tkinter import filedialog from tkinter import messagebox -from tkinter.simpledialog import askstring +from tkinter.simpledialog import askstring # loadfile encoding. from idlelib.config import idleConf from idlelib.util import py_extensions @@ -180,24 +180,25 @@ def loadfile(self, filename): return True def maybesave(self): + """Return 'yes', 'no', 'cancel' as appropriate. + + Tkinter messagebox.askyesnocancel converts these tk responses + to True, False, None. Convert back, as now expected elsewhere. + """ if self.get_saved(): return "yes" - message = "Do you want to save %s before closing?" % ( - self.filename or "this untitled document") + message = ("Do you want to save " + f"{self.filename or 'this untitled document'}" + " before closing?") confirm = messagebox.askyesnocancel( title="Save On Close", message=message, default=messagebox.YES, parent=self.text) if confirm: - reply = "yes" self.save(None) - if not self.get_saved(): - reply = "cancel" - elif confirm is None: - reply = "cancel" - else: - reply = "no" + reply = "yes" if self.get_saved() else "cancel" + else: reply = "cancel" if confirm is None else "no" self.text.focus_set() return reply diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000000..7e01eda916aeb5 --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,512 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +import io +import ntpath +import os +import posixpath + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from . import _abc + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +UnsupportedOperation = _abc.UnsupportedOperation + + +class PurePath(_abc.PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.pathmod is ntpath and self.pathmod is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + if path: + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + self._resolving = False + + def __reduce__(self): + # Using the parts tuple helps share interned path parts + # when pickling related paths. + return (self.__class__, tuple(self._raw_paths)) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if _abc._is_case_sensitive(self.pathmod): + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + pathmod = ntpath + __slots__ = () + + +class Path(_abc.PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported(cls, method_name): + msg = f"{cls.__name__}.{method_name}() is unsupported on this system" + raise UnsupportedOperation(msg) + + def __init__(self, *args, **kwargs): + if kwargs: + import warnings + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return (self._make_child_relpath(name) for name in os.listdir(self)) + + def _scandir(self): + return os.scandir(self) + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail, self.has_trailing_sep) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + result = self.with_segments(cwd) + result._str = cwd + return result + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail, self.has_trailing_sep) + tail = rel.split(self.pathmod.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail, self.has_trailing_sep) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail, _ = self._parse_path(homedir) + tail.extend(self._tail[1:]) + has_trailing_sep = self.has_trailing_sep and bool(tail) + return self._from_parsed_parts(drv, root, tail, has_trailing_sep) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib.py b/Lib/pathlib/_abc.py similarity index 70% rename from Lib/pathlib.py rename to Lib/pathlib/_abc.py index 237536389deb84..e09535503f7610 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib/_abc.py @@ -1,10 +1,3 @@ -"""Object-oriented filesystem paths. - -This module provides classes to represent abstract paths and concrete -paths with operations that have semantics appropriate for different -operating systems. -""" - import functools import io import ntpath @@ -17,27 +10,11 @@ from itertools import chain from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - # # Internals # -# Maximum number of symlinks to follow in _PathBase.resolve() +# Maximum number of symlinks to follow in PathBase.resolve() _MAX_SYMLINKS = 40 # Reference for Windows paths can be found at @@ -164,10 +141,6 @@ def _select_unique(paths): yielded.clear() -# -# Public API -# - class UnsupportedOperation(NotImplementedError): """An exception that is raised when an unsupported operation is called on a path object. @@ -204,7 +177,7 @@ def __repr__(self): return "<{}.parents>".format(type(self._path).__name__) -class _PurePathBase: +class PurePathBase: """Base class for pure path objects. This class *does not* provide several magic methods that are defined in @@ -237,24 +210,17 @@ class _PurePathBase: '_str', # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `_PathBase.resolve()`. This prevents duplicate + # is being processed by `PathBase.resolve()`. This prevents duplicate # work from occurring when `resolve()` calls `stat()` or `readlink()`. '_resolving', ) pathmod = os.path def __init__(self, *args): - self._load_raw_paths(args) - - def _load_raw_paths(self, args): paths = [] - for path in args: - if not isinstance(path, str): - raise TypeError( - "argument should be a str, " - f"not {type(path).__name__!r}") - if path: - paths.append(path) + for arg in args: + if arg: + paths.append(arg) self._raw_paths = paths self._resolving = False @@ -488,7 +454,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, remove=(3, 14)) other = self.with_segments(other, *_deprecated) - elif not isinstance(other, _PurePathBase): + elif not isinstance(other, PurePathBase): other = self.with_segments(other) for step, path in enumerate(other._ancestors): if path in self._ancestors: @@ -513,7 +479,7 @@ def is_relative_to(self, other, /, *_deprecated): warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", msg, remove=(3, 14)) other = self.with_segments(other, *_deprecated) - elif not isinstance(other, _PurePathBase): + elif not isinstance(other, PurePathBase): other = self.with_segments(other) return other.without_trailing_sep() in self._ancestors @@ -600,7 +566,7 @@ def match(self, path_pattern, *, case_sensitive=None): """ Return True if this path matches the given pattern. """ - if not isinstance(path_pattern, _PurePathBase): + if not isinstance(path_pattern, PurePathBase): path_pattern = self.with_segments(path_pattern) if case_sensitive is None: case_sensitive = _is_case_sensitive(self.pathmod) @@ -616,183 +582,8 @@ def match(self, path_pattern, *, case_sensitive=None): return match(str(self)) is not None -class PurePath(_PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def _load_raw_paths(self, args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - if path: - paths.append(path) - self._raw_paths = paths - self._resolving = False - - def __reduce__(self): - return (self.__class__, tuple(self._raw_paths)) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _is_case_sensitive(self.pathmod): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = ntpath - __slots__ = () - - -# Filesystem-accessing classes - -class _PathBase(_PurePathBase): +class PathBase(PurePathBase): """Base class for concrete path objects. This class provides dummy implementations for many methods that derived @@ -810,8 +601,6 @@ class _PathBase(_PurePathBase): @classmethod def _unsupported(cls, method_name): msg = f"{cls.__name__}.{method_name}() is unsupported" - if issubclass(cls, Path): - msg += " on this system" raise UnsupportedOperation(msg) def stat(self, *, follow_symlinks=True): @@ -1418,301 +1207,3 @@ def from_uri(cls, uri): def as_uri(self): """Return the path as a URI.""" self._unsupported("as_uri") - - -class Path(_PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - return (self._make_child_relpath(name) for name in os.listdir(self)) - - def _scandir(self): - return os.scandir(self) - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail, self.has_trailing_sep) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - result = self.with_segments(cwd) - result._str = cwd - return result - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail, self.has_trailing_sep) - tail = rel.split(self.pathmod.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail, self.has_trailing_sep) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail, _ = self._parse_path(homedir) - tail.extend(self._tail[1:]) - has_trailing_sep = self.has_trailing_sep and bool(tail) - return self._from_parsed_parts(drv, root, tail, has_trailing_sep) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 7dffe79a0da04e..759020c33aa700 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -92,13 +92,28 @@ def fail(self, cmd_line): # Executing the interpreter in a subprocess @support.requires_subprocess() def run_python_until_end(*args, **env_vars): + """Used to implement assert_python_*. + + *args are the command line flags to pass to the python interpreter. + **env_vars keyword arguments are environment variables to set on the process. + + If __run_using_command= is supplied, it must be a list of + command line arguments to prepend to the command line used. + Useful when you want to run another command that should launch the + python interpreter via its own arguments. ["/bin/echo", "--"] for + example could print the unquoted python command line instead of + run it. + """ env_required = interpreter_requires_environment() + run_using_command = env_vars.pop('__run_using_command', None) cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required cmd_line = [sys.executable, '-X', 'faulthandler'] + if run_using_command: + cmd_line = run_using_command + cmd_line if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 349790ecd7d075..0ea4dc4566a4a4 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1785,6 +1785,12 @@ def __init__(self, *args): super().__init__(*args) self.maxDiff = None + def test_instruction_str(self): + # smoke test for __str__ + instrs = dis.get_instructions(simple) + for instr in instrs: + str(instr) + def test_default_first_line(self): actual = dis.get_instructions(simple) self.assertInstructionsEqual(list(actual), expected_opinfo_simple) @@ -1955,15 +1961,16 @@ def test_jump_target(self): self.assertEqual(10 + 2 + 1*2 + 100*2, instruction.jump_target) def test_argval_argrepr(self): - def f(*args): - return dis.Instruction._get_argval_argrepr( - *args, labels_map={24: 1}) + def f(opcode, oparg, offset, *init_args): + arg_resolver = dis.ArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) offset = 42 co_consts = (0, 1, 2, 3) names = {1: 'a', 2: 'b'} varname_from_oparg = lambda i : names[i] - args = (offset, co_consts, names, varname_from_oparg) + labels_map = {24: 1} + args = (offset, co_consts, names, varname_from_oparg, labels_map) self.assertEqual(f(opcode.opmap["POP_TOP"], None, *args), (None, '')) self.assertEqual(f(opcode.opmap["LOAD_CONST"], 1, *args), (1, '1')) self.assertEqual(f(opcode.opmap["LOAD_GLOBAL"], 2, *args), ('a', 'a')) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a52c8beaa8b023..23bf2593fc8222 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -51,7 +51,7 @@ def test_is_notimplemented(self): class PurePathBaseTest(unittest.TestCase): - cls = pathlib._PurePathBase + cls = pathlib._abc.PurePathBase def test_magic_methods(self): P = self.cls @@ -66,7 +66,7 @@ def test_magic_methods(self): self.assertIs(P.__ge__, object.__ge__) -class DummyPurePath(pathlib._PurePathBase): +class DummyPurePath(pathlib._abc.PurePathBase): def __eq__(self, other): if not isinstance(other, DummyPurePath): return NotImplemented @@ -1787,7 +1787,7 @@ class cls(pathlib.PurePath): # class PathBaseTest(PurePathBaseTest): - cls = pathlib._PathBase + cls = pathlib._abc.PathBase def test_unsupported_operation(self): P = self.cls @@ -1868,7 +1868,7 @@ def close(self): super().close() -class DummyPath(pathlib._PathBase): +class DummyPath(pathlib._abc.PathBase): """ Simple implementation of PathBase that keeps files and directories in memory. diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index dedbc828784184..298e78ccee3875 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2760,6 +2760,132 @@ def test_patma_255(self): self.assertEqual(y, 1) self.assertIs(z, x) + def test_patma_runtime_checkable_protocol(self): + # Runtime-checkable protocol + from typing import Protocol, runtime_checkable + + @runtime_checkable + class P(Protocol): + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + + def test_patma_generic_protocol(self): + # Runtime-checkable generic protocol + from typing import Generic, TypeVar, Protocol, runtime_checkable + + T = TypeVar('T') # not using PEP695 to be able to backport changes + + @runtime_checkable + class P(Protocol[T]): + a: T + b: T + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class G(Generic[T]): + def __init__(self, x: T, y: T): + self.x = x + self.y = y + + for cls in (A, G): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P(): + w = 1 + self.assertEqual(w, 0) + + def test_patma_protocol_with_match_args(self): + # Runtime-checkable protocol with `__match_args__` + from typing import Protocol, runtime_checkable + + # Used to fail before + # https://github.com/python/cpython/issues/110682 + @runtime_checkable + class P(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + j = 0 + match inst: + case P(x=1, y=2): + j = 1 + self.assertEqual(j, 1) + + g = 0 + match inst: + case P(x, y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + g = 1 + self.assertEqual(g, 1) + + h = 0 + match inst: + case P(1, 2): + h = 1 + self.assertEqual(h, 1) + class TestSyntaxErrors(unittest.TestCase): @@ -3198,6 +3324,35 @@ def test_class_pattern_not_type(self): w = 0 self.assertIsNone(w) + def test_regular_protocol(self): + from typing import Protocol + class P(Protocol): ... + msg = ( + 'Instance and class checks can only be used ' + 'with @runtime_checkable protocols' + ) + w = None + with self.assertRaisesRegex(TypeError, msg): + match 1: + case P(): + w = 0 + self.assertIsNone(w) + + def test_positional_patterns_with_regular_protocol(self): + from typing import Protocol + class P(Protocol): + x: int # no `__match_args__` + y: int + class A: + x = 1 + y = 2 + w = None + with self.assertRaises(TypeError): + match A(): + case P(x, y): + w = 0 + self.assertIsNone(w) + class TestValueErrors(unittest.TestCase): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d7061b2f9d8724..b29d316352f219 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1670,6 +1670,17 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(root_dir, 'archive2.tar') tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir] + if sys.platform == 'darwin': + # macOS tar can include extended attributes, + # ACLs and other mac specific metadata into the + # archive (an recentish version of the OS). + # + # This feature can be disabled with the + # '--no-mac-metadata' option on macOS 11 or + # later. + import platform + if int(platform.mac_ver()[0].split('.')[0]) >= 11: + tar_cmd.insert(1, '--no-mac-metadata') subprocess.check_call(tar_cmd, cwd=root_dir, stdout=subprocess.DEVNULL) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 319bc0d2638563..5eeea54fd55f1a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1561,21 +1561,6 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) - class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -3360,6 +3345,89 @@ def exit_handler(): self.assertEqual(out, b'') self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @mock.patch("subprocess._fork_exec") + def test__use_vfork(self, mock_fork_exec): + self.assertTrue(subprocess._USE_VFORK) # The default value regardless. + mock_fork_exec.side_effect = RuntimeError("just testing args") + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + mock_fork_exec.assert_called_once() + # NOTE: These assertions are *ugly* as they require the last arg + # to remain the have_vfork boolean. We really need to refactor away + # from the giant "wall of args" internal C extension API. + self.assertTrue(mock_fork_exec.call_args.args[-1]) + with mock.patch.object(subprocess, '_USE_VFORK', False): + with self.assertRaises(RuntimeError): + subprocess.run([sys.executable, "-c", "pass"]) + self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) + + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + def test_vfork_used_when_expected(self): + # This is a performance regression test to ensure we default to using + # vfork() when possible. + strace_binary = "/usr/bin/strace" + # The only system calls we are interested in. + strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" + true_binary = "/bin/true" + strace_command = [strace_binary, strace_filter] + + try: + does_strace_work_process = subprocess.run( + strace_command + [true_binary], + stderr=subprocess.PIPE, + stdout=subprocess.DEVNULL, + ) + rc = does_strace_work_process.returncode + stderr = does_strace_work_process.stderr + except OSError: + rc = -1 + stderr = "" + if rc or (b"+++ exited with 0 +++" not in stderr): + self.skipTest("strace not found or not working as expected.") + + with self.subTest(name="default_is_vfork"): + vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])"""), + __run_using_command=strace_command, + ) + # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) + self.assertRegex(vfork_result.err, br"(?i)vfork") + # Do NOT check that fork() or other clones did not happen. + # If the OS denys the vfork it'll fallback to plain fork(). + + # Test that each individual thing that would disable the use of vfork + # actually disables it. + for sub_name, preamble, sp_kwarg, expect_permission_error in ( + ("!use_vfork", "subprocess._USE_VFORK = False", "", False), + ("preexec", "", "preexec_fn=lambda: None", False), + ("setgid", "", f"group={os.getgid()}", True), + ("setuid", "", f"user={os.getuid()}", True), + ("setgroups", "", "extra_groups=[]", True), + ): + with self.subTest(name=sub_name): + non_vfork_result = assert_python_ok( + "-c", + textwrap.dedent(f"""\ + import subprocess + {preamble} + try: + subprocess.check_call( + [{true_binary!r}], **dict({sp_kwarg})) + except PermissionError: + if not {expect_permission_error}: + raise"""), + __run_using_command=strace_command, + ) + # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). + self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 6ca11f080dcc3f..42a7545be7e666 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2164,6 +2164,7 @@ LIBSUBDIRS= asyncio \ json \ logging \ multiprocessing multiprocessing/dummy \ + pathlib \ pydoc_data \ re \ site-packages \ diff --git a/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst b/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst new file mode 100644 index 00000000000000..a36814854882bb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-08-11-33-37.gh-issue-112867.ZzDfXQ.rst @@ -0,0 +1 @@ +Fix the build for the case that WITH_PYMALLOC_RADIX_TREE=0 set. diff --git a/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst b/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst new file mode 100644 index 00000000000000..aeaad6e5055522 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-12-04-15-56-11.gh-issue-112334.FFc9Ti.rst @@ -0,0 +1,2 @@ +Adds a regression test to verify that ``vfork()`` is used when expected by +:mod:`subprocess` on vfork enabled POSIX systems (Linux). diff --git a/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst b/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst new file mode 100644 index 00000000000000..c475a33919db98 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-12-09-21-27-46.gh-issue-109980.y--500.rst @@ -0,0 +1,2 @@ +Fix ``test_tarfile_vs_tar`` in ``test_shutil`` for macOS, where system tar +can include more information in the archive than :mod:`shutil.make_archive`. diff --git a/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict new file mode 100644 index 00000000000000..c6a44d946284ef --- /dev/null +++ b/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict @@ -0,0 +1,165 @@ +# bits of syntax +"( " +") " +"[ " +"] " +": " +", " +"; " +"{ " +"} " + +# operators +"+ " +"- " +"* " +"** " +"/ " +"// " +"| " +"& " +"< " +"> " +"= " +". " +"% " +"` " +"^ " +"~ " +"@ " +"== " +"!= " +"<> " +"<< " +"<= " +">= " +">> " +"+= " +"-= " +"*= " +"** " +"/= " +"//= " +"|= " +"%= " +"&= " +"^= " +"<<= " +">>= " +"**= " +":= " +"@= " + +# whitespace +" " +":\\n " + +# type signatures and functions +"-> " +": List[int]" +": Dict[int, str]" + +"# type:" +"# type: List[int]" +"# type: Dict[int, str]" + +", *" +", /" +", *args" +", **kwargs" +", x=42" + + +# literals +"0x0a" +"0b0000" +"42" +"0o70" +"42j" +"42.01" +"-5" +"+42e-3" +"0_0_0" +"1e1_0" +".1_4" + +"{}" + +# variable names +"x" +"y" + +# strings +"r'x'" + +"b'x'" + +"rb\"x\"" + +"br\"x\"" + +"f'{x + 5}'" +"f\"{x + 5}\"" + +"'''" +"\"\"\"" + +"\\u" +"\\x" + +# keywords +"def " +"del " +"pass " +"break " +"continue " +"return " +"raise " +"from " +"import " +".. " +"... " +"__future__ " +"as " +"global " +"nonlocal " +"assert " +"print " +"if " +"elif " +"else: " +"while " +"try: " +"except " +"finally: " +"with " +"lambda " +"or " +"and " +"not " +"None " +"__peg_parser__" +"True " +"False " +"yield " +"async " +"await " +"for " +"in " +"is " +"class " + +# shebangs and encodings +"#!" +"# coding:" +"# coding=" +"# coding: latin-1" +"# coding=latin-1" +"# coding: utf-8" +"# coding=utf-8" +"# coding: ascii" +"# coding=ascii" +"# coding: cp860" +"# coding=cp860" +"# coding: gbk" +"# coding=gbk" diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py new file mode 100644 index 00000000000000..c43994dda29eed --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +def test() -> None: + x: list[int] = [] + x: dict[int, str] = {} + x: set[bytes] = {} + print(5 + 42 * 3, x) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py new file mode 100644 index 00000000000000..7be326e95be0eb --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py @@ -0,0 +1,5 @@ +class Foo(metaclass=42): + __slots__ = ['x'] + pass + +foo = Foo() diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py new file mode 100644 index 00000000000000..9bc3a45ebe75da --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py @@ -0,0 +1,6 @@ +def evens(): + i = 0 + while True: + i += 1 + if i % 2 == 0: + yield i diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py new file mode 100644 index 00000000000000..490de90fb97b39 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py @@ -0,0 +1,3 @@ +async def hello(name: str): + await name + print(name) diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py new file mode 100644 index 00000000000000..4cfcfe590ebc95 --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py @@ -0,0 +1,7 @@ +try: + eval('importer exporter... really long matches') +except SyntaxError: + print("nothing to see here") +finally: + print("all done here") + raise diff --git a/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py new file mode 100644 index 00000000000000..d8e59ade503a8c --- /dev/null +++ b/Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py @@ -0,0 +1,8 @@ +"""Some module docstring""" +import sys + +def main(): + print("Hello world!", file=sys.stderr) + +if __name__ == '__main__': + main() diff --git a/Modules/_xxtestfuzz/fuzz_tests.txt b/Modules/_xxtestfuzz/fuzz_tests.txt index 40aa22110e7d27..ea6f982eefc9da 100644 --- a/Modules/_xxtestfuzz/fuzz_tests.txt +++ b/Modules/_xxtestfuzz/fuzz_tests.txt @@ -8,3 +8,4 @@ fuzz_csv_reader fuzz_struct_unpack fuzz_ast_literal_eval fuzz_elementtree_parsewhole +fuzz_pycompile diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 77d29ce773a04b..e133b4d3c44480 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -501,6 +501,63 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) { return 0; } +#define MAX_PYCOMPILE_TEST_SIZE 16384 +static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE]; + +static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input}; +const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]); + +static const int optimize_vals[] = {-1, 0, 1, 2}; +const size_t NUM_OPTIMIZE_VALS = sizeof(optimize_vals) / sizeof(optimize_vals[0]); + +/* Fuzz `PyCompileStringExFlags` using a variety of input parameters. + * That function is essentially behind the `compile` builtin */ +static int fuzz_pycompile(const char* data, size_t size) { + // Ignore overly-large inputs, and account for a NUL terminator + if (size > MAX_PYCOMPILE_TEST_SIZE - 1) { + return 0; + } + + // Need 2 bytes for parameter selection + if (size < 2) { + return 0; + } + + // Use first byte to determine element of `start_vals` to use + unsigned char start_idx = (unsigned char) data[0]; + int start = start_vals[start_idx % NUM_START_VALS]; + + // Use second byte to determine element of `optimize_vals` to use + unsigned char optimize_idx = (unsigned char) data[1]; + int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS]; + + // Create a NUL-terminated C string from the remaining input + memcpy(pycompile_scratch, data + 2, size - 2); + // Put a NUL terminator just after the copied data. (Space was reserved already.) + pycompile_scratch[size - 2] = '\0'; + + // XXX: instead of always using NULL for the `flags` value to + // `Py_CompileStringExFlags`, there are many flags that conditionally + // change parser behavior: + // + // #define PyCF_TYPE_COMMENTS 0x1000 + // #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 + // #define PyCF_ONLY_AST 0x0400 + // + // It would be good to test various combinations of these, too. + PyCompilerFlags *flags = NULL; + + PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "", start, flags, optimize); + if (result == NULL) { + /* compilation failed, most likely from a syntax error */ + PyErr_Clear(); + } else { + Py_DECREF(result); + } + + return 0; +} + /* Run fuzzer and abort on failure. */ static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) { int rv = fuzzer((const char*) data, size); @@ -642,6 +699,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } rv |= _run_fuzz(data, size, fuzz_elementtree_parsewhole); +#endif +#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_pycompile) + rv |= _run_fuzz(data, size, fuzz_pycompile); #endif return rv; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index d59e0ddcdfde4e..ff6e1ef4f993ba 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -599,6 +599,9 @@ Modules/_xxtestfuzz/fuzzer.c - re_error_exception - Modules/_xxtestfuzz/fuzzer.c - struct_error - Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method - Modules/_xxtestfuzz/fuzzer.c - xmlparser_type - +Modules/_xxtestfuzz/fuzzer.c - pycompile_scratch - +Modules/_xxtestfuzz/fuzzer.c - start_vals - +Modules/_xxtestfuzz/fuzzer.c - optimize_vals - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED - Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED - diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6c76f66a81abd4..816ce0e6efed61 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -3184,7 +3184,7 @@ def closure(f: CConverterClassT) -> CConverterClassT: class CConverterAutoRegister(type): def __init__( - cls, name: str, bases: tuple[type, ...], classdict: dict[str, Any] + cls, name: str, bases: tuple[type[object], ...], classdict: dict[str, Any] ) -> None: converter_cls = cast(type["CConverter"], cls) add_c_converter(converter_cls) @@ -3217,7 +3217,7 @@ class CConverter(metaclass=CConverterAutoRegister): # If not None, default must be isinstance() of this type. # (You can also specify a tuple of types.) - default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None + default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None # "default" converted into a C value, as a string. # Or None if there is no default. @@ -3683,7 +3683,7 @@ def add_include(self, name: str, reason: str, ReturnConverterDict = dict[str, ReturnConverterType] return_converters: ReturnConverterDict = {} -TypeSet = set[bltns.type[Any]] +TypeSet = set[bltns.type[object]] class bool_converter(CConverter): @@ -4347,7 +4347,7 @@ class buffer: pass class rwbuffer: pass class robuffer: pass -StrConverterKeyType = tuple[frozenset[type], bool, bool] +StrConverterKeyType = tuple[frozenset[type[object]], bool, bool] def str_converter_key( types: TypeSet, encoding: bool | str | None, zeroes: bool @@ -4846,7 +4846,7 @@ class CReturnConverterAutoRegister(type): def __init__( cls: ReturnConverterType, name: str, - bases: tuple[type, ...], + bases: tuple[type[object], ...], classdict: dict[str, Any] ) -> None: add_c_return_converter(cls)