Merge branch 'main' into merge-copy-copytree

Doc/library/pathlib.rst

@@ -1618,7 +1618,7 @@ Copying, renaming and deleting
 .. method:: Path.unlink(missing_ok=False)
 
    Remove this file or symbolic link.  If the path points to a directory,
-   use :func:`Path.rmdir` instead.
+   use :func:`Path.rmdir` or :func:`Path.delete` instead.
 
    If *missing_ok* is false (the default), :exc:`FileNotFoundError` is
    raised if the path does not exist.
@@ -1632,33 +1632,40 @@ Copying, renaming and deleting
 
 .. method:: Path.rmdir()
 
-   Remove this directory.  The directory must be empty.
+   Remove this directory.  The directory must be empty; use
+   :meth:`Path.delete` to remove a non-empty directory.
 
 
-.. method:: Path.rmtree(ignore_errors=False, on_error=None)
+.. method:: Path.delete(ignore_errors=False, on_error=None)
 
-   Recursively delete this entire directory tree. The path must not refer to a symlink.
+   Delete this file or directory. If this path refers to a non-empty
+   directory, its files and sub-directories are deleted recursively.
 
-   If *ignore_errors* is true, errors resulting from failed removals will be
-   ignored. If *ignore_errors* is false or omitted, and a function is given to
-   *on_error*, it will be called each time an exception is raised. If neither
-   *ignore_errors* nor *on_error* are supplied, exceptions are propagated to
-   the caller.
+   If *ignore_errors* is true, errors resulting from failed deletions will be
+   ignored. If *ignore_errors* is false or omitted, and a callable is given as
+   the optional *on_error* argument, it will be called with one argument of
+   type :exc:`OSError` each time an exception is raised. The callable can
+   handle the error to continue the deletion process or re-raise it to stop.
+   Note that the filename is available as the :attr:`~OSError.filename`
+   attribute of the exception object. If neither *ignore_errors* nor
+   *on_error* are supplied, exceptions are propagated to the caller.
 
    .. note::
 
-      On platforms that support the necessary fd-based functions, a symlink
-      attack-resistant version of :meth:`~Path.rmtree` is used by default. On
-      other platforms, the :func:`~Path.rmtree` implementation is susceptible
-      to a symlink attack: given proper timing and circumstances, attackers
-      can manipulate symlinks on the filesystem to delete files they would not
-      be able to access otherwise.
-
-   If the optional argument *on_error* is specified, it should be a callable;
-   it will be called with one argument of type :exc:`OSError`. The
-   callable can handle the error to continue the deletion process or re-raise
-   it to stop. Note that the filename is available as the :attr:`~OSError.filename`
-   attribute of the exception object.
+      When deleting non-empty directories on platforms that lack the necessary
+      file descriptor-based functions, the :meth:`~Path.delete` implementation
+      is susceptible to a symlink attack: given proper timing and
+      circumstances, attackers can manipulate symlinks on the filesystem to
+      delete files they would not be able to access otherwise. Applications
+      can use the :data:`~Path.delete.avoids_symlink_attacks` method attribute
+      to determine whether the implementation is immune to this attack.
+
+   .. attribute:: delete.avoids_symlink_attacks
+
+      Indicates whether the current platform and implementation provides a
+      symlink attack resistant version of :meth:`~Path.delete`.  Currently
+      this is only true for platforms supporting fd-based directory access
+      functions.
 
    .. versionadded:: 3.14
 

Doc/library/site.rst

@@ -32,14 +32,22 @@ It starts by constructing up to four directories from a head and a tail part.
 For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads
 are skipped.  For the tail part, it uses the empty string and then
 :file:`lib/site-packages` (on Windows) or
-:file:`lib/python{X.Y}/site-packages` (on Unix and macOS).  For each
+:file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The
+optional suffix "t" indicates the :term:`free threading` build, and is
+appended if ``"t"`` is present in the :attr:`sys.abiflags` constant.)
+For each
 of the distinct head-tail combinations, it sees if it refers to an existing
 directory, and if so, adds it to ``sys.path`` and also inspects the newly
 added path for configuration files.
 
 .. versionchanged:: 3.5
    Support for the "site-python" directory has been removed.
 
+.. versionchanged:: 3.13
+   On Unix, :term:`Free threading <free threading>` Python installations are
+   identified by the "t" suffix in the version-specific directory name, such as
+   :file:`lib/python3.13t/`.
+
 If a file named "pyvenv.cfg" exists one directory above sys.executable,
 sys.prefix and sys.exec_prefix are set to that directory and
 it is also checked for site-packages (sys.base_prefix and
@@ -188,11 +196,12 @@ Module contents
 
    Path to the user site-packages for the running Python.  Can be ``None`` if
    :func:`getusersitepackages` hasn't been called yet.  Default value is
-   :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework
+   :file:`~/.local/lib/python{X.Y}[t]/site-packages` for UNIX and non-framework
    macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS
    framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages`
-   on Windows.  This directory is a site directory, which means that
-   :file:`.pth` files in it will be processed.
+   on Windows.  The optional "t" indicates the free-threaded build.  This
+   directory is a site directory, which means that :file:`.pth` files in it
+   will be processed.
 
 
 .. data:: USER_BASE

Doc/whatsnew/3.14.rst

@@ -139,8 +139,7 @@ pathlib
 
   * :meth:`~pathlib.Path.copy` copies a file or directory tree to a given
     destination.
-  * :meth:`~pathlib.Path.rmtree` recursively removes a directory tree, like
-    :func:`shutil.rmtree`.
+  * :meth:`~pathlib.Path.delete` removes a file or directory tree.
 
   (Contributed by Barney Gale in :gh:`73991`.)
 

Include/cpython/object.h

@@ -270,6 +270,9 @@ typedef struct _heaptypeobject {
     PyObject *ht_module;
     char *_ht_tpname;  // Storage for "tp_name"; see PyType_FromModuleAndSpec
     struct _specialization_cache _spec_cache; // For use by the specializer.
+#ifdef Py_GIL_DISABLED
+    Py_ssize_t unique_id;  // ID used for thread-local refcounting
+#endif
     /* here are optional user slots, followed by the members. */
 } PyHeapTypeObject;
 

Include/internal/pycore_gc.h

@@ -381,10 +381,6 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
-#ifdef Py_GIL_DISABLED
-// gh-117783: Immortalize objects that use deferred reference counting
-extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp);
-#endif
 
 #ifdef __cplusplus
 }

Include/internal/pycore_interp.h

@@ -35,6 +35,7 @@ extern "C" {
 #include "pycore_qsbr.h"          // struct _qsbr_state
 #include "pycore_tstate.h"        // _PyThreadStateImpl
 #include "pycore_tuple.h"         // struct _Py_tuple_state
+#include "pycore_typeid.h"        // struct _Py_type_id_pool
 #include "pycore_typeobject.h"    // struct types_state
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "pycore_warnings.h"      // struct _warnings_runtime_state
@@ -220,6 +221,7 @@ struct _is {
 #if defined(Py_GIL_DISABLED)
     struct _mimalloc_interp_state mimalloc;
     struct _brc_state brc;  // biased reference counting state
+    struct _Py_type_id_pool type_ids;
     PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
 #endif
 

Include/internal/pycore_object.h

@@ -14,10 +14,19 @@ extern "C" {
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_pyatomic_ft_wrappers.h"  // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
+#include "pycore_typeid.h"        // _PyType_IncrefSlow
 
 
 #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
 
+// This value is added to `ob_ref_shared` for objects that use deferred
+// reference counting so that they are not immediately deallocated when the
+// non-deferred reference count drops to zero.
+//
+// The value is half the maximum shared refcount because the low two bits of
+// `ob_ref_shared` are used for flags.
+#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8)
+
 // gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when
 // comparing the reference count to stay compatible with C extensions built
 // with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF
@@ -280,6 +289,67 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp);
 extern void _PyObject_FiniState(PyInterpreterState *interp);
 extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
 
+#ifndef Py_GIL_DISABLED
+#  define _Py_INCREF_TYPE Py_INCREF
+#  define _Py_DECREF_TYPE Py_DECREF
+#else
+static inline void
+_Py_INCREF_TYPE(PyTypeObject *type)
+{
+    if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
+        assert(_Py_IsImmortal(type));
+        return;
+    }
+
+    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+    PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
+
+    // Unsigned comparison so that `unique_id=-1`, which indicates that
+    // per-thread refcounting has been disabled on this type, is handled by
+    // the "else".
+    if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
+#  ifdef Py_REF_DEBUG
+        _Py_INCREF_IncRefTotal();
+#  endif
+        _Py_INCREF_STAT_INC();
+        tstate->types.refcounts[ht->unique_id]++;
+    }
+    else {
+        // The slow path resizes the thread-local refcount array if necessary.
+        // It handles the unique_id=-1 case to keep the inlinable function smaller.
+        _PyType_IncrefSlow(ht);
+    }
+}
+
+static inline void
+_Py_DECREF_TYPE(PyTypeObject *type)
+{
+    if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
+        assert(_Py_IsImmortal(type));
+        return;
+    }
+
+    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+    PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
+
+    // Unsigned comparison so that `unique_id=-1`, which indicates that
+    // per-thread refcounting has been disabled on this type, is handled by
+    // the "else".
+    if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
+#  ifdef Py_REF_DEBUG
+        _Py_DECREF_DecRefTotal();
+#  endif
+        _Py_DECREF_STAT_INC();
+        tstate->types.refcounts[ht->unique_id]--;
+    }
+    else {
+        // Directly decref the type if the type id is not assigned or if
+        // per-thread refcounting has been disabled on this type.
+        Py_DECREF(type);
+    }
+}
+#endif
+
 /* Inline functions trading binary compatibility for speed:
    _PyObject_Init() is the fast version of PyObject_Init(), and
    _PyObject_InitVar() is the fast version of PyObject_InitVar().
@@ -291,7 +361,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
     assert(op != NULL);
     Py_SET_TYPE(op, typeobj);
     assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj));
-    Py_INCREF(typeobj);
+    _Py_INCREF_TYPE(typeobj);
     _Py_NewReference(op);
 }
 

Include/internal/pycore_tstate.h

@@ -31,6 +31,16 @@ typedef struct _PyThreadStateImpl {
     struct _mimalloc_thread_state mimalloc;
     struct _Py_freelists freelists;
     struct _brc_thread_state brc;
+    struct {
+        // The thread-local refcounts for heap type objects
+        Py_ssize_t *refcounts;
+
+        // Size of the refcounts array.
+        Py_ssize_t size;
+
+        // If set, don't use thread-local refcounts
+        int is_finalized;
+    } types;
 #endif
 
 #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)

Include/internal/pycore_typeid.h

@@ -0,0 +1,75 @@
+#ifndef Py_INTERNAL_TYPEID_H
+#define Py_INTERNAL_TYPEID_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_GIL_DISABLED
+
+// This contains code for allocating unique ids to heap type objects
+// and re-using those ids when the type is deallocated.
+//
+// The type ids are used to implement per-thread reference counts of
+// heap type objects to avoid contention on the reference count fields
+// of heap type objects. Static type objects are immortal, so contention
+// is not an issue for those types.
+//
+// Type id of -1 is used to indicate a type doesn't use thread-local
+// refcounting. This value is used when a type object is finalized by the GC
+// and during interpreter shutdown to allow the type object to be
+// deallocated promptly when the object's refcount reaches zero.
+//
+// Each entry implicitly represents a type id based on it's offset in the
+// table. Non-allocated entries form a free-list via the 'next' pointer.
+// Allocated entries store the corresponding PyTypeObject.
+typedef union _Py_type_id_entry {
+    // Points to the next free type id, when part of the freelist
+    union _Py_type_id_entry *next;
+
+    // Stores the type object when the id is assigned
+    PyHeapTypeObject *type;
+} _Py_type_id_entry;
+
+struct _Py_type_id_pool {
+    PyMutex mutex;
+
+    // combined table of types with allocated type ids and unallocated
+    // type ids.
+    _Py_type_id_entry *table;
+
+    // Next entry to allocate inside 'table' or NULL
+    _Py_type_id_entry *freelist;
+
+    // size of 'table'
+    Py_ssize_t size;
+};
+
+// Assigns the next id from the pool of type ids.
+extern void _PyType_AssignId(PyHeapTypeObject *type);
+
+// Releases the allocated type id back to the pool.
+extern void _PyType_ReleaseId(PyHeapTypeObject *type);
+
+// Merges the thread-local reference counts into the corresponding types.
+extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
+
+// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local
+// array of refcounts.
+extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
+
+// Frees the interpreter's pool of type ids.
+extern void _PyType_FinalizeIdPool(PyInterpreterState *interp);
+
+// Increfs the type, resizing the thread-local refcount array if necessary.
+PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
+
+#endif   /* Py_GIL_DISABLED */
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_TYPEID_H */

Lib/ensurepip/__init__.py

@@ -10,7 +10,7 @@
 
 
 __all__ = ["version", "bootstrap"]
-_PIP_VERSION = "24.1.1"
+_PIP_VERSION = "24.2"
 
 # Directory of system wheel packages. Some Linux distribution packaging
 # policies recommend against bundling dependencies. For example, Fedora