diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h
new file mode 100644
index 00000000000000..b725986528d864
--- /dev/null
+++ b/Include/internal/pycore_freelist.h
@@ -0,0 +1,35 @@
+#ifndef Py_INTERNAL_FREELIST_H
+#define Py_INTERNAL_FREELIST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifndef WITH_FREELISTS
+// without freelists
+# define PyList_MAXFREELIST 0
+#endif
+
+/* Empty list reuse scheme to save calls to malloc and free */
+#ifndef PyList_MAXFREELIST
+# define PyList_MAXFREELIST 80
+#endif
+
+struct _Py_list_state {
+#if PyList_MAXFREELIST > 0
+ PyListObject *free_list[PyList_MAXFREELIST];
+ int numfree;
+#endif
+};
+
+typedef struct _Py_freelist_state {
+ struct _Py_list_state list;
+} _PyFreeListState;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FREELIST_H */
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index 2a79c403803ed1..5d90d3a7f865da 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -8,6 +8,8 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_freelist.h" // _PyFreeListState
+
/* GC information is stored BEFORE the object structure. */
typedef struct {
// Pointer to next object in the list.
@@ -238,9 +240,11 @@ extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generat
extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
// Functions to clear types free lists
+extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
+extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization);
extern void _PyTuple_ClearFreeList(PyInterpreterState *interp);
extern void _PyFloat_ClearFreeList(PyInterpreterState *interp);
-extern void _PyList_ClearFreeList(PyInterpreterState *interp);
+extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
extern void _PyContext_ClearFreeList(PyInterpreterState *interp);
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 4512b1edb4b9b3..4d49fa2a51b88c 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -179,6 +179,9 @@ struct _is {
// One bit is set for each non-NULL entry in code_watchers
uint8_t active_code_watchers;
+#if !defined(Py_GIL_DISABLED)
+ struct _Py_freelist_state freelist_state;
+#endif
struct _py_object_state object_state;
struct _Py_unicode_state unicode;
struct _Py_float_state float_state;
@@ -190,7 +193,6 @@ struct _is {
PySliceObject *slice_cache;
struct _Py_tuple_state tuple;
- struct _Py_list_state list;
struct _Py_dict_state dict_state;
struct _Py_async_gen_state async_gen;
struct _Py_context_state context;
diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h
index 55d67b32bc8a63..6c29d882335512 100644
--- a/Include/internal/pycore_list.h
+++ b/Include/internal/pycore_list.h
@@ -8,6 +8,7 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_freelist.h" // _PyFreeListState
extern PyObject* _PyList_Extend(PyListObject *, PyObject *);
extern void _PyList_DebugMallocStats(FILE *out);
@@ -15,28 +16,9 @@ extern void _PyList_DebugMallocStats(FILE *out);
/* runtime lifecycle */
-extern void _PyList_Fini(PyInterpreterState *);
+extern void _PyList_Fini(_PyFreeListState *);
-/* other API */
-
-#ifndef WITH_FREELISTS
-// without freelists
-# define PyList_MAXFREELIST 0
-#endif
-
-/* Empty list reuse scheme to save calls to malloc and free */
-#ifndef PyList_MAXFREELIST
-# define PyList_MAXFREELIST 80
-#endif
-
-struct _Py_list_state {
-#if PyList_MAXFREELIST > 0
- PyListObject *free_list[PyList_MAXFREELIST];
- int numfree;
-#endif
-};
-
#define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)
extern int
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 37b45faf8a74a0..348c5c634284b0 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -8,7 +8,9 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_runtime.h" // _PyRuntime
+#include "pycore_tstate.h" // _PyThreadStateImpl
// Values for PyThreadState.state. A thread must be in the "attached" state
@@ -239,6 +241,20 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
// See also PyInterpreterState_Get() and _PyInterpreterState_GET().
extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
+static inline _PyFreeListState* _PyFreeListState_GET(void)
+{
+ PyThreadState *tstate = _PyThreadState_GET();
+#ifdef Py_DEBUG
+ _Py_EnsureTstateNotNULL(tstate);
+#endif
+
+#ifdef Py_GIL_DISABLED
+ return &((_PyThreadStateImpl*)tstate)->freelist_state;
+#else
+ return &tstate->interp->freelist_state;
+#endif
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index 856ddd5e7e5ff0..472fa08154e8f9 100644
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -8,6 +8,7 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_freelist.h" // struct _Py_freelist_state
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
@@ -20,6 +21,7 @@ typedef struct _PyThreadStateImpl {
#ifdef Py_GIL_DISABLED
struct _mimalloc_thread_state mimalloc;
+ struct _Py_freelist_state freelist_state;
#endif
} _PyThreadStateImpl;
diff --git a/Makefile.pre.in b/Makefile.pre.in
index abbd4b1b1fbd6c..15d419b930c181 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -418,6 +418,8 @@ PYTHON_OBJS= \
Python/frozenmain.o \
Python/future.o \
Python/gc.o \
+ Python/gc_free_threading.o \
+ Python/gc_gil.o \
Python/getargs.o \
Python/getcompiler.o \
Python/getcopyright.o \
@@ -1828,6 +1830,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_floatobject.h \
$(srcdir)/Include/internal/pycore_format.h \
$(srcdir)/Include/internal/pycore_frame.h \
+ $(srcdir)/Include/internal/pycore_freelist.h \
$(srcdir)/Include/internal/pycore_function.h \
$(srcdir)/Include/internal/pycore_genobject.h \
$(srcdir)/Include/internal/pycore_getopt.h \
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 5cd4a059c668ba..2fc57e13f632f8 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -24,8 +24,9 @@ _Py_DECLARE_STR(list_err, "list index out of range");
static struct _Py_list_state *
get_list_state(void)
{
- PyInterpreterState *interp = _PyInterpreterState_GET();
- return &interp->list;
+ _PyFreeListState *state = _PyFreeListState_GET();
+ assert(state != NULL);
+ return &state->list;
}
#endif
@@ -120,26 +121,25 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
}
void
-_PyList_ClearFreeList(PyInterpreterState *interp)
+_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
{
#if PyList_MAXFREELIST > 0
- struct _Py_list_state *state = &interp->list;
- while (state->numfree) {
+ struct _Py_list_state *state = &freelist_state->list;
+ while (state->numfree > 0) {
PyListObject *op = state->free_list[--state->numfree];
assert(PyList_CheckExact(op));
PyObject_GC_Del(op);
}
+ if (is_finalization) {
+ state->numfree = -1;
+ }
#endif
}
void
-_PyList_Fini(PyInterpreterState *interp)
+_PyList_Fini(_PyFreeListState *state)
{
- _PyList_ClearFreeList(interp);
-#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0
- struct _Py_list_state *state = &interp->list;
- state->numfree = -1;
-#endif
+ _PyList_ClearFreeList(state, 1);
}
/* Print summary info about the state of the optimized allocator */
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index f16a763772e42e..610581bc96cb1a 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -208,6 +208,8 @@
+
+
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 7f03cfea1b3e6f..3141913c043869 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -169,6 +169,12 @@
Source Files
+
+ Source Files
+
+
+ Source Files
+
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index c599b68e0d8e66..a8b753ca489ab7 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -231,6 +231,7 @@
+
@@ -568,6 +569,8 @@
+
+
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 13582dff3f40c4..965efa2e3d34b9 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -618,6 +618,12 @@
Include\internal
+
+ Include\internal
+
+
+ Include\internal
+
Include\internal
@@ -1286,6 +1292,12 @@
Python
+
+ Python
+
+
+ Python
+
Python
diff --git a/Python/gc.c b/Python/gc.c
index f47c74f87a9166..9f9a755f6ac95e 100644
--- a/Python/gc.c
+++ b/Python/gc.c
@@ -1019,21 +1019,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate,
}
}
-/* Clear all free lists
- * All free lists are cleared during the collection of the highest generation.
- * Allocated items in the free list may keep a pymalloc arena occupied.
- * Clearing the free lists may give back memory to the OS earlier.
- */
-static void
-clear_freelists(PyInterpreterState *interp)
-{
- _PyTuple_ClearFreeList(interp);
- _PyFloat_ClearFreeList(interp);
- _PyList_ClearFreeList(interp);
- _PyDict_ClearFreeList(interp);
- _PyAsyncGen_ClearFreeLists(interp);
- _PyContext_ClearFreeList(interp);
-}
// Show stats for objects in each generations
static void
@@ -1449,7 +1434,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
/* Clear free list only during the collection of the highest
* generation */
if (generation == NUM_GENERATIONS-1) {
- clear_freelists(tstate->interp);
+ _PyGC_ClearAllFreeLists(tstate->interp);
}
if (_PyErr_Occurred(tstate)) {
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
new file mode 100644
index 00000000000000..aea272840f950d
--- /dev/null
+++ b/Python/gc_free_threading.c
@@ -0,0 +1,32 @@
+#include "Python.h"
+#include "pycore_pystate.h" // _PyFreeListState_GET()
+#include "pycore_tstate.h" // _PyThreadStateImpl
+
+#ifdef Py_GIL_DISABLED
+
+/* Clear all free lists
+ * All free lists are cleared during the collection of the highest generation.
+ * Allocated items in the free list may keep a pymalloc arena occupied.
+ * Clearing the free lists may give back memory to the OS earlier.
+ * Free-threading version: Since freelists are managed per thread,
+ * GC should clear all freelists by traversing all threads.
+ */
+void
+_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
+{
+ _PyTuple_ClearFreeList(interp);
+ _PyFloat_ClearFreeList(interp);
+ _PyDict_ClearFreeList(interp);
+ _PyAsyncGen_ClearFreeLists(interp);
+ _PyContext_ClearFreeList(interp);
+
+ HEAD_LOCK(&_PyRuntime);
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;
+ while (tstate != NULL) {
+ _Py_ClearFreeLists(&tstate->freelist_state, 0);
+ tstate = (_PyThreadStateImpl *)tstate->base.next;
+ }
+ HEAD_UNLOCK(&_PyRuntime);
+}
+
+#endif
diff --git a/Python/gc_gil.c b/Python/gc_gil.c
new file mode 100644
index 00000000000000..b0961cdbee904f
--- /dev/null
+++ b/Python/gc_gil.c
@@ -0,0 +1,23 @@
+#include "Python.h"
+#include "pycore_pystate.h" // _Py_ClearFreeLists()
+
+#ifndef Py_GIL_DISABLED
+
+/* Clear all free lists
+ * All free lists are cleared during the collection of the highest generation.
+ * Allocated items in the free list may keep a pymalloc arena occupied.
+ * Clearing the free lists may give back memory to the OS earlier.
+ */
+void
+_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
+{
+ _PyTuple_ClearFreeList(interp);
+ _PyFloat_ClearFreeList(interp);
+ _PyDict_ClearFreeList(interp);
+ _PyAsyncGen_ClearFreeLists(interp);
+ _PyContext_ClearFreeList(interp);
+
+ _Py_ClearFreeLists(&interp->freelist_state, 0);
+}
+
+#endif
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 1d8af26e4a1cb7..bd6475f99e464b 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1752,13 +1752,16 @@ finalize_interp_types(PyInterpreterState *interp)
_PyUnicode_ClearInterned(interp);
_PyDict_Fini(interp);
- _PyList_Fini(interp);
_PyTuple_Fini(interp);
_PySlice_Fini(interp);
_PyUnicode_Fini(interp);
_PyFloat_Fini(interp);
+
+ _PyFreeListState *state = _PyFreeListState_GET();
+ _PyList_Fini(state);
+
#ifdef Py_DEBUG
_PyStaticObjects_CheckRefcnt(interp);
#endif
diff --git a/Python/pystate.c b/Python/pystate.c
index 21f16b7bcdff0d..ddd57f75f3f363 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -1455,6 +1455,12 @@ clear_datastack(PyThreadState *tstate)
}
}
+void
+_Py_ClearFreeLists(_PyFreeListState *state, int is_finalization)
+{
+ _PyList_ClearFreeList(state, is_finalization);
+}
+
void
PyThreadState_Clear(PyThreadState *tstate)
{
@@ -1537,6 +1543,11 @@ PyThreadState_Clear(PyThreadState *tstate)
// don't call _PyInterpreterState_SetNotRunningMain() yet.
tstate->on_delete(tstate->on_delete_data);
}
+#ifdef Py_GIL_DISABLED
+ // Each thread should clear own freelists in free-threading builds.
+ _PyFreeListState *freelist_state = &((_PyThreadStateImpl*)tstate)->freelist_state;
+ _Py_ClearFreeLists(freelist_state, 0);
+#endif
_PyThreadState_ClearMimallocHeaps(tstate);