Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@colesbury
colesbury committed Sep 20, 2024
1 parent d67b1eb commit 45814d4
Show file tree
Hide file tree
Showing 8 changed files with 29 additions and 91 deletions.
2 changes: 1 addition & 1 deletion Doc/howto/free-threading-python.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ behavior.

.. note::

It's recommended to use the :class:`threading.Lock` or other synchronization
It's recommended to use :class:`threading.Lock` or other synchronization
primitives instead of relying on the internal locks of built-in types, when
possible.

Expand Down
13 changes: 4 additions & 9 deletions Include/cpython/dictobject.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,11 @@ typedef struct {
/* Number of items in the dictionary */
Py_ssize_t ma_used;

/* Dictionary version: globally unique, value change each time
the dictionary is modified */
#ifdef Py_BUILD_CORE
/* Bits 0-7 are for dict watchers.
/* This is a private field for CPython's internal use.
* Bits 0-7 are for dict watchers.
* Bits 8-11 are for the watched mutation counter (used by tier2 optimization)
* The remaining bits (12-63) are the actual version tag. */
uint64_t ma_version_tag;
#else
Py_DEPRECATED(3.12) uint64_t ma_version_tag;
#endif
* The remaining bits are not currently used. */
uint64_t ma_watcher_tag;

PyDictKeysObject *ma_keys;

Expand Down
5 changes: 2 additions & 3 deletions Include/internal/pycore_dict.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,20 +263,19 @@ _PyDict_SendEvent(int watcher_bits,
PyObject *key,
PyObject *value);

static inline uint64_t
static inline void
_PyDict_NotifyEvent(PyInterpreterState *interp,
PyDict_WatchEvent event,
PyDictObject *mp,
PyObject *key,
PyObject *value)
{
assert(Py_REFCNT((PyObject*)mp) > 0);
int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK;
int watcher_bits = mp->ma_watcher_tag & DICT_WATCHER_MASK;
if (watcher_bits) {
RARE_EVENT_STAT_INC(watched_dict_modification);
_PyDict_SendEvent(watcher_bits, event, mp, key, value);
}
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
}

extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj);
Expand Down
14 changes: 0 additions & 14 deletions Modules/_testcapi/dict.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,19 +181,6 @@ dict_popstring_null(PyObject *self, PyObject *args)
RETURN_INT(PyDict_PopString(dict, key, NULL));
}

static PyObject *
dict_version(PyObject *self, PyObject *dict)
{
if (!PyDict_Check(dict)) {
PyErr_SetString(PyExc_TypeError, "expected dict");
return NULL;
}
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag);
_Py_COMP_DIAG_POP
}

static PyMethodDef test_methods[] = {
{"dict_containsstring", dict_containsstring, METH_VARARGS},
{"dict_getitemref", dict_getitemref, METH_VARARGS},
Expand All @@ -204,7 +191,6 @@ static PyMethodDef test_methods[] = {
{"dict_pop_null", dict_pop_null, METH_VARARGS},
{"dict_popstring", dict_popstring, METH_VARARGS},
{"dict_popstring_null", dict_popstring_null, METH_VARARGS},
{"dict_version", dict_version, METH_O},
{NULL},
};

Expand Down
20 changes: 0 additions & 20 deletions Modules/_testcapimodule.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1909,25 +1909,6 @@ getitem_with_error(PyObject *self, PyObject *args)
return PyObject_GetItem(map, key);
}

static PyObject *
dict_get_version(PyObject *self, PyObject *args)
{
PyDictObject *dict;
uint64_t version;

if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict))
return NULL;

_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
version = dict->ma_version_tag;
_Py_COMP_DIAG_POP

static_assert(sizeof(unsigned long long) >= sizeof(version),
"version is larger than unsigned long long");
return PyLong_FromUnsignedLongLong((unsigned long long)version);
}


static PyObject *
raise_SIGINT_then_send_None(PyObject *self, PyObject *args)
Expand Down Expand Up @@ -3407,7 +3388,6 @@ static PyMethodDef TestMethods[] = {
{"return_result_with_error", return_result_with_error, METH_NOARGS},
{"getitem_with_error", getitem_with_error, METH_VARARGS},
{"Py_CompileString", pycompilestring, METH_O},
{"dict_get_version", dict_get_version, METH_VARARGS},
{"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
{"stack_pointer", stack_pointer, METH_NOARGS},
#ifdef W_STOPCODE
Expand Down
58 changes: 19 additions & 39 deletions Objects/dictobject.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -879,7 +879,7 @@ new_dict(PyInterpreterState *interp,
mp->ma_keys = keys;
mp->ma_values = values;
mp->ma_used = used;
mp->ma_version_tag = DICT_NEXT_VERSION(interp);
mp->ma_watcher_tag = 0;
ASSERT_CONSISTENT(mp);
return (PyObject *)mp;
}
Expand Down Expand Up @@ -1680,8 +1680,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp,
}
}

uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_ADDED, mp, key, value);
_PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);
mp->ma_keys->dk_version = 0;

Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash);
Expand All @@ -1700,7 +1699,6 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp,
STORE_VALUE(ep, value);
STORE_HASH(ep, hash);
}
mp->ma_version_tag = new_version;
STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1);
STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1);
assert(mp->ma_keys->dk_usable >= 0);
Expand Down Expand Up @@ -1746,16 +1744,14 @@ insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key,
MAINTAIN_TRACKING(mp, key, value);
PyObject *old_value = mp->ma_values->values[ix];
if (old_value == NULL) {
uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);
_PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);
STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value));
_PyDictValues_AddToInsertionOrder(mp->ma_values, ix);
STORE_USED(mp, mp->ma_used + 1);
mp->ma_version_tag = new_version;
}
else {
uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value);
_PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value);
STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value));
mp->ma_version_tag = new_version;
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_DECREF(old_value);
Expand Down Expand Up @@ -1817,8 +1813,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp,
}

if (old_value != value) {
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_MODIFIED, mp, key, value);
_PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value);
assert(old_value != NULL);
assert(!_PyDict_HasSplitTable(mp));
if (DK_IS_UNICODE(mp->ma_keys)) {
Expand All @@ -1829,7 +1824,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp,
PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix];
STORE_VALUE(ep, value);
}
mp->ma_version_tag = new_version;
}
Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */
ASSERT_CONSISTENT(mp);
Expand Down Expand Up @@ -1859,8 +1853,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp,
Py_DECREF(value);
return -1;
}
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_ADDED, mp, key, value);
_PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);

/* We don't decref Py_EMPTY_KEYS here because it is immortal. */
assert(mp->ma_values == NULL);
Expand All @@ -1881,7 +1874,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp,
STORE_VALUE(ep, value);
}
STORE_USED(mp, mp->ma_used + 1);
mp->ma_version_tag = new_version;
newkeys->dk_usable--;
newkeys->dk_nentries++;
// We store the keys last so no one can see them in a partially inconsistent
Expand Down Expand Up @@ -2614,7 +2606,7 @@ delete_index_from_values(PyDictValues *values, Py_ssize_t ix)

static void
delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
PyObject *old_value, uint64_t new_version)
PyObject *old_value)
{
PyObject *old_key;

Expand All @@ -2624,7 +2616,6 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
assert(hashpos >= 0);

STORE_USED(mp, mp->ma_used - 1);
mp->ma_version_tag = new_version;
if (_PyDict_HasSplitTable(mp)) {
assert(old_value == mp->ma_values->values[ix]);
STORE_SPLIT_VALUE(mp, ix, NULL);
Expand Down Expand Up @@ -2694,9 +2685,8 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash)
}

PyInterpreterState *interp = _PyInterpreterState_GET();
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_DELETED, mp, key, NULL);
delitem_common(mp, hash, ix, old_value, new_version);
_PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL);
delitem_common(mp, hash, ix, old_value);
return 0;
}

Expand Down Expand Up @@ -2742,8 +2732,7 @@ delitemif_lock_held(PyObject *op, PyObject *key,

if (res > 0) {
PyInterpreterState *interp = _PyInterpreterState_GET();
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_DELETED, mp, key, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL);
delitem_common(mp, hash, ix, old_value, new_version);
return 1;
} else {
Expand Down Expand Up @@ -2788,11 +2777,9 @@ clear_lock_held(PyObject *op)
}
/* Empty the dict... */
PyInterpreterState *interp = _PyInterpreterState_GET();
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_CLEARED, mp, NULL, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_CLEARED, mp, NULL, NULL);
// We don't inc ref empty keys because they're immortal
ensure_shared_on_resize(mp);
mp->ma_version_tag = new_version;
STORE_USED(mp, 0);
if (oldvalues == NULL) {
set_keys(mp, Py_EMPTY_KEYS);
Expand Down Expand Up @@ -2952,8 +2939,7 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash,

assert(old_value != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET();
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_DELETED, mp, key, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL);
delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version);

ASSERT_CONSISTENT(mp);
Expand Down Expand Up @@ -3719,8 +3705,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
) {
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
PyDictKeysObject *keys = clone_combined_dict_keys(other);
if (keys == NULL)
return -1;
Expand All @@ -3729,7 +3714,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp));
mp->ma_keys = keys;
STORE_USED(mp, other->ma_used);
mp->ma_version_tag = new_version;
ASSERT_CONSISTENT(mp);

if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) {
Expand Down Expand Up @@ -3984,7 +3968,7 @@ copy_lock_held(PyObject *o)
split_copy->ma_values = newvalues;
split_copy->ma_keys = mp->ma_keys;
split_copy->ma_used = mp->ma_used;
split_copy->ma_version_tag = DICT_NEXT_VERSION(interp);
split_copy->ma_watcher_tag = 0;
dictkeys_incref(mp->ma_keys);
if (_PyObject_GC_IS_TRACKED(mp))
_PyObject_GC_TRACK(split_copy);
Expand Down Expand Up @@ -4475,8 +4459,7 @@ dict_popitem_impl(PyDictObject *self)
assert(i >= 0);

key = ep0[i].me_key;
new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_DELETED, self, key, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL);
hash = unicode_get_hash(key);
value = ep0[i].me_value;
ep0[i].me_key = NULL;
Expand All @@ -4491,8 +4474,7 @@ dict_popitem_impl(PyDictObject *self)
assert(i >= 0);

key = ep0[i].me_key;
new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_DELETED, self, key, NULL);
_PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL);
hash = ep0[i].me_hash;
value = ep0[i].me_value;
ep0[i].me_key = NULL;
Expand All @@ -4510,7 +4492,6 @@ dict_popitem_impl(PyDictObject *self)
/* We can't dk_usable++ since there is DKIX_DUMMY in indices */
STORE_KEYS_NENTRIES(self->ma_keys, i);
STORE_USED(self, self->ma_used - 1);
self->ma_version_tag = new_version;
ASSERT_CONSISTENT(self);
return res;
}
Expand Down Expand Up @@ -4761,8 +4742,7 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
PyDictObject *d = (PyDictObject *)self;

d->ma_used = 0;
d->ma_version_tag = DICT_NEXT_VERSION(
_PyInterpreterState_GET());
d->ma_watcher_tag = 0;
dictkeys_incref(Py_EMPTY_KEYS);
d->ma_keys = Py_EMPTY_KEYS;
d->ma_values = NULL;
Expand Down Expand Up @@ -7379,7 +7359,7 @@ PyDict_Watch(int watcher_id, PyObject* dict)
if (validate_watcher_id(interp, watcher_id)) {
return -1;
}
((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id);
((PyDictObject*)dict)->ma_watcher_tag |= (1LL << watcher_id);
return 0;
}

Expand All @@ -7394,7 +7374,7 @@ PyDict_Unwatch(int watcher_id, PyObject* dict)
if (validate_watcher_id(interp, watcher_id)) {
return -1;
}
((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id);
((PyDictObject*)dict)->ma_watcher_tag &= ~(1LL << watcher_id);
return 0;
}

Expand Down
4 changes: 1 addition & 3 deletions Python/bytecodes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2255,7 +2255,6 @@ dummy_func(
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *old_value;
uint64_t new_version;
DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
Expand All @@ -2265,9 +2264,8 @@ dummy_func(
}
old_value = ep->me_value;
PyDict_WatchEvent event = old_value == NULL ? PyDict_EVENT_ADDED : PyDict_EVENT_MODIFIED;
new_version = _PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value));
_PyDict_NotifyEvent(tstate->interp, event, dict, name, PyStackRef_AsPyObjectBorrow(value));
ep->me_value = PyStackRef_AsPyObjectSteal(value);
dict->ma_version_tag = new_version; // PEP 509
// old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault,
// when dict only holds the strong reference to value in ep->me_value.
Py_XDECREF(old_value);
Expand Down
4 changes: 2 additions & 2 deletions Python/optimizer_analysis.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,14 @@ static int
get_mutations(PyObject* dict) {
assert(PyDict_CheckExact(dict));
PyDictObject *d = (PyDictObject *)dict;
return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1);
return (d->ma_watcher_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1);
}

static void
increment_mutations(PyObject* dict) {
assert(PyDict_CheckExact(dict));
PyDictObject *d = (PyDictObject *)dict;
d->ma_version_tag += (1 << DICT_MAX_WATCHERS);
d->ma_watcher_tag += (1 << DICT_MAX_WATCHERS);
}

/* The first two dict watcher IDs are reserved for CPython,
Expand Down

0 comments on commit 45814d4

Please sign in to comment.