From 37d8189170ee7145ef2ee6bf8bc343d152a246fe Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Mon, 18 May 2026 13:46:19 +0100 Subject: [PATCH 1/4] Add `frozendict` to ast constants --- Lib/test/test_ast/test_ast.py | 11 +++++++++-- Python/ast.c | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 0112c9163fd0cd..5fb3aa747c7f18 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -2683,6 +2683,11 @@ def test_validation(self): self.assertEqual(str(cm.exception), "got an invalid type in Constant: list") + with self.assertRaises(TypeError) as cm: + self.compile_constant(frozendict({1: [2, 3]})) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + def test_singletons(self): for const in (None, False, True, Ellipsis, b''): with self.subTest(const=const): @@ -2692,13 +2697,15 @@ def test_singletons(self): def test_values(self): nested_tuple = (1,) nested_frozenset = frozenset({1}) + nested_frozendict = frozendict({1: 1}) for level in range(3): nested_tuple = (nested_tuple, 2) nested_frozenset = frozenset({nested_frozenset, 2}) + nested_frozendict = frozendict({nested_frozendict: 2}) values = (123, 123.0, 123j, "unicode", b'bytes', - tuple("tuple"), frozenset("frozenset"), - nested_tuple, nested_frozenset) + tuple("tuple"), frozenset("frozenset"), frozendict({"a": 1}), + nested_tuple, nested_frozenset, nested_frozendict) for value in values: with self.subTest(value=value): result = self.compile_constant(value) diff --git a/Python/ast.c b/Python/ast.c index 4cfa2ff559a5f7..286787a0e8ca9f 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -198,6 +198,23 @@ validate_constant(PyObject *value) return 1; } + if (PyFrozenDict_CheckExact(value)) { + ENTER_RECURSIVE(); + + Py_ssize_t pos = 0; + PyObject *key, *val; + + while (PyDict_Next(value, &pos, &key, &val)) { + if (!validate_constant(key) || !validate_constant(val)) { + LEAVE_RECURSIVE(); + return 0; + } + } + + LEAVE_RECURSIVE(); + return 1; + } + if (!PyErr_Occurred()) { PyErr_Format(PyExc_TypeError, "got an invalid type in Constant: %s", From b027839854079185515aea79b6ea9509e3f3142b Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Tue, 19 May 2026 01:15:35 +0100 Subject: [PATCH 2/4] Add `frozendict` support to `_PyCode_ConstantKey` --- Lib/test/test_compile.py | 32 ++++++++++++++++++++++++++++++++ Objects/codeobject.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9edbca3c383b43..47fcc9a9456475 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -776,6 +776,13 @@ def check_constant(self, func, expected): self.fail("unable to find constant %r in %r" % (expected, func.__code__.co_consts)) + @staticmethod + def _frozen_dict_consts(*consts): + """Use AST to make frozendict constants since it has no literal syntax""" + m = ast.Interactive([ast.Expr(ast.Constant(c)) for c in consts]) + ast.fix_missing_locations(m) + return compile(m, "", "single") + # Merging equal constants is not a strict requirement for the Python # semantics, it's a more an implementation detail. @support.cpython_only @@ -820,6 +827,21 @@ def check_same_constant(const): self.check_constant(f1, frozenset({0})) self.assertTrue(f1(0)) + # two identical frozendicts merge into one constant + c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0: 1})) + self.assertEqual(c.co_consts, (frozendict({0: 1}),)) + + # empty frozendicts also merge + c = self._frozen_dict_consts(frozendict(), frozendict()) + self.assertEqual(c.co_consts, (frozendict(),)) + + # frozendicts containing a nested frozendict value merge + c = self._frozen_dict_consts( + frozendict({0: frozendict({1: 2})}), + frozendict({0: frozendict({1: 2})}), + ) + self.assertEqual(c.co_consts, (frozendict({0: frozendict({1: 2})}),)) + # Merging equal co_linetable is not a strict requirement # for the Python semantics, it's a more an implementation detail. @support.cpython_only @@ -1033,6 +1055,16 @@ def check_different_constants(const1, const2): self.assertTrue(f1(0)) self.assertTrue(f2(0.0)) + # frozendicts with type-distinct keys must not merge (0 vs 0.0) + c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0.0: 1})) + self.assertEqual(c.co_consts, (frozendict({0: 1}), frozendict({0.0: 1}))) + self.assertIsNot(c.co_consts[0], c.co_consts[1]) + + # frozendicts with type-distinct values must not merge (1 vs 1.0) + c = self._frozen_dict_consts(frozendict({0: 1}), frozendict({0: 1.0})) + self.assertEqual(c.co_consts, (frozendict({0: 1}), frozendict({0: 1.0}))) + self.assertIsNot(c.co_consts[0], c.co_consts[1]) + def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 8be85b1accbdca..62be2e614535e1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -3039,6 +3039,45 @@ _PyCode_ConstantKey(PyObject *op) Py_DECREF(set); return key; } + else if (PyFrozenDict_CheckExact(op)) { + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + + Py_ssize_t pos = 0; + PyObject *k_obj, *v_obj; + while (PyDict_Next(op, &pos, &k_obj, &v_obj)) { + PyObject *k_key = _PyCode_ConstantKey(k_obj); + if (k_key == NULL) { + Py_DECREF(dict); + return NULL; + } + PyObject *v_key = _PyCode_ConstantKey(v_obj); + if (v_key == NULL) { + Py_DECREF(k_key); + Py_DECREF(dict); + return NULL; + } + int res = PyDict_SetItem(dict, k_key, v_key); + Py_DECREF(k_key); + Py_DECREF(v_key); + if (res < 0) { + Py_DECREF(dict); + return NULL; + } + } + + PyObject *fdict = PyFrozenDict_New(dict); + Py_DECREF(dict); + if (fdict == NULL) { + return NULL; + } + + key = _PyTuple_FromPair(fdict, op); + Py_DECREF(fdict); + return key; + } else if (PySlice_Check(op)) { PySliceObject *slice = (PySliceObject *)op; PyObject *start_key = NULL; From 992b4b6f07c5a15a7d7fe3ee94ff7d42f8890faa Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Tue, 19 May 2026 01:42:49 +0100 Subject: [PATCH 3/4] Add `frozendict` support for recursive constant merging --- Lib/test/test_compile.py | 15 ++++++++ Python/compile.c | 77 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 47fcc9a9456475..a3338d54f37041 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -842,6 +842,21 @@ def check_same_constant(const): ) self.assertEqual(c.co_consts, (frozendict({0: frozendict({1: 2})}),)) + # A tuple value inside a frozendict is merged with the same + # constant used elsewhere. Use a variable to ensure the two tuple + # objects are distinct before they are merged. + name = "not a name" + t_standalone = (name,) + m = ast.Interactive([ + ast.Expr(ast.Constant(t_standalone)), + ast.Expr(ast.Constant(frozendict({0: (name,)}))), + ast.Expr(ast.Constant(frozendict({(name,): 0}))), + ]) + ast.fix_missing_locations(m) + c = compile(m, "", "single") + self.assertIs(c.co_consts[0], c.co_consts[1][0]) + self.assertIs(c.co_consts[0], next(iter(c.co_consts[2]))) + # Merging equal co_linetable is not a strict requirement # for the Python semantics, it's a more an implementation detail. @support.cpython_only diff --git a/Python/compile.c b/Python/compile.c index eb9fc827bea40a..972f83d97d47c5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -332,7 +332,7 @@ compiler_set_qualname(compiler *c) } /* Merge const *o* and return constant key object. - * If recursive, insert all elements if o is a tuple or frozen set. + * If recursive, insert all elements if o is a tuple, frozenset, or frozendict. */ static PyObject* const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) @@ -364,7 +364,7 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) } // We registered o in const_cache. - // When o is a tuple or frozenset, we want to merge its + // When o is a tuple, frozenset, or frozendict, we want to merge its // items too. if (PyTuple_CheckExact(o)) { Py_ssize_t len = PyTuple_GET_SIZE(o); @@ -431,7 +431,7 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) } // Instead of rewriting o, we create new frozenset and embed in the - // key tuple. Caller should get merged frozenset from the key tuple. + // key tuple. Caller should get merged frozenset from the key tuple. PyObject *new = PyFrozenSet_New(tuple); Py_DECREF(tuple); if (new == NULL) { @@ -442,6 +442,77 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) Py_DECREF(o); PyTuple_SET_ITEM(key, 1, new); } + else if (PyFrozenDict_CheckExact(o)) { + // *key* is tuple. And its first item is frozendict of + // constant keys. + // See _PyCode_ConstantKey() for detail. + assert(PyTuple_CheckExact(key)); + assert(PyTuple_GET_SIZE(key) == 2); + + if (PyDict_GET_SIZE(o) == 0) { // empty frozendict should not be re-created. + return key; + } + PyObject *new_dict = PyDict_New(); + if (new_dict == NULL) { + Py_DECREF(key); + return NULL; + } + Py_ssize_t pos = 0; + PyObject *k_obj, *v_obj; + while (PyDict_Next(o, &pos, &k_obj, &v_obj)) { + PyObject *k_result = const_cache_insert(const_cache, k_obj, recursive); + if (k_result == NULL) { + Py_DECREF(new_dict); + Py_DECREF(key); + return NULL; + } + PyObject *k_merged; + if (PyTuple_CheckExact(k_result)) { + k_merged = Py_NewRef(PyTuple_GET_ITEM(k_result, 1)); + Py_DECREF(k_result); + } + else { + k_merged = k_result; + } + + PyObject *v_result = const_cache_insert(const_cache, v_obj, recursive); + if (v_result == NULL) { + Py_DECREF(k_merged); + Py_DECREF(new_dict); + Py_DECREF(key); + return NULL; + } + PyObject *v_merged; + if (PyTuple_CheckExact(v_result)) { + v_merged = Py_NewRef(PyTuple_GET_ITEM(v_result, 1)); + Py_DECREF(v_result); + } + else { + v_merged = v_result; + } + + int res = PyDict_SetItem(new_dict, k_merged, v_merged); + Py_DECREF(k_merged); + Py_DECREF(v_merged); + if (res < 0) { + Py_DECREF(new_dict); + Py_DECREF(key); + return NULL; + } + } + + // Instead of rewriting o, we create new frozendict and embed in + // the key tuple. Caller should get merged frozendict from the key tuple. + PyObject *new = PyFrozenDict_New(new_dict); + Py_DECREF(new_dict); + if (new == NULL) { + Py_DECREF(key); + return NULL; + } + assert(PyTuple_GET_ITEM(key, 1) == o); + Py_DECREF(o); + PyTuple_SET_ITEM(key, 1, new); + } return key; } From aeed04a0562d7c981c3fc724db56df0a111236ed Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Tue, 19 May 2026 03:37:54 +0100 Subject: [PATCH 4/4] Fix free-threading --- Include/internal/pycore_dict.h | 1 + Objects/codeobject.c | 76 ++++++++++++++++++++++++++++++++++ Objects/dictobject.c | 28 +++++++++++++ 3 files changed, 105 insertions(+) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index ff6588b3e9718c..55c6329c76d139 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -176,6 +176,7 @@ extern void _PyDict_Clear_LockHeld(PyObject *op); #ifdef Py_GIL_DISABLED PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); +extern void _PyFrozenDict_ClearInternal(PyObject *op); #endif // Export for '_elementtree' shared extension diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 62be2e614535e1..5d3c358bac29e9 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_FreeDelayed() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_dict.h" // _PyFrozenDict_ClearInternal() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() @@ -167,6 +168,16 @@ should_immortalize_constant(PyObject *v) } return 1; } + else if (PyFrozenDict_CheckExact(v)) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(v, &pos, &key, &value)) { + if (!_Py_IsImmortal(key) || !_Py_IsImmortal(value)) { + return 0; + } + } + return 1; + } else if (PySlice_Check(v)) { PySliceObject *slice = (PySliceObject *)v; return (_Py_IsImmortal(slice->start) && @@ -248,6 +259,53 @@ intern_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } + else if (PyFrozenDict_CheckExact(v)) { + PyObject *w = v; + PyObject *tmp = PyTuple_New(2 * PyDict_GET_SIZE(v)); + if (tmp == NULL) { + return -1; + } + Py_ssize_t pos = 0; + PyObject *k, *val; + Py_ssize_t j = 0; + while (PyDict_Next(v, &pos, &k, &val)) { + PyTuple_SET_ITEM(tmp, j++, Py_NewRef(k)); + PyTuple_SET_ITEM(tmp, j++, Py_NewRef(val)); + } + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + PyObject *new_dict = PyDict_New(); + if (new_dict == NULL) { + Py_DECREF(tmp); + return -1; + } + for (j = 0; j < PyTuple_GET_SIZE(tmp); j += 2) { + if (PyDict_SetItem(new_dict, + PyTuple_GET_ITEM(tmp, j), + PyTuple_GET_ITEM(tmp, j + 1)) < 0) { + Py_DECREF(tmp); + Py_DECREF(new_dict); + return -1; + } + } + v = PyFrozenDict_New(new_dict); + Py_DECREF(new_dict); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(w); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } #ifdef Py_GIL_DISABLED else if (PySlice_Check(v)) { PySliceObject *slice = (PySliceObject *)v; @@ -3199,6 +3257,21 @@ compare_constants(const void *key1, const void *key2) } return 1; } + else if (PyFrozenDict_CheckExact(op1)) { + if (PyDict_GET_SIZE(op1) != PyDict_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *k1, *k2, *v1, *v2; + while (PyDict_Next(op1, &pos1, &k1, &v1) && + PyDict_Next(op2, &pos2, &k2, &v2)) + { + if (k1 != k2 || v1 != v2) { + return 0; + } + } + return 1; + } else if (PySlice_Check(op1)) { PySliceObject *s1 = (PySliceObject *)op1; PySliceObject *s2 = (PySliceObject *)op2; @@ -3273,6 +3346,9 @@ clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, else if (PyFrozenSet_CheckExact(op)) { _PySet_ClearInternal((PySetObject *)op); } + else if (PyFrozenDict_CheckExact(op)) { + _PyFrozenDict_ClearInternal(op); + } return 0; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7d67812bec925..4e8c92834c3283 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3098,6 +3098,34 @@ _PyDict_Clear_LockHeld(PyObject *op) { clear_lock_held(op); } +#ifdef Py_GIL_DISABLED +void +_PyFrozenDict_ClearInternal(PyObject *op) +{ + assert(PyFrozenDict_CheckExact(op)); + PyDictObject *mp = (PyDictObject *)op; + PyDictKeysObject *keys = mp->ma_keys; + if (keys == Py_EMPTY_KEYS) { + return; + } + assert(mp->ma_values == NULL); + if (DK_IS_UNICODE(keys)) { + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + Py_CLEAR(entries[i].me_key); + Py_CLEAR(entries[i].me_value); + } + } + else { + PyDictKeyEntry *entries = DK_ENTRIES(keys); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + Py_CLEAR(entries[i].me_key); + Py_CLEAR(entries[i].me_value); + } + } +} +#endif + void PyDict_Clear(PyObject *op) {