FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-124218: Use per-thread refcounts for code objects (#125216) · wrongnull/cpython@3ea488a · GitHub

/ cpython Public
forked from python/cpython

Commit 3ea488a

Browse files
colesbury
authored
pythongh-124218: Use per-thread refcounts for code objects (python#125216)
Use per-thread refcounting for the reference from function objects to their corresponding code object. This can be a source of contention when frequently creating nested functions. Deferred refcounting alone isn't a great fit here because these references are on the heap and may be modified by other libraries.
1 parent 206de41 commit 3ea488a

11 files changed

Lines changed: 119 additions & 76 deletions

File tree

‎Include/cpython/code.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ typedef struct {
132132
_PyCoCached *_co_cached; /* cached co_* attributes */ \
133133
uintptr_t _co_instrumentation_version; /* current instrumentation version */ \
134134
_PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \
135+
Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \
135136
int _co_firsttraceable; /* index of first traceable instruction */ \
136137
/* Scratch space for extra data relating to the code object. \
137138
Type is a void* to keep the format private in codeobject.c to force \

‎Include/cpython/object.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ typedef struct _heaptypeobject {
272272
void *ht_token; // Storage for the "Py_tp_token" slot
273273
struct _specialization_cache _spec_cache; // For use by the specializer.
274274
#ifdef Py_GIL_DISABLED
275-
Py_ssize_t unique_id; // ID used for thread-local refcounting
275+
Py_ssize_t unique_id; // ID used for per-thread refcounting
276276
#endif
277277
/* here are optional user slots, followed by the members. */
278278
} PyHeapTypeObject;

‎Include/internal/pycore_object.h‎

Lines changed: 53 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ extern "C" {
1414
#include "pycore_interp.h" // PyInterpreterState.gc
1515
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
1616
#include "pycore_pystate.h" // _PyInterpreterState_GET()
17-
#include "pycore_uniqueid.h" // _PyType_IncrefSlow
17+
#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow()
1818

1919
// This value is added to `ob_ref_shared` for objects that use deferred
2020
// reference counting so that they are not immediately deallocated when the
@@ -291,7 +291,31 @@ extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
291291
#ifndef Py_GIL_DISABLED
292292
# define _Py_INCREF_TYPE Py_INCREF
293293
# define _Py_DECREF_TYPE Py_DECREF
294+
# define _Py_INCREF_CODE Py_INCREF
295+
# define _Py_DECREF_CODE Py_DECREF
294296
#else
297+
static inline void
298+
_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
299+
{
300+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
301+
302+
// Unsigned comparison so that `unique_id=-1`, which indicates that
303+
// per-thread refcounting has been disabled on this object, is handled by
304+
// the "else".
305+
if ((size_t)unique_id < (size_t)tstate->refcounts.size) {
306+
# ifdef Py_REF_DEBUG
307+
_Py_INCREF_IncRefTotal();
308+
# endif
309+
_Py_INCREF_STAT_INC();
310+
tstate->refcounts.values[unique_id]++;
311+
}
312+
else {
313+
// The slow path resizes the per-thread refcount array if necessary.
314+
// It handles the unique_id=-1 case to keep the inlinable function smaller.
315+
_PyObject_ThreadIncrefSlow(obj, unique_id);
316+
}
317+
}
318+
295319
static inline void
296320
_Py_INCREF_TYPE(PyTypeObject *type)
297321
{
@@ -308,29 +332,38 @@ _Py_INCREF_TYPE(PyTypeObject *type)
308332
# pragma GCC diagnostic push
309333
# pragma GCC diagnostic ignored "-Warray-bounds"
310334
#endif
335+
_Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id);
336+
#if defined(__GNUC__) && __GNUC__ >= 11
337+
# pragma GCC diagnostic pop
338+
#endif
339+
}
340+
341+
static inline void
342+
_Py_INCREF_CODE(PyCodeObject *co)
343+
{
344+
_Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id);
345+
}
311346

347+
static inline void
348+
_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
349+
{
312350
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
313-
PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
314351

315352
// Unsigned comparison so that `unique_id=-1`, which indicates that
316-
// per-thread refcounting has been disabled on this type, is handled by
353+
// per-thread refcounting has been disabled on this object, is handled by
317354
// the "else".
318-
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
355+
if ((size_t)unique_id < (size_t)tstate->refcounts.size) {
319356
# ifdef Py_REF_DEBUG
320-
_Py_INCREF_IncRefTotal();
357+
_Py_DECREF_DecRefTotal();
321358
# endif
322-
_Py_INCREF_STAT_INC();
323-
tstate->refcounts.values[ht->unique_id]++;
359+
_Py_DECREF_STAT_INC();
360+
tstate->refcounts.values[unique_id]--;
324361
}
325362
else {
326-
// The slow path resizes the thread-local refcount array if necessary.
327-
// It handles the unique_id=-1 case to keep the inlinable function smaller.
328-
_PyType_IncrefSlow(ht);
363+
// Directly decref the object if the id is not assigned or if
364+
// per-thread refcounting has been disabled on this object.
365+
Py_DECREF(obj);
329366
}
330-
331-
#if defined(__GNUC__) && __GNUC__ >= 11
332-
# pragma GCC diagnostic pop
333-
#endif
334367
}
335368

336369
static inline void
@@ -341,25 +374,14 @@ _Py_DECREF_TYPE(PyTypeObject *type)
341374
_Py_DECREF_IMMORTAL_STAT_INC();
342375
return;
343376
}
344-
345-
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
346377
PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
378+
_Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id);
379+
}
347380

348-
// Unsigned comparison so that `unique_id=-1`, which indicates that
349-
// per-thread refcounting has been disabled on this type, is handled by
350-
// the "else".
351-
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
352-
# ifdef Py_REF_DEBUG
353-
_Py_DECREF_DecRefTotal();
354-
# endif
355-
_Py_DECREF_STAT_INC();
356-
tstate->refcounts.values[ht->unique_id]--;
357-
}
358-
else {
359-
// Directly decref the type if the type id is not assigned or if
360-
// per-thread refcounting has been disabled on this type.
361-
Py_DECREF(type);
362-
}
381+
static inline void
382+
_Py_DECREF_CODE(PyCodeObject *co)
383+
{
384+
_Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id);
363385
}
364386
#endif
365387

‎Include/internal/pycore_uniqueid.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ struct _Py_unique_id_pool {
4949
extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
5050

5151
// Releases the allocated id back to the pool.
52-
extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id);
52+
extern void _PyObject_DisablePerThreadRefcounting(PyObject *obj);
5353

5454
// Merges the per-thread reference counts into the corresponding objects.
5555
extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate);
@@ -61,8 +61,8 @@ extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate);
6161
// Frees the interpreter's pool of type ids.
6262
extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp);
6363

64-
// Increfs the type, resizing the per-thread refcount array if necessary.
65-
PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
64+
// Increfs the object, resizing the thread-local refcount array if necessary.
65+
PyAPI_FUNC(void) _PyObject_ThreadIncrefSlow(PyObject *obj, Py_ssize_t unique_id);
6666

6767
#endif /* Py_GIL_DISABLED */
6868

‎Objects/codeobject.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "pycore_pystate.h" // _PyInterpreterState_GET()
1515
#include "pycore_setobject.h" // _PySet_NextEntry()
1616
#include "pycore_tuple.h" // _PyTuple_ITEMS()
17+
#include "pycore_uniqueid.h" // _PyObject_AssignUniqueId()
1718
#include "clinic/codeobject.c.h"
1819

1920
static const char *
@@ -676,7 +677,7 @@ _PyCode_New(struct _PyCodeConstructor *con)
676677
}
677678
init_code(co, con);
678679
#ifdef Py_GIL_DISABLED
679-
_PyObject_SetDeferredRefcount((PyObject *)co);
680+
co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co);
680681
_PyObject_GC_TRACK(co);
681682
#endif
682683
Py_XDECREF(replacement_locations);
@@ -1864,6 +1865,9 @@ code_dealloc(PyCodeObject *co)
18641865
Py_XDECREF(co->co_qualname);
18651866
Py_XDECREF(co->co_linetable);
18661867
Py_XDECREF(co->co_exceptiontable);
1868+
#ifdef Py_GIL_DISABLED
1869+
assert(co->_co_unique_id == -1);
1870+
#endif
18671871
if (co->_co_cached != NULL) {
18681872
Py_XDECREF(co->_co_cached->_co_code);
18691873
Py_XDECREF(co->_co_cached->_co_cellvars);

‎Objects/funcobject.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr)
116116
op->func_builtins = Py_NewRef(constr->fc_builtins);
117117
op->func_name = Py_NewRef(constr->fc_name);
118118
op->func_qualname = Py_NewRef(constr->fc_qualname);
119-
op->func_code = Py_NewRef(constr->fc_code);
119+
_Py_INCREF_CODE((PyCodeObject *)constr->fc_code);
120+
op->func_code = constr->fc_code;
120121
op->func_defaults = Py_XNewRef(constr->fc_defaults);
121122
op->func_kwdefaults = Py_XNewRef(constr->fc_kwdefaults);
122123
op->func_closure = Py_XNewRef(constr->fc_closure);
@@ -146,7 +147,8 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
146147

147148
PyThreadState *tstate = _PyThreadState_GET();
148149

149-
PyCodeObject *code_obj = (PyCodeObject *)Py_NewRef(code);
150+
PyCodeObject *code_obj = (PyCodeObject *)code;
151+
_Py_INCREF_CODE(code_obj);
150152

151153
assert(code_obj->co_name != NULL);
152154
PyObject *name = Py_NewRef(code_obj->co_name);
@@ -1094,7 +1096,7 @@ func_dealloc(PyObject *self)
10941096
}
10951097
(void)func_clear((PyObject*)op);
10961098
// These aren't cleared by func_clear().
1097-
Py_DECREF(op->func_code);
1099+
_Py_DECREF_CODE((PyCodeObject *)op->func_code);
10981100
Py_DECREF(op->func_name);
10991101
Py_DECREF(op->func_qualname);
11001102
PyObject_GC_Del(op);

‎Objects/typeobject.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5025,7 +5025,7 @@ PyType_FromMetaclass(
50255025
type->tp_dictoffset = dictoffset;
50265026

50275027
#ifdef Py_GIL_DISABLED
5028-
// Assign a type id to enable thread-local refcounting
5028+
// Assign a unique id to enable per-thread refcounting
50295029
res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
50305030
#endif
50315031

@@ -6043,7 +6043,7 @@ type_dealloc(PyObject *self)
60436043
Py_XDECREF(et->ht_module);
60446044
PyMem_Free(et->_ht_tpname);
60456045
#ifdef Py_GIL_DISABLED
6046-
_PyObject_ReleaseUniqueId(et->unique_id);
6046+
assert(et->unique_id == -1);
60476047
#endif
60486048
et->ht_token = NULL;
60496049
Py_TYPE(type)->tp_free((PyObject *)type);

‎Python/gc_free_threading.c‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
#include "pycore_tstate.h" // _PyThreadStateImpl
1616
#include "pycore_weakref.h" // _PyWeakref_ClearRef()
1717
#include "pydtrace.h"
18-
#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts
18+
#include "pycore_uniqueid.h" // _PyObject_MergeThreadLocalRefcounts()
1919

2020
#ifdef Py_GIL_DISABLED
2121

@@ -215,15 +215,10 @@ disable_deferred_refcounting(PyObject *op)
215215
op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED;
216216
op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0);
217217
merge_refcount(op, 0);
218-
}
219218

220-
// Heap types also use per-thread refcounting -- disable it here.
221-
if (PyType_Check(op)) {
222-
if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
223-
PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
224-
_PyObject_ReleaseUniqueId(ht->unique_id);
225-
ht->unique_id = -1;
226-
}
219+
// Heap types and code objects also use per-thread refcounting, which
220+
// should also be disabled when we turn off deferred refcounting.
221+
_PyObject_DisablePerThreadRefcounting(op);
227222
}
228223

229224
// Generators and frame objects may contain deferred references to other

‎Python/pylifecycle.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
#include "pycore_sliceobject.h" // _PySlice_Fini()
2929
#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
3030
#include "pycore_traceback.h" // _Py_DumpTracebackThreads()
31-
#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool()
31+
#include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool()
3232
#include "pycore_typeobject.h" // _PyTypes_InitTypes()
3333
#include "pycore_typevarobject.h" // _Py_clear_generic_types()
3434
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()

‎Python/pystate.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include "pycore_runtime_init.h" // _PyRuntimeState_INIT
2121
#include "pycore_sysmodule.h" // _PySys_Audit()
2222
#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap()
23-
#include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts()
23+
#include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts()
2424

2525
/* --------------------------------------------------------------------------
2626
CAUTION

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL