FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-124218: Use per-thread refcounts for code objects · python/cpython@018b2a1 · GitHub

/ cpython Public

Commit 018b2a1

Browse files
colesbury
committed
gh-124218: Use per-thread refcounts for code objects
Use per-thread refcounting for the reference from function objects to their corresponding code object. This can be a source of contention when frequently creating nested functions. Deferred refcounting alone isn't a great fit here because these references are on the heap and may be modified by other libraries.
1 parent 9bda775 commit 018b2a1

11 files changed

Lines changed: 118 additions & 75 deletions

File tree

‎Include/cpython/code.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ typedef struct {
132132
_PyCoCached *_co_cached; /* cached co_* attributes */ \
133133
uintptr_t _co_instrumentation_version; /* current instrumentation version */ \
134134
_PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \
135+
Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \
135136
int _co_firsttraceable; /* index of first traceable instruction */ \
136137
/* Scratch space for extra data relating to the code object. \
137138
Type is a void* to keep the format private in codeobject.c to force \

‎Include/cpython/object.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ typedef struct _heaptypeobject {
272272
void *ht_token; // Storage for the "Py_tp_token" slot
273273
struct _specialization_cache _spec_cache; // For use by the specializer.
274274
#ifdef Py_GIL_DISABLED
275-
Py_ssize_t unique_id; // ID used for thread-local refcounting
275+
Py_ssize_t unique_id; // ID used for per-thread refcounting
276276
#endif
277277
/* here are optional user slots, followed by the members. */
278278
} PyHeapTypeObject;

‎Include/internal/pycore_object.h‎

Lines changed: 53 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ extern "C" {
1414
#include "pycore_interp.h" // PyInterpreterState.gc
1515
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
1616
#include "pycore_pystate.h" // _PyInterpreterState_GET()
17-
#include "pycore_uniqueid.h" // _PyType_IncrefSlow
17+
#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow()
1818

1919

2020
#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
@@ -311,7 +311,31 @@ extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
311311
#ifndef Py_GIL_DISABLED
312312
# define _Py_INCREF_TYPE Py_INCREF
313313
# define _Py_DECREF_TYPE Py_DECREF
314+
# define _Py_INCREF_CODE Py_INCREF
315+
# define _Py_DECREF_CODE Py_DECREF
314316
#else
317+
static inline void
318+
_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
319+
{
320+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
321+
322+
// Unsigned comparison so that `unique_id=-1`, which indicates that
323+
// per-thread refcounting has been disabled on this object, is handled by
324+
// the "else".
325+
if ((size_t)unique_id < (size_t)tstate->refcounts.size) {
326+
# ifdef Py_REF_DEBUG
327+
_Py_INCREF_IncRefTotal();
328+
# endif
329+
_Py_INCREF_STAT_INC();
330+
tstate->refcounts.values[unique_id]++;
331+
}
332+
else {
333+
// The slow path resizes the thread-local refcount array if necessary.
334+
// It handles the unique_id=-1 case to keep the inlinable function smaller.
335+
_PyObject_ThreadIncrefSlow(obj, unique_id);
336+
}
337+
}
338+
315339
static inline void
316340
_Py_INCREF_TYPE(PyTypeObject *type)
317341
{
@@ -328,29 +352,38 @@ _Py_INCREF_TYPE(PyTypeObject *type)
328352
# pragma GCC diagnostic push
329353
# pragma GCC diagnostic ignored "-Warray-bounds"
330354
#endif
355+
_Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id);
356+
#if defined(__GNUC__) && __GNUC__ >= 11
357+
# pragma GCC diagnostic pop
358+
#endif
359+
}
331360

361+
static inline void
362+
_Py_INCREF_CODE(PyCodeObject *co)
363+
{
364+
_Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id);
365+
}
366+
367+
static inline void
368+
_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
369+
{
332370
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
333-
PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
334371

335372
// Unsigned comparison so that `unique_id=-1`, which indicates that
336-
// per-thread refcounting has been disabled on this type, is handled by
373+
// per-thread refcounting has been disabled on this object, is handled by
337374
// the "else".
338-
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
375+
if ((size_t)unique_id < (size_t)tstate->refcounts.size) {
339376
# ifdef Py_REF_DEBUG
340-
_Py_INCREF_IncRefTotal();
377+
_Py_DECREF_DecRefTotal();
341378
# endif
342-
_Py_INCREF_STAT_INC();
343-
tstate->refcounts.values[ht->unique_id]++;
379+
_Py_DECREF_STAT_INC();
380+
tstate->refcounts.values[unique_id]--;
344381
}
345382
else {
346-
// The slow path resizes the thread-local refcount array if necessary.
347-
// It handles the unique_id=-1 case to keep the inlinable function smaller.
348-
_PyType_IncrefSlow(ht);
383+
// Directly decref the type if the type id is not assigned or if
384+
// per-thread refcounting has been disabled on this type.
385+
Py_DECREF(obj);
349386
}
350-
351-
#if defined(__GNUC__) && __GNUC__ >= 11
352-
# pragma GCC diagnostic pop
353-
#endif
354387
}
355388

356389
static inline void
@@ -361,25 +394,14 @@ _Py_DECREF_TYPE(PyTypeObject *type)
361394
_Py_DECREF_IMMORTAL_STAT_INC();
362395
return;
363396
}
364-
365-
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
366397
PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
398+
_Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id);
399+
}
367400

368-
// Unsigned comparison so that `unique_id=-1`, which indicates that
369-
// per-thread refcounting has been disabled on this type, is handled by
370-
// the "else".
371-
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
372-
# ifdef Py_REF_DEBUG
373-
_Py_DECREF_DecRefTotal();
374-
# endif
375-
_Py_DECREF_STAT_INC();
376-
tstate->refcounts.values[ht->unique_id]--;
377-
}
378-
else {
379-
// Directly decref the type if the type id is not assigned or if
380-
// per-thread refcounting has been disabled on this type.
381-
Py_DECREF(type);
382-
}
401+
static inline void
402+
_Py_DECREF_CODE(PyCodeObject *co)
403+
{
404+
_Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id);
383405
}
384406
#endif
385407

‎Include/internal/pycore_uniqueid.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ struct _Py_unique_id_pool {
4949
extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
5050

5151
// Releases the allocated id back to the pool.
52-
extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id);
52+
extern void _PyObject_DisablePerThreadRefcounting(PyObject *obj);
5353

5454
// Merges the per-thread reference counts into the corresponding objects.
5555
extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate);
@@ -61,8 +61,8 @@ extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate);
6161
// Frees the interpreter's pool of type ids.
6262
extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp);
6363

64-
// Increfs the type, resizing the per-thread refcount array if necessary.
65-
PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
64+
// Increfs the object, resizing the thread-local refcount array if necessary.
65+
PyAPI_FUNC(void) _PyObject_ThreadIncrefSlow(PyObject *obj, Py_ssize_t unique_id);
6666

6767
#endif /* Py_GIL_DISABLED */
6868

‎Objects/codeobject.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "pycore_pystate.h" // _PyInterpreterState_GET()
1515
#include "pycore_setobject.h" // _PySet_NextEntry()
1616
#include "pycore_tuple.h" // _PyTuple_ITEMS()
17+
#include "pycore_uniqueid.h" // _PyObject_AssignUniqueId()
1718
#include "clinic/codeobject.c.h"
1819

1920
static const char *
@@ -676,7 +677,7 @@ _PyCode_New(struct _PyCodeConstructor *con)
676677
}
677678
init_code(co, con);
678679
#ifdef Py_GIL_DISABLED
679-
_PyObject_SetDeferredRefcount((PyObject *)co);
680+
co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co);
680681
_PyObject_GC_TRACK(co);
681682
#endif
682683
Py_XDECREF(replacement_locations);
@@ -1864,6 +1865,9 @@ code_dealloc(PyCodeObject *co)
18641865
Py_XDECREF(co->co_qualname);
18651866
Py_XDECREF(co->co_linetable);
18661867
Py_XDECREF(co->co_exceptiontable);
1868+
#ifdef Py_GIL_DISABLED
1869+
assert(co->_co_unique_id == -1);
1870+
#endif
18671871
if (co->_co_cached != NULL) {
18681872
Py_XDECREF(co->_co_cached->_co_code);
18691873
Py_XDECREF(co->_co_cached->_co_cellvars);

‎Objects/funcobject.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr)
116116
op->func_builtins = Py_NewRef(constr->fc_builtins);
117117
op->func_name = Py_NewRef(constr->fc_name);
118118
op->func_qualname = Py_NewRef(constr->fc_qualname);
119-
op->func_code = Py_NewRef(constr->fc_code);
119+
_Py_INCREF_CODE((PyCodeObject *)constr->fc_code);
120+
op->func_code = constr->fc_code;
120121
op->func_defaults = Py_XNewRef(constr->fc_defaults);
121122
op->func_kwdefaults = Py_XNewRef(constr->fc_kwdefaults);
122123
op->func_closure = Py_XNewRef(constr->fc_closure);
@@ -146,7 +147,8 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
146147

147148
PyThreadState *tstate = _PyThreadState_GET();
148149

149-
PyCodeObject *code_obj = (PyCodeObject *)Py_NewRef(code);
150+
PyCodeObject *code_obj = (PyCodeObject *)code;
151+
_Py_INCREF_CODE(code_obj);
150152

151153
assert(code_obj->co_name != NULL);
152154
PyObject *name = Py_NewRef(code_obj->co_name);
@@ -1094,7 +1096,7 @@ func_dealloc(PyObject *self)
10941096
}
10951097
(void)func_clear((PyObject*)op);
10961098
// These aren't cleared by func_clear().
1097-
Py_DECREF(op->func_code);
1099+
_Py_DECREF_CODE((PyCodeObject *)op->func_code);
10981100
Py_DECREF(op->func_name);
10991101
Py_DECREF(op->func_qualname);
11001102
PyObject_GC_Del(op);

‎Objects/typeobject.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5025,7 +5025,7 @@ PyType_FromMetaclass(
50255025
type->tp_dictoffset = dictoffset;
50265026

50275027
#ifdef Py_GIL_DISABLED
5028-
// Assign a type id to enable thread-local refcounting
5028+
// Assign a unique id to enable per-thread refcounting
50295029
res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
50305030
#endif
50315031

@@ -6080,7 +6080,7 @@ type_dealloc(PyObject *self)
60806080
Py_XDECREF(et->ht_module);
60816081
PyMem_Free(et->_ht_tpname);
60826082
#ifdef Py_GIL_DISABLED
6083-
_PyObject_ReleaseUniqueId(et->unique_id);
6083+
assert(et->unique_id == -1);
60846084
#endif
60856085
et->ht_token = NULL;
60866086
Py_TYPE(type)->tp_free((PyObject *)type);

‎Python/gc_free_threading.c‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
#include "pycore_tstate.h" // _PyThreadStateImpl
1616
#include "pycore_weakref.h" // _PyWeakref_ClearRef()
1717
#include "pydtrace.h"
18-
#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts
18+
#include "pycore_uniqueid.h" // _PyObject_MergeThreadLocalRefcounts()
1919

2020
#ifdef Py_GIL_DISABLED
2121

@@ -215,15 +215,10 @@ disable_deferred_refcounting(PyObject *op)
215215
op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED;
216216
op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0);
217217
merge_refcount(op, 0);
218-
}
219218

220-
// Heap types also use per-thread refcounting -- disable it here.
221-
if (PyType_Check(op)) {
222-
if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
223-
PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
224-
_PyObject_ReleaseUniqueId(ht->unique_id);
225-
ht->unique_id = -1;
226-
}
219+
// Heap types and code objects also use per-thread refcounting, which
220+
// should also be disabled when we turn off deferred refcounting.
221+
_PyObject_DisablePerThreadRefcounting(op);
227222
}
228223

229224
// Generators and frame objects may contain deferred references to other

‎Python/pylifecycle.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
#include "pycore_sliceobject.h" // _PySlice_Fini()
2929
#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
3030
#include "pycore_traceback.h" // _Py_DumpTracebackThreads()
31-
#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool()
31+
#include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool()
3232
#include "pycore_typeobject.h" // _PyTypes_InitTypes()
3333
#include "pycore_typevarobject.h" // _Py_clear_generic_types()
3434
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()

‎Python/pystate.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include "pycore_runtime_init.h" // _PyRuntimeState_INIT
2121
#include "pycore_sysmodule.h" // _PySys_Audit()
2222
#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap()
23-
#include "pycore_uniqueid.h" // _PyType_FinalizeThreadLocalRefcounts()
23+
#include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts()
2424

2525
/* --------------------------------------------------------------------------
2626
CAUTION

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL