#include "Python.h"

#include "pycore_ceval.h" // _PyEval_SignalReceived()

#include "pycore_initconfig.h" // _PyStatus_OK()

#include "pycore_interp.h" // _Py_RunGC()

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pylifecycle.h" // _PyErr_Print()

#include "pycore_pymem.h" // _PyMem_IsPtrFreed()

#include "pycore_pystats.h" // _Py_PrintSpecializationStats()

#if defined(_MSC_VER) && SIZEOF_INT == 4

#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value)))

#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL)

static inline void

update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState *tstate)

{

if (tstate == NULL) {

return;

}

if (_Py_IsMainThread()) {

int32_t calls_to_do = _Py_atomic_load_int32_relaxed(

&_PyRuntime.ceval.pending_mainthread.calls_to_do);

if (calls_to_do) {

_Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1);

}

if (_Py_ThreadCanHandleSignals(interp)) {

if (_Py_atomic_load_int(&_PyRuntime.signals.is_tripped)) {

_Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1);

}

}

}

if (tstate->async_exc != NULL) {

_Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1);

}

}

static inline void

SET_GIL_DROP_REQUEST(PyInterpreterState *interp)

{

_Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1);

}

static inline void

RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)

{

_Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0);

}

static inline void

SIGNAL_PENDING_CALLS(PyInterpreterState *interp)

{

_Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1);

}

static inline void

UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)

{

_Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0);

}

#include <stdlib.h>

#include <errno.h>

#include "condvar.h"

#define MUTEX_INIT(mut) \

if (PyMUTEX_INIT(&(mut))) { \

Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };

#define MUTEX_FINI(mut) \

if (PyMUTEX_FINI(&(mut))) { \

Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };

#define MUTEX_LOCK(mut) \

if (PyMUTEX_LOCK(&(mut))) { \

Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };

#define MUTEX_UNLOCK(mut) \

if (PyMUTEX_UNLOCK(&(mut))) { \

Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };

#define COND_INIT(cond) \

if (PyCOND_INIT(&(cond))) { \

Py_FatalError("PyCOND_INIT(" #cond ") failed"); };

#define COND_FINI(cond) \

if (PyCOND_FINI(&(cond))) { \

Py_FatalError("PyCOND_FINI(" #cond ") failed"); };

#define COND_SIGNAL(cond) \

if (PyCOND_SIGNAL(&(cond))) { \

Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };

#define COND_WAIT(cond, mut) \

if (PyCOND_WAIT(&(cond), &(mut))) { \

Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };

#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \

{ \

int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \

if (r < 0) \

Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \

if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \

timeout_result = 1; \

else \

timeout_result = 0; \

} \

#define DEFAULT_INTERVAL 5000

static void _gil_initialize(struct _gil_runtime_state *gil)

{

gil->locked = -1;

gil->interval = DEFAULT_INTERVAL;

}

static int gil_created(struct _gil_runtime_state *gil)

{

if (gil == NULL) {

return 0;

}

return (_Py_atomic_load_int_acquire(&gil->locked) >= 0);

}

static void create_gil(struct _gil_runtime_state *gil)

{

MUTEX_INIT(gil->mutex);

#ifdef FORCE_SWITCHING

MUTEX_INIT(gil->switch_mutex);

COND_INIT(gil->cond);

#ifdef FORCE_SWITCHING

COND_INIT(gil->switch_cond);

_Py_atomic_store_ptr_relaxed(&gil->last_holder, 0);

_Py_ANNOTATE_RWLOCK_CREATE(&gil->locked);

_Py_atomic_store_int_release(&gil->locked, 0);

}

static void destroy_gil(struct _gil_runtime_state *gil)

{

/* some pthread-like implementations tie the mutex to the cond

COND_FINI(gil->cond);

MUTEX_FINI(gil->mutex);

#ifdef FORCE_SWITCHING

COND_FINI(gil->switch_cond);

MUTEX_FINI(gil->switch_mutex);

_Py_atomic_store_int_release(&gil->locked, -1);

_Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

}

#ifdef HAVE_FORK

static void recreate_gil(struct _gil_runtime_state *gil)

{

_Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked);

/* XXX should we destroy the old OS resources here? */

create_gil(gil);

}

static void

drop_gil(PyInterpreterState *interp, PyThreadState *tstate)

{

struct _ceval_state *ceval = &interp->ceval;

/* If tstate is NULL, the caller is indicating that we're releasing

// XXX It may be more correct to check tstate->_status.finalizing.

// XXX assert(tstate == NULL || !tstate->_status.cleared);

struct _gil_runtime_state *gil = ceval->gil;

if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) {

Py_FatalError("drop_gil: GIL is not locked");

}

/* tstate is allowed to be NULL (early interpreter init) */

if (tstate != NULL) {

/* Sub-interpreter support: threads might have been switched

_Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate);

}

MUTEX_LOCK(gil->mutex);

_Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1);

_Py_atomic_store_int_relaxed(&gil->locked, 0);

COND_SIGNAL(gil->cond);

MUTEX_UNLOCK(gil->mutex);

#ifdef FORCE_SWITCHING

/* We check tstate first in case we might be releasing the GIL for

if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) {

MUTEX_LOCK(gil->switch_mutex);

/* Not switched yet => wait */

if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

RESET_GIL_DROP_REQUEST(tstate->interp);

/* NOTE: if COND_WAIT does not atomically start waiting when

COND_WAIT(gil->switch_cond, gil->switch_mutex);

}

MUTEX_UNLOCK(gil->switch_mutex);

}

}

static void

take_gil(PyThreadState *tstate)

{

int err = errno;

assert(tstate != NULL);

/* We shouldn't be using a thread state that isn't viable any more. */

// XXX It may be more correct to check tstate->_status.finalizing.

// XXX assert(!tstate->_status.cleared);

if (_PyThreadState_MustExit(tstate)) {

/* bpo-39877: If Py_Finalize() has been called and tstate is not the

PyThread_exit_thread();

}

assert(_PyThreadState_CheckConsistency(tstate));

PyInterpreterState *interp = tstate->interp;

struct _gil_runtime_state *gil = interp->ceval.gil;

/* Check that _PyEval_InitThreads() was called to create the lock */

assert(gil_created(gil));

MUTEX_LOCK(gil->mutex);

if (!_Py_atomic_load_int_relaxed(&gil->locked)) {

goto _ready;

}

int drop_requested = 0;

while (_Py_atomic_load_int_relaxed(&gil->locked)) {

unsigned long saved_switchnum = gil->switch_number;

unsigned long interval = (gil->interval >= 1 ? gil->interval : 1);

int timed_out = 0;

COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);

/* If we timed out and no switch occurred in the meantime, it is time

if (timed_out &&

_Py_atomic_load_int_relaxed(&gil->locked) &&

gil->switch_number == saved_switchnum)

{

if (_PyThreadState_MustExit(tstate)) {

MUTEX_UNLOCK(gil->mutex);

// gh-96387: If the loop requested a drop request in a previous

// iteration, reset the request. Otherwise, drop_gil() can

// block forever waiting for the thread which exited. Drop

// requests made by other threads are also reset: these threads

// may have to request again a drop request (iterate one more

// time).

if (drop_requested) {

RESET_GIL_DROP_REQUEST(interp);

}

PyThread_exit_thread();

}

assert(_PyThreadState_CheckConsistency(tstate));

SET_GIL_DROP_REQUEST(interp);

drop_requested = 1;

}

}

_ready:

#ifdef FORCE_SWITCHING

/* This mutex must be taken before modifying gil->last_holder:

MUTEX_LOCK(gil->switch_mutex);

/* We now hold the GIL */

_Py_atomic_store_int_relaxed(&gil->locked, 1);

_Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1);

if (tstate != (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) {

_Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate);

++gil->switch_number;

}

#ifdef FORCE_SWITCHING

COND_SIGNAL(gil->switch_cond);

MUTEX_UNLOCK(gil->switch_mutex);

if (_PyThreadState_MustExit(tstate)) {

/* bpo-36475: If Py_Finalize() has been called and tstate is not

MUTEX_UNLOCK(gil->mutex);

drop_gil(interp, tstate);

PyThread_exit_thread();

}

assert(_PyThreadState_CheckConsistency(tstate));

RESET_GIL_DROP_REQUEST(interp);

update_eval_breaker_from_thread(interp, tstate);

MUTEX_UNLOCK(gil->mutex);

errno = err;

}

void _PyEval_SetSwitchInterval(unsigned long microseconds)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

struct _gil_runtime_state *gil = interp->ceval.gil;

assert(gil != NULL);

gil->interval = microseconds;

}

unsigned long _PyEval_GetSwitchInterval(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

struct _gil_runtime_state *gil = interp->ceval.gil;

assert(gil != NULL);

return gil->interval;

}

_PyEval_ThreadsInitialized(void)

{

/* XXX This is only needed for an assert in PyGILState_Ensure(),

PyInterpreterState *interp = _PyInterpreterState_Main();

if (interp == NULL) {

return 0;

}

struct _gil_runtime_state *gil = interp->ceval.gil;

return gil_created(gil);

}

PyAPI_FUNC(int)

PyEval_ThreadsInitialized(void)

{

return _PyEval_ThreadsInitialized();

}

static inline int

current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate)

{

if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) != tstate) {

return 0;

}

return _Py_atomic_load_int_relaxed(&gil->locked);

}

static void

init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)

{

assert(gil_created(gil));

interp->ceval.gil = gil;

interp->ceval.own_gil = 0;

}

static void

init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)

{

assert(!gil_created(gil));

create_gil(gil);

assert(gil_created(gil));

interp->ceval.gil = gil;

interp->ceval.own_gil = 1;

}

_PyEval_InitGIL(PyThreadState *tstate, int own_gil)

{

assert(tstate->interp->ceval.gil == NULL);

if (!own_gil) {

/* The interpreter will share the main interpreter's instead. */

PyInterpreterState *main_interp = _PyInterpreterState_Main();

assert(tstate->interp != main_interp);

struct _gil_runtime_state *gil = main_interp->ceval.gil;

init_shared_gil(tstate->interp, gil);

assert(!current_thread_holds_gil(gil, tstate));

}

else {

PyThread_init_thread();

init_own_gil(tstate->interp, &tstate->interp->_gil);

}

// Lock the GIL and mark the current thread as attached.

_PyThreadState_Attach(tstate);

return _PyStatus_OK();

}

_PyEval_FiniGIL(PyInterpreterState *interp)

{

struct _gil_runtime_state *gil = interp->ceval.gil;

if (gil == NULL) {

/* It was already finalized (or hasn't been initialized yet). */

assert(!interp->ceval.own_gil);

return;

}

else if (!interp->ceval.own_gil) {

#ifdef Py_DEBUG

PyInterpreterState *main_interp = _PyInterpreterState_Main();

assert(main_interp != NULL && interp != main_interp);

assert(interp->ceval.gil == main_interp->ceval.gil);

interp->ceval.gil = NULL;

return;

}

if (!gil_created(gil)) {

/* First Py_InitializeFromConfig() call: the GIL doesn't exist

return;

}

destroy_gil(gil);

assert(!gil_created(gil));

interp->ceval.gil = NULL;

}

PyAPI_FUNC(void)

PyEval_InitThreads(void)

{

/* Do nothing: kept for backward compatibility */

}

_PyEval_Fini(void)

{

#ifdef Py_STATS

_Py_PrintSpecializationStats(1);

}

PyAPI_FUNC(void)

PyEval_AcquireLock(void)

{

PyThreadState *tstate = _PyThreadState_GET();

_Py_EnsureTstateNotNULL(tstate);

take_gil(tstate);

}

PyAPI_FUNC(void)

PyEval_ReleaseLock(void)

{

PyThreadState *tstate = _PyThreadState_GET();

/* This function must succeed when the current thread state is NULL.

drop_gil(tstate->interp, tstate);

}

_PyEval_AcquireLock(PyThreadState *tstate)

{

_Py_EnsureTstateNotNULL(tstate);

take_gil(tstate);

}

_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate)

{

/* If tstate is NULL then we do not expect the current thread

assert(tstate == NULL || tstate->interp == interp);

drop_gil(interp, tstate);

}

PyEval_AcquireThread(PyThreadState *tstate)

{

_Py_EnsureTstateNotNULL(tstate);

_PyThreadState_Attach(tstate);

}

PyEval_ReleaseThread(PyThreadState *tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

_PyThreadState_Detach(tstate);

}

#ifdef HAVE_FORK

_PyEval_ReInitThreads(PyThreadState *tstate)

{

assert(tstate->interp == _PyInterpreterState_Main());

struct _gil_runtime_state *gil = tstate->interp->ceval.gil;

if (!gil_created(gil)) {

return _PyStatus_OK();

}

recreate_gil(gil);

take_gil(tstate);

struct _pending_calls *pending = &tstate->interp->ceval.pending;

if (_PyThread_at_fork_reinit(&pending->lock) < 0) {

return _PyStatus_ERR("Can't reinitialize pending calls lock");

}

/* Destroy all threads except the current one */

_PyThreadState_DeleteExcept(tstate);

return _PyStatus_OK();

}

_PyEval_SignalAsyncExc(PyInterpreterState *interp)

{

_Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1);

}

PyThreadState *

PyEval_SaveThread(void)

{

PyThreadState *tstate = _PyThreadState_GET();

_PyThreadState_Detach(tstate);

return tstate;

}

PyEval_RestoreThread(PyThreadState *tstate)

{

_Py_EnsureTstateNotNULL(tstate);

_PyThreadState_Attach(tstate);

}

_PyEval_SignalReceived(PyInterpreterState *interp)

{

if (_Py_ThreadCanHandleSignals(interp)) {

_Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1);

}

}

static int

_push_pending_call(struct _pending_calls *pending,

_Py_pending_call_func func, void *arg, int flags)

{

int i = pending->last;

int j = (i + 1) % NPENDINGCALLS;

if (j == pending->first) {

return -1; /* Queue full */

}

pending->calls[i].func = func;

pending->calls[i].arg = arg;

pending->calls[i].flags = flags;

pending->last = j;

assert(pending->calls_to_do < NPENDINGCALLS);

pending->calls_to_do++;

return 0;

}

static int

_next_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg, int *flags)

{

int i = pending->first;

if (i == pending->last) {

/* Queue empty */

assert(pending->calls[i].func == NULL);

return -1;

}

*func = pending->calls[i].func;

*arg = pending->calls[i].arg;

*flags = pending->calls[i].flags;

return i;

}

static void

_pop_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg, int *flags)

{

int i = _next_pending_call(pending, func, arg, flags);

if (i >= 0) {

pending->calls[i] = (struct _pending_call){0};

pending->first = (i + 1) % NPENDINGCALLS;

assert(pending->calls_to_do > 0);

pending->calls_to_do--;

}

}

_PyEval_AddPendingCall(PyInterpreterState *interp,

_Py_pending_call_func func, void *arg, int flags)

{

assert(!(flags & _Py_PENDING_MAINTHREADONLY)

|| _Py_IsMainInterpreter(interp));

struct _pending_calls *pending = &interp->ceval.pending;

if (flags & _Py_PENDING_MAINTHREADONLY) {

/* The main thread only exists in the main interpreter. */

assert(_Py_IsMainInterpreter(interp));

pending = &_PyRuntime.ceval.pending_mainthread;

}

/* Ensure that _PyEval_InitState() was called

assert(pending->lock != NULL);

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

int result = _push_pending_call(pending, func, arg, flags);

PyThread_release_lock(pending->lock);

/* signal main loop */

SIGNAL_PENDING_CALLS(interp);

return result;

}

Py_AddPendingCall(_Py_pending_call_func func, void *arg)

{

/* Legacy users of this API will continue to target the main thread

PyInterpreterState *interp = _PyInterpreterState_Main();

return _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY);

}

static int

handle_signals(PyThreadState *tstate)

{

assert(_PyThreadState_CheckConsistency(tstate));

_Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0);

if (!_Py_ThreadCanHandleSignals(tstate->interp)) {

return 0;

}

if (_PyErr_CheckSignalsTstate(tstate) < 0) {

/* On failure, re-schedule a call to handle_signals(). */

_Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1);

return -1;

}

return 0;

}

static int

_make_pending_calls(struct _pending_calls *pending)

{

/* perform a bounded number of calls, in case of recursion */

for (int i=0; i<NPENDINGCALLS; i++) {

_Py_pending_call_func func = NULL;

void *arg = NULL;

int flags = 0;

/* pop one item off the queue while holding the lock */

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

_pop_pending_call(pending, &func, &arg, &flags);

PyThread_release_lock(pending->lock);

/* having released the lock, perform the callback */

if (func == NULL) {

break;

}

int res = func(arg);

if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) {

PyMem_RawFree(arg);

}

if (res != 0) {

return -1;

}

}

return 0;

}

static int

make_pending_calls(PyInterpreterState *interp)

{

struct _pending_calls *pending = &interp->ceval.pending;

struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread;

/* Only one thread (per interpreter) may run the pending calls

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

if (pending->busy) {

/* A pending call was added after another thread was already

PyThread_release_lock(pending->lock);

return 0;

}

pending->busy = 1;

PyThread_release_lock(pending->lock);

/* unsignal before starting to call callbacks, so that any callback

UNSIGNAL_PENDING_CALLS(interp);

if (_make_pending_calls(pending) != 0) {

pending->busy = 0;

/* There might not be more calls to make, but we play it safe. */

SIGNAL_PENDING_CALLS(interp);

return -1;

}

if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) {

if (_make_pending_calls(pending_main) != 0) {

pending->busy = 0;

/* There might not be more calls to make, but we play it safe. */

SIGNAL_PENDING_CALLS(interp);

return -1;

}

}

pending->busy = 0;

return 0;

}

_Py_FinishPendingCalls(PyThreadState *tstate)

{

assert(PyGILState_Check());

assert(_PyThreadState_CheckConsistency(tstate));

if (make_pending_calls(tstate->interp) < 0) {

PyObject *exc = _PyErr_GetRaisedException(tstate);

PyErr_BadInternalCall();

_PyErr_ChainExceptions1(exc);

_PyErr_Print(tstate);

}

}

_PyEval_MakePendingCalls(PyThreadState *tstate)

{

int res;

if (_Py_IsMainThread() && _Py_IsMainInterpreter(tstate->interp)) {

/* Python signal handler doesn't really queue a callback:

res = handle_signals(tstate);

if (res != 0) {

return res;

}

}

res = make_pending_calls(tstate->interp);

if (res != 0) {

return res;

}

return 0;

}

Py_MakePendingCalls(void)

{

assert(PyGILState_Check());

PyThreadState *tstate = _PyThreadState_GET();

assert(_PyThreadState_CheckConsistency(tstate));

/* Only execute pending calls on the main thread. */

if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(tstate->interp)) {

return 0;

}

return _PyEval_MakePendingCalls(tstate);

}

_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock)

{

_gil_initialize(&interp->_gil);

struct _pending_calls *pending = &interp->ceval.pending;

assert(pending->lock == NULL);

pending->lock = pending_lock;

}

_PyEval_FiniState(struct _ceval_state *ceval)

{

struct _pending_calls *pending = &ceval->pending;

if (pending->lock != NULL) {

PyThread_free_lock(pending->lock);

pending->lock = NULL;

}

}

_Py_HandlePending(PyThreadState *tstate)

{

PyInterpreterState *interp = tstate->interp;

/* Pending signals */

if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) {

if (handle_signals(tstate) != 0) {

return -1;

}

}

/* Pending calls */

if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) {

if (make_pending_calls(interp) != 0) {

return -1;

}

}

/* GC scheduled to run */

if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) {

_Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0);

_Py_RunGC(tstate);

}

/* GIL drop request */

if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) {

/* Give another thread a chance */