#ifdef WITH_DTRACE

#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0)

#define OR_DTRACE_LINE

#ifdef HAVE_COMPUTED_GOTOS

#ifndef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 1

#endif

#if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS

#error "Computed gotos are not supported on this compiler."

#endif

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#ifdef Py_STATS

#define INSTRUCTION_STATS(op) \

do { \

PyStats *s = _PyStats_GET(); \

OPCODE_EXE_INC(op); \

if (s) s->opcode_stats[lastopcode].pair_count[op]++; \

lastopcode = op; \

} while (0)

#define INSTRUCTION_STATS(op) ((void)0)

#ifdef Py_STATS

# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode

# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode

# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg

# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg

#if _Py_TAIL_CALL_INTERP

# if defined(__clang__) || defined(__GNUC__)

# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail)

# error "This compiler does not have support for efficient tail calling."

# elif defined(_MSC_VER) && (_MSC_VER < 1950)

# error "You need at least VS 2026 / PlatformToolset v145 for tail calling."

# if defined(_MSC_VER) && !defined(__clang__)

# define Py_MUSTTAIL [[msvc::musttail]]

# define Py_PRESERVE_NONE_CC __preserve_none

# define Py_MUSTTAIL __attribute__((musttail))

# define Py_PRESERVE_NONE_CC __attribute__((preserve_none))

typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS);

# define DISPATCH_TABLE_VAR instruction_funcptr_table

# define DISPATCH_TABLE instruction_funcptr_handler_table

# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table

# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS)

# define DISPATCH_GOTO() \

do { \

Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \

} while (0)

# define DISPATCH_GOTO_NON_TRACING() \

do { \

Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \

} while (0)

# define JUMP_TO_LABEL(name) \

do { \

Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \

} while (0)

# ifdef Py_STATS

# define JUMP_TO_PREDICTED(name) \

do { \

Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \

} while (0)

# define JUMP_TO_PREDICTED(name) \

do { \

Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \

} while (0)

# define LABEL(name) TARGET(name)

#elif USE_COMPUTED_GOTOS

# define DISPATCH_TABLE_VAR opcode_targets

# define DISPATCH_TABLE opcode_targets_table

# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table

# define TARGET(op) TARGET_##op:

# define DISPATCH_GOTO() goto *opcode_targets[opcode]

# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode];

# define JUMP_TO_LABEL(name) goto name;

# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name;

# define LABEL(name) name:

# define TARGET(op) case op: TARGET_##op:

# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode

# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode

# define JUMP_TO_LABEL(name) goto name;

# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name;

# define LABEL(name) name:

#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2

# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE)

# define ENTER_TRACING() \

DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;

# define LEAVE_TRACING() \

DISPATCH_TABLE_VAR = DISPATCH_TABLE;

# define IS_JIT_TRACING() (tracing_mode != 0)

# define ENTER_TRACING() tracing_mode = 255

# define LEAVE_TRACING() tracing_mode = 0

#if _Py_TIER2

#define STOP_TRACING() \

do { \

if (IS_JIT_TRACING()) { \

LEAVE_TRACING(); \

_PyJit_FinalizeTracing(tstate, 0); \

} \

} while (0);

#define STOP_TRACING() ((void)(0));

#ifdef Py_DEBUG

#define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \

lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); }

#define PRE_DISPATCH_GOTO() ((void)0)

#ifdef Py_DEBUG

#define LLTRACE_RESUME_FRAME() \

do { \

_PyFrame_SetStackPointer(frame, stack_pointer); \

int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \

stack_pointer = _PyFrame_GetStackPointer(frame); \

frame->lltrace = lltrace; \

} while (0)

#define LLTRACE_RESUME_FRAME() ((void)0)

#ifdef Py_GIL_DISABLED

#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr)

#define QSBR_QUIESCENT_STATE(tstate)

#define DISPATCH() \

{ \

_PyFrame_StackAssertInvalid(frame); \

NEXTOPARG(); \

PRE_DISPATCH_GOTO(); \

DISPATCH_GOTO(); \

}

#define DISPATCH_NON_TRACING() \

{ \

_PyFrame_StackAssertInvalid(frame); \

NEXTOPARG(); \

PRE_DISPATCH_GOTO(); \

DISPATCH_GOTO_NON_TRACING(); \

}

#define DISPATCH_SAME_OPARG() \

{ \

opcode = next_instr->op.code; \

PRE_DISPATCH_GOTO(); \

DISPATCH_GOTO_NON_TRACING(); \

}

#define DISPATCH_INLINED(NEW_FRAME) \

do { \

assert(!IS_PEP523_HOOKED(tstate)); \

_PyFrame_SetStackPointer(frame, stack_pointer); \

_PyFrame_StackPointerValidate(frame); \

assert((NEW_FRAME)->previous == frame); \

frame = tstate->current_frame = (NEW_FRAME); \

CALL_STAT_INC(inlined_py_calls); \

JUMP_TO_LABEL(start_frame); \

} while (0)

#ifndef Py_DEBUG

#define GETITEM(v, i) PyTuple_GET_ITEM((v), (i))

static inline PyObject *

GETITEM(PyObject *v, Py_ssize_t i) {

assert(PyTuple_Check(v));

assert(i >= 0);

assert(i < PyTuple_GET_SIZE(v));

return PyTuple_GET_ITEM(v, i);

}

#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame)))

#define NEXTOPARG() do { \

_Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \

opcode = word.op.code; \

oparg = word.op.arg; \

} while (0)

#define JUMPBY(x) (next_instr += (x))

#define SKIP_OVER(x) (next_instr += (x))

#define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame)))

#define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize)

#define WITHIN_STACK_BOUNDS() \

(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()))

#if defined(Py_DEBUG) && !defined(_Py_JIT)

#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \

assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE()))

#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS

#define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts)

#define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names)

#define LOCALS_ARRAY (frame->localsplus)

#define GETLOCAL(i) (frame->localsplus[i])

#ifdef Py_STATS

#define UPDATE_MISS_STATS(INSTNAME) \

do { \

STAT_INC(opcode, miss); \

STAT_INC((INSTNAME), miss); \

/* The counter is always the first cache entry: */ \

if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \

STAT_INC((INSTNAME), deopt); \

} \

} while (0)

#define UPDATE_MISS_STATS(INSTNAME) ((void)0)

#ifdef Py_GIL_DISABLED

# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex)

# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex)

# define LOCK_OBJECT(op) (1)

# define UNLOCK_OBJECT(op) ((void)0)

#define GLOBALS() frame->f_globals

#define BUILTINS() frame->f_builtins

#define LOCALS() frame->f_locals

#define CONSTS() _PyFrame_GetCode(frame)->co_consts

#define NAMES() _PyFrame_GetCode(frame)->co_names

#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE)

static void dtrace_function_entry(_PyInterpreterFrame *);

static void dtrace_function_return(_PyInterpreterFrame *);

#define DTRACE_FUNCTION_ENTRY() \

if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \

dtrace_function_entry(frame); \

}

#define DTRACE_FUNCTION_RETURN() \

if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \

dtrace_function_return(frame); \

}

#define DTRACE_FUNCTION_ENTRY() ((void)0)

#define DTRACE_FUNCTION_RETURN() ((void)0)

#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \

backoff_counter_triggers(forge_backoff_counter((COUNTER)))

#ifdef Py_GIL_DISABLED

#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \

do { \

_Py_BackoffCounter cnt = (COUNTER); \

if (!backoff_counter_is_unreachable(cnt)) { \

(COUNTER) = advance_backoff_counter(cnt); \

} \

} while (0);

#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \

do { \

_Py_BackoffCounter cnt = (COUNTER); \

if (!backoff_counter_is_unreachable(cnt)) { \

(COUNTER) = pause_backoff_counter(cnt); \

} \

} while (0);

#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \

do { \

(COUNTER) = advance_backoff_counter((COUNTER)); \

} while (0);

#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \

do { \

(COUNTER) = pause_backoff_counter((COUNTER)); \

} while (0);

#ifdef ENABLE_SPECIALIZATION

#define RECORD_BRANCH_TAKEN(bitset, flag) \

FT_ATOMIC_STORE_UINT16_RELAXED( \

bitset, (FT_ATOMIC_LOAD_UINT16_RELAXED(bitset) << 1) | (flag))

#define RECORD_BRANCH_TAKEN(bitset, flag)

#define UNBOUNDLOCAL_ERROR_MSG \

"cannot access local variable '%s' where it is not associated with a value"

#define UNBOUNDFREE_ERROR_MSG \

"cannot access free variable '%s' where it is not associated with a value" \

" in enclosing scope"

#define NAME_ERROR_MSG "name '%.200s' is not defined"

#define INSTRUMENTED_JUMP(src, dest, event) \

do { \

_Py_CODEUNIT *_dest = (dest); \

if (tstate->tracing) {\

next_instr = _dest; \

} else { \

_PyFrame_SetStackPointer(frame, stack_pointer); \

next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \

stack_pointer = _PyFrame_GetStackPointer(frame); \

if (next_instr == NULL) { \

next_instr = _dest + 1; \

JUMP_TO_LABEL(error); \

} \

} \

} while (0);

static inline int _Py_EnterRecursivePy(PyThreadState *tstate) {

return (tstate->py_recursion_remaining-- <= 0) &&

_Py_CheckRecursiveCallPy(tstate);

}

static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) {

tstate->py_recursion_remaining++;

}

#define LOAD_IP(OFFSET) do { \

next_instr = frame->instr_ptr + (OFFSET); \

} while (0)

#define LOAD_SP() \

stack_pointer = _PyFrame_GetStackPointer(frame)

#define SAVE_SP() \

_PyFrame_SetStackPointer(frame, stack_pointer)

#define TIER1_TO_TIER2(EXECUTOR) \

do { \

OPT_STAT_INC(traces_executed); \

next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \

frame = tstate->current_frame; \

stack_pointer = _PyFrame_GetStackPointer(frame); \

int keep_tracing_bit = (uintptr_t)next_instr & 1; \

next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \

if (next_instr == NULL) { \

/* gh-140104: The exception handler expects frame->instr_ptr

to after this_instr, not this_instr! */ \

next_instr = frame->instr_ptr + 1; \

JUMP_TO_LABEL(error); \

} \

if (keep_tracing_bit) { \

assert(uop_buffer_length(&((_PyThreadStateImpl *)tstate)->jit_tracer_state->code_buffer)); \

ENTER_TRACING(); \

DISPATCH_NON_TRACING(); \

} \

DISPATCH(); \

} while (0)

#define TIER2_TO_TIER2(EXECUTOR) \

do { \

OPT_STAT_INC(traces_executed); \

current_executor = (EXECUTOR); \

goto tier2_start; \

} while (0)

#define GOTO_TIER_ONE_SETUP \

#define GOTO_TIER_ONE(TARGET) \

do \

{ \

GOTO_TIER_ONE_SETUP \

return (_Py_CODEUNIT *)(TARGET); \

} while (0)

#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \

do \

{ \

GOTO_TIER_ONE_SETUP \

return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \

} while (0)

#define CURRENT_OPARG() (next_uop[-1].oparg)

#define CURRENT_OPERAND0_64() (next_uop[-1].operand0)

#define CURRENT_OPERAND1_64() (next_uop[-1].operand1)

#define CURRENT_OPERAND0_32() (next_uop[-1].operand0)

#define CURRENT_OPERAND1_32() (next_uop[-1].operand1)

#define CURRENT_OPERAND0_16() (next_uop[-1].operand0)

#define CURRENT_OPERAND1_16() (next_uop[-1].operand1)

#define CURRENT_TARGET() (next_uop[-1].target)

#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target

#define JUMP_TO_ERROR() goto jump_to_error_target

#define MAX_STACKREF_SCRATCH 10

#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \

/* +1 because vectorcall might use -1 to write self */ \

PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \

PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp);

#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \

/* +1 because we +1 previously */ \

_PyObjectArray_Free(NAME - 1, NAME##_temp);

#define CONVERSION_FAILED(NAME) ((NAME) == NULL)

#if defined(Py_DEBUG) && !defined(_Py_JIT)

#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N)

#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N))

#define SET_CURRENT_CACHED_VALUES(N) ((void)0)

#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0)

#define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL)

static inline int

check_periodics(PyThreadState *tstate) {

_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();

QSBR_QUIESCENT_STATE(tstate);

if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {

return _Py_HandlePending(tstate);

}

return 0;

}

static inline int

check_periodics_at_end(PyThreadState *tstate, _PyInterpreterFrame *frame) {

_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();

QSBR_QUIESCENT_STATE(tstate);

if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {

// Do not handle pending interrupts if the previous instruction was LOAD_SPECIAL

// This may also not handle interrupts if a cache looks like LOAD_SPECIAL,

// but this is benign as we won't skip periodic checks indefinitely.

if (frame->instr_ptr[-1].op.code == LOAD_SPECIAL) {

return 0;

}

return _Py_HandlePending(tstate);

}

return 0;

}

static inline bool

gen_try_set_executing(PyGenObject *gen)

{

#ifdef Py_GIL_DISABLED

if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) {

int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state);

while (frame_state < FRAME_SUSPENDED_YIELD_FROM_LOCKED) {

if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state,

&frame_state,

FRAME_EXECUTING)) {

return true;

}

}

// NB: We return false for FRAME_SUSPENDED_YIELD_FROM_LOCKED as well.

// That case is rare enough that we can just handle it in the deopt.

return false;

}

// Use faster non-atomic modifications in the GIL-enabled build and when

// the object is uniquely referenced in the free-threaded build.

if (gen->gi_frame_state < FRAME_EXECUTING) {

assert(gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED);

gen->gi_frame_state = FRAME_EXECUTING;

return true;

}

return false;

}

#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \

do { \

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \

assert(PyFloat_CheckExact(left_o)); \

assert(PyFloat_CheckExact(right_o)); \

assert(_PyObject_IsUniquelyReferenced( \

PyStackRef_AsPyObjectBorrow(TARGET))); \

STAT_INC(BINARY_OP, hit); \

double _dres = \

((PyFloatObject *)left_o)->ob_fval \

OP ((PyFloatObject *)right_o)->ob_fval; \

((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \

->ob_fval = _dres; \

} while (0)

#define FLOAT_INPLACE_DIVOP(left, right, TARGET) \

int _divop_err = 0; \

do { \

PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \

PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \

assert(PyFloat_CheckExact(left_o)); \

assert(PyFloat_CheckExact(right_o)); \

assert(_PyObject_IsUniquelyReferenced( \

PyStackRef_AsPyObjectBorrow(TARGET))); \

STAT_INC(BINARY_OP, hit); \

double _divisor = ((PyFloatObject *)right_o)->ob_fval; \

if (_divisor == 0.0) { \

PyErr_SetString(PyExc_ZeroDivisionError, \

"float division by zero"); \

_divop_err = 1; \

break; \

} \

double _dres = ((PyFloatObject *)left_o)->ob_fval / _divisor; \

((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \

->ob_fval = _dres; \

} while (0)

#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \

_PyStackRef _int_inplace_res = PyStackRef_NULL; \

do { \

PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \

if (_Py_IsImmortal(target_o)) { \

break; \

} \

assert(_PyObject_IsUniquelyReferenced(target_o)); \

Py_ssize_t left_val = _PyLong_CompactValue( \

(PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \

Py_ssize_t right_val = _PyLong_CompactValue( \

(PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \

Py_ssize_t result = left_val OP right_val; \

if (!_PY_IS_SMALL_INT(result) \

&& ((twodigits)((stwodigits)result) + PyLong_MASK \

< (twodigits)PyLong_MASK + PyLong_BASE)) \

{ \

_PyLong_SetSignAndDigitCount( \

(PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \

((PyLongObject *)target_o)->long_value.ob_digit[0] = \

(digit)(result < 0 ? -result : result); \

_int_inplace_res = PyStackRef_DUP(TARGET); \

break; \

} \

} while (0); \

if (PyStackRef_IsNull(_int_inplace_res)) { \

_int_inplace_res = FUNC( \

(PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \

(PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \

}

#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \

Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX))